mirror/mm
1
0
Fork 0
mirror of https://github.com/zeldaret/mm.git synced 2026-07-08 21:34:48 -04:00
Code Issues Packages Projects Releases Wiki Activity

ZAPD fixes in sys_initial_check, update subrepos (#507)

Browse Source 
* git subrepo pull --force tools/ZAPD

subrepo:
  subdir:   "tools/ZAPD"
  merged:   "a3363333d"
upstream:
  origin:   "https://github.com/zeldaret/ZAPD.git"
  branch:   "master"
  commit:   "a3363333d"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo.git"
  commit:   "2f68596"

* git subrepo pull tools/asm-differ --force

subrepo:
  subdir:   "tools/asm-differ"
  merged:   "70c33cc12"
upstream:
  origin:   "https://github.com/simonlindholm/asm-differ.git"
  branch:   "main"
  commit:   "70c33cc12"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo.git"
  commit:   "2f68596"

* git subrepo pull (merge) tools/z64compress --force

subrepo:
  subdir:   "tools/z64compress"
  merged:   "ac5b1a0d0"
upstream:
  origin:   "https://github.com/z64me/z64compress.git"
  branch:   "main"
  commit:   "ac5b1a0d0"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo.git"
  commit:   "2f68596"

* Use defines for texture sizes in sys_initial_check

* Update extract_assets.py

* Add null check

* git subrepo pull --force tools/ZAPD

subrepo:
  subdir:   "tools/ZAPD"
  merged:   "50242eca9"
upstream:
  origin:   "https://github.com/zeldaret/ZAPD.git"
  branch:   "master"
  commit:   "50242eca9"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo.git"
  commit:   "2f68596"
This commit is contained in:
EllipticEllipsis
2021-12-16 04:05:29 +00:00
committed by GitHub
parent ff231e093a
commit 9ca4ec7604
82 changed files with 1646 additions and 757 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/asm-differ/diff.py
+187 -117
View File
@@ -640,6 +640,7 @@ class Text:
class TableMetadata:
headers: Tuple[Text, ...]
current_score: int
max_score: int
previous_score: Optional[int]
@@ -832,6 +833,7 @@ class JsonFormatter(Formatter):
for h, name in zip(meta.headers, ("base", "current", "previous"))
}
output["current_score"] = meta.current_score
output["max_score"] = meta.max_score
if meta.previous_score is not None:
output["previous_score"] = meta.previous_score
output_rows: List[Dict[str, Any]] = []
@@ -1184,8 +1186,9 @@ def parse_elf_data_references(data: bytes) -> List[Tuple[int, int, str]]:
assert len(symtab_sections) == 1
symtab = sections[symtab_sections[0]]
text_sections = [i for i in range(e_shnum) if sec_names[i] == b".text"]
assert len(text_sections) == 1
text_sections = [i for i in range(e_shnum) if sec_names[i] == b".text" and sections[i].sh_size != 0]
if len(text_sections) != 1:
return []
text_section = text_sections[0]
ret: List[Tuple[int, int, str]] = []
@@ -1327,11 +1330,23 @@ def dump_binary(
(objdump_flags + flags2, project.myimg, None),
)
DATA_POOL_PLACEHOLDER = "DATA_POOL_PLACEHOLDER-OFFSET_{}"
DATA_POOL_PLACEHOLDER_PATTERN = re.compile(
r"DATA_POOL_PLACEHOLDER-OFFSET_([a-zA-z0-9]+)"
)
class DifferenceNormalizer:
# Example: "ldr r4, [pc, #56] ; (4c <AddCoins+0x4c>)"
ARM32_LOAD_POOL_PATTERN = r"(ldr\s+r([0-9]|1[0-3]),\s+\[pc,.*;\s*)(\([a-fA-F0-9]+.*\))"
# The base class is a no-op.
class AsmProcessor:
def __init__(self, config: Config) -> None:
self.config = config
def process_reloc(self, row: str, prev: str) -> str:
return prev
def normalize(self, mnemonic: str, row: str) -> str:
"""This should be called exactly once for each line."""
arch = self.config.arch
@@ -1342,9 +1357,143 @@ class DifferenceNormalizer:
def _normalize_arch_specific(self, mnemonic: str, row: str) -> str:
return row
def post_process(self, lines: List["Line"]) -> None:
return
class DifferenceNormalizerAArch64(DifferenceNormalizer):
class AsmProcessorMIPS(AsmProcessor):
def process_reloc(self, row: str, prev: str) -> str:
arch = self.config.arch
if "R_MIPS_NONE" in row:
# GNU as emits no-op relocations immediately after real ones when
# assembling with -mabi=64. Return without trying to parse 'imm' as an
# integer.
return prev
before, imm, after = parse_relocated_line(prev)
repl = row.split()[-1]
if imm != "0":
# MIPS uses relocations with addends embedded in the code as immediates.
# If there is an immediate, show it as part of the relocation. Ideally
# we'd show this addend in both %lo/%hi, but annoyingly objdump's output
# doesn't include enough information to pair up %lo's and %hi's...
# TODO: handle unambiguous cases where all addends for a symbol are the
# same, or show "+???".
mnemonic = prev.split()[0]
if (
mnemonic in arch.instructions_with_address_immediates
and not imm.startswith("0x")
):
imm = "0x" + imm
repl += "+" + imm if int(imm, 0) > 0 else imm
if "R_MIPS_LO16" in row:
repl = f"%lo({repl})"
elif "R_MIPS_HI16" in row:
# Ideally we'd pair up R_MIPS_LO16 and R_MIPS_HI16 to generate a
# correct addend for each, but objdump doesn't give us the order of
# the relocations, so we can't find the right LO16. :(
repl = f"%hi({repl})"
elif "R_MIPS_26" in row:
# Function calls
pass
elif "R_MIPS_PC16" in row:
# Branch to glabel. This gives confusing output, but there's not much
# we can do here.
pass
else:
assert False, f"unknown relocation type '{row}' for line '{prev}'"
return before + repl + after
class AsmProcessorPPC(AsmProcessor):
def process_reloc(self, row: str, prev: str) -> str:
arch = self.config.arch
assert any(
r in row for r in ["R_PPC_REL24", "R_PPC_ADDR16", "R_PPC_EMB_SDA21"]
), f"unknown relocation type '{row}' for line '{prev}'"
before, imm, after = parse_relocated_line(prev)
repl = row.split()[-1]
if "R_PPC_REL24" in row:
# function calls
pass
elif "R_PPC_ADDR16_HI" in row:
# absolute hi of addr
repl = f"{repl}@h"
elif "R_PPC_ADDR16_HA" in row:
# adjusted hi of addr
repl = f"{repl}@ha"
elif "R_PPC_ADDR16_LO" in row:
# lo of addr
repl = f"{repl}@l"
elif "R_PPC_ADDR16" in row:
# 16-bit absolute addr
if "+0x7" in repl:
# remove the very large addends as they are an artifact of (label-_SDA(2)_BASE_)
# computations and are unimportant in a diff setting.
if int(repl.split("+")[1], 16) > 0x70000000:
repl = repl.split("+")[0]
elif "R_PPC_EMB_SDA21" in row:
# small data area
pass
return before + repl + after
class AsmProcessorARM32(AsmProcessor):
def process_reloc(self, row: str, prev: str) -> str:
arch = self.config.arch
before, imm, after = parse_relocated_line(prev)
repl = row.split()[-1]
return before + repl + after
def _normalize_arch_specific(self, mnemonic: str, row: str) -> str:
if self.config.ignore_addr_diffs:
row = self._normalize_bl(mnemonic, row)
row = self._normalize_data_pool(row)
return row
def _normalize_bl(self, mnemonic: str, row: str) -> str:
if mnemonic != "bl":
return row
row, _ = split_off_address(row)
return row + "<ignore>"
def _normalize_data_pool(self, row: str) -> str:
pool_match = re.search(ARM32_LOAD_POOL_PATTERN, row)
if pool_match:
offset = pool_match.group(3).split(" ")[0][1:]
repl = DATA_POOL_PLACEHOLDER.format(offset)
return pool_match.group(1) + repl
return row
def post_process(self, lines: List["Line"]) -> None:
lines_by_line_number = {}
for line in lines:
lines_by_line_number[line.line_num] = line
for line in lines:
reloc_match = re.search(
DATA_POOL_PLACEHOLDER_PATTERN, line.normalized_original
)
if reloc_match is None:
continue
# Get value at relocation
reloc = reloc_match.group(0)
line_number = re.search(
DATA_POOL_PLACEHOLDER_PATTERN, reloc).group(1)
line_original = lines_by_line_number[int(line_number, 16)].original
value = line_original.split()[1]
# Replace relocation placeholder with value
replaced = re.sub(
DATA_POOL_PLACEHOLDER_PATTERN,
f"={value} ({line_number})",
line.normalized_original,
)
line.original = replaced
class AsmProcessorAArch64(AsmProcessor):
def __init__(self, config: Config) -> None:
super().__init__(config)
self._adrp_pair_registers: Set[str] = set()
@@ -1394,23 +1543,6 @@ class DifferenceNormalizerAArch64(DifferenceNormalizer):
return row
class DifferenceNormalizerARM32(DifferenceNormalizer):
def __init__(self, config: Config) -> None:
super().__init__(config)
def _normalize_arch_specific(self, mnemonic: str, row: str) -> str:
if self.config.ignore_addr_diffs:
row = self._normalize_bl(mnemonic, row)
return row
def _normalize_bl(self, mnemonic: str, row: str) -> str:
if mnemonic != "bl":
return row
row, _ = split_off_address(row)
return row + "<ignore>"
@dataclass
class ArchSettings:
name: str
@@ -1420,14 +1552,15 @@ class ArchSettings:
re_sprel: Pattern[str]
re_large_imm: Pattern[str]
re_imm: Pattern[str]
re_reloc: Pattern[str]
branch_instructions: Set[str]
instructions_with_address_immediates: Set[str]
forbidden: Set[str] = field(default_factory=lambda: set(string.ascii_letters + "_"))
arch_flags: List[str] = field(default_factory=list)
branch_likely_instructions: Set[str] = field(default_factory=set)
difference_normalizer: Type[DifferenceNormalizer] = DifferenceNormalizer
proc: Type[AsmProcessor] = AsmProcessor
big_endian: Optional[bool] = True
delay_slot_instructions: Set[str] = field(default_factory=set)
MIPS_BRANCH_LIKELY_INSTRUCTIONS = {
"beql",
@@ -1543,10 +1676,13 @@ MIPS_SETTINGS = ArchSettings(
re_sprel=re.compile(r"(?<=,)([0-9]+|0x[0-9a-f]+)\(sp\)"),
re_large_imm=re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}"),
re_imm=re.compile(r"(\b|-)([0-9]+|0x[0-9a-fA-F]+)\b(?!\(sp)|%(lo|hi)\([^)]*\)"),
re_reloc=re.compile(r"R_MIPS_"),
arch_flags=["-m", "mips:4300"],
branch_likely_instructions=MIPS_BRANCH_LIKELY_INSTRUCTIONS,
branch_instructions=MIPS_BRANCH_INSTRUCTIONS,
instructions_with_address_immediates=MIPS_BRANCH_INSTRUCTIONS.union({"jal", "j"}),
delay_slot_instructions=MIPS_BRANCH_INSTRUCTIONS.union({"j", "jal", "jr", "jalr"}),
proc=AsmProcessorMIPS,
)
MIPSEL_SETTINGS = replace(MIPS_SETTINGS, name="mipsel", big_endian=False)
@@ -1566,9 +1702,10 @@ ARM32_SETTINGS = ArchSettings(
re_sprel=re.compile(r"sp, #-?(0x[0-9a-fA-F]+|[0-9]+)\b"),
re_large_imm=re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}"),
re_imm=re.compile(r"(?<!sp, )#-?(0x[0-9a-fA-F]+|[0-9]+)\b"),
re_reloc=re.compile(r"R_ARM_"),
branch_instructions=ARM32_BRANCH_INSTRUCTIONS,
instructions_with_address_immediates=ARM32_BRANCH_INSTRUCTIONS.union({"adr"}),
difference_normalizer=DifferenceNormalizerARM32,
proc=AsmProcessorARM32,
)
AARCH64_SETTINGS = ArchSettings(
@@ -1581,9 +1718,10 @@ AARCH64_SETTINGS = ArchSettings(
re_sprel=re.compile(r"sp, #-?(0x[0-9a-fA-F]+|[0-9]+)\b"),
re_large_imm=re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}"),
re_imm=re.compile(r"(?<!sp, )#-?(0x[0-9a-fA-F]+|[0-9]+)\b"),
re_reloc=re.compile(r"R_AARCH64_"),
branch_instructions=AARCH64_BRANCH_INSTRUCTIONS,
instructions_with_address_immediates=AARCH64_BRANCH_INSTRUCTIONS.union({"bl", "adrp"}),
difference_normalizer=DifferenceNormalizerAArch64,
proc=AsmProcessorAArch64,
)
PPC_SETTINGS = ArchSettings(
@@ -1594,8 +1732,10 @@ PPC_SETTINGS = ArchSettings(
re_sprel=re.compile(r"(?<=,)(-?[0-9]+|-?0x[0-9a-f]+)\(r1\)"),
re_large_imm=re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}"),
re_imm=re.compile(r"(\b|-)([0-9]+|0x[0-9a-fA-F]+)\b(?!\(r1)|[^@]*@(ha|h|lo)"),
re_reloc=re.compile(r"R_PPC_"),
branch_instructions=PPC_BRANCH_INSTRUCTIONS,
instructions_with_address_immediates=PPC_BRANCH_INSTRUCTIONS.union({"bl"}),
proc=AsmProcessorPPC,
)
ARCH_SETTINGS = [
@@ -1639,84 +1779,6 @@ def parse_relocated_line(line: str) -> Tuple[str, str, str]:
return before, imm, after
def process_mips_reloc(row: str, prev: str, arch: ArchSettings) -> str:
if "R_MIPS_NONE" in row:
# GNU as emits no-op relocations immediately after real ones when
# assembling with -mabi=64. Return without trying to parse 'imm' as an
# integer.
return prev
before, imm, after = parse_relocated_line(prev)
repl = row.split()[-1]
if imm != "0":
# MIPS uses relocations with addends embedded in the code as immediates.
# If there is an immediate, show it as part of the relocation. Ideally
# we'd show this addend in both %lo/%hi, but annoyingly objdump's output
# doesn't include enough information to pair up %lo's and %hi's...
# TODO: handle unambiguous cases where all addends for a symbol are the
# same, or show "+???".
mnemonic = prev.split()[0]
if (
mnemonic in arch.instructions_with_address_immediates
and not imm.startswith("0x")
):
imm = "0x" + imm
repl += "+" + imm if int(imm, 0) > 0 else imm
if "R_MIPS_LO16" in row:
repl = f"%lo({repl})"
elif "R_MIPS_HI16" in row:
# Ideally we'd pair up R_MIPS_LO16 and R_MIPS_HI16 to generate a
# correct addend for each, but objdump doesn't give us the order of
# the relocations, so we can't find the right LO16. :(
repl = f"%hi({repl})"
elif "R_MIPS_26" in row:
# Function calls
pass
elif "R_MIPS_PC16" in row:
# Branch to glabel. This gives confusing output, but there's not much
# we can do here.
pass
else:
assert False, f"unknown relocation type '{row}' for line '{prev}'"
return before + repl + after
def process_ppc_reloc(row: str, prev: str) -> str:
assert any(
r in row for r in ["R_PPC_REL24", "R_PPC_ADDR16", "R_PPC_EMB_SDA21"]
), f"unknown relocation type '{row}' for line '{prev}'"
before, imm, after = parse_relocated_line(prev)
repl = row.split()[-1]
if "R_PPC_REL24" in row:
# function calls
pass
elif "R_PPC_ADDR16_HI" in row:
# absolute hi of addr
repl = f"{repl}@h"
elif "R_PPC_ADDR16_HA" in row:
# adjusted hi of addr
repl = f"{repl}@ha"
elif "R_PPC_ADDR16_LO" in row:
# lo of addr
repl = f"{repl}@l"
elif "R_PPC_ADDR16" in row:
# 16-bit absolute addr
if "+0x7" in repl:
# remove the very large addends as they are an artifact of (label-_SDA(2)_BASE_)
# computations and are unimportant in a diff setting.
if int(repl.split("+")[1], 16) > 0x70000000:
repl = repl.split("+")[0]
elif "R_PPC_EMB_SDA21" in row:
# small data area
pass
return before + repl + after
def process_arm_reloc(row: str, prev: str, arch: ArchSettings) -> str:
before, imm, after = parse_relocated_line(prev)
repl = row.split()[-1]
return before + repl + after
def pad_mnemonic(line: str) -> str:
if "\t" not in line:
return line
@@ -1741,7 +1803,7 @@ class Line:
def process(dump: str, config: Config) -> List[Line]:
arch = config.arch
normalizer = arch.difference_normalizer(config)
processor = arch.proc(config)
skip_next = False
source_lines = []
source_filename = None
@@ -1783,7 +1845,7 @@ def process(dump: str, config: Config) -> List[Line]:
)
break
if not re.match(r"^ +[0-9a-f]+:\t", row):
if not re.match(r"^\s+[0-9a-f]+:\s+", row):
# This regex is conservative, and assumes the file path does not contain "weird"
# characters like colons, tabs, or angle brackets.
if re.match(
@@ -1797,10 +1859,11 @@ def process(dump: str, config: Config) -> List[Line]:
m_comment = re.search(arch.re_comment, row)
comment = m_comment[0] if m_comment else None
row = re.sub(arch.re_comment, "", row)
line_num_str = row.split(":")[0]
row = row.rstrip()
tabs = row.split("\t")
row = "\t".join(tabs[2:])
line_num = eval_line_num(tabs[0].strip())
line_num = eval_line_num(line_num_str.strip())
if line_num in data_refs:
refs = data_refs[line_num]
@@ -1835,20 +1898,13 @@ def process(dump: str, config: Config) -> List[Line]:
while i < len(lines):
reloc_row = lines[i]
if "R_AARCH64_" in reloc_row:
# TODO: handle relocation
pass
elif "R_MIPS_" in reloc_row:
original = process_mips_reloc(reloc_row, original, arch)
elif "R_PPC_" in reloc_row:
original = process_ppc_reloc(reloc_row, original)
elif "R_ARM_" in reloc_row:
original = process_arm_reloc(reloc_row, original, arch)
if re.search(arch.re_reloc, reloc_row):
original = processor.process_reloc(reloc_row, original)
else:
break
i += 1
normalized_original = normalizer.normalize(mnemonic, original)
normalized_original = processor.normalize(mnemonic, original)
scorable_line = normalized_original
if not config.score_stack_differences:
@@ -1904,6 +1960,7 @@ def process(dump: str, config: Config) -> List[Line]:
elif stop_after_delay_slot:
break
processor.post_process(output)
return output
@@ -2123,8 +2180,15 @@ class OutputLine:
class Diff:
lines: List[OutputLine]
score: int
max_score: int
def trim_nops(lines: List[Line], arch: ArchSettings) -> List[Line]:
lines = lines[:]
while lines and lines[-1].mnemonic == "nop" and (len(lines) == 1 or lines[-2].mnemonic not in arch.delay_slot_instructions):
lines.pop()
return lines
def do_diff(lines1: List[Line], lines2: List[Line], config: Config) -> Diff:
if config.show_source:
import cxxfilt
@@ -2152,8 +2216,12 @@ def do_diff(lines1: List[Line], lines2: List[Line], config: Config) -> Diff:
btset.add(bt)
sc(str(bt))
lines1 = trim_nops(lines1, arch)
lines2 = trim_nops(lines2, arch)
diffed_lines = diff_lines(lines1, lines2, config.algorithm)
score = score_diff_lines(diffed_lines, config)
max_score = len(lines1) * config.penalty_deletion
line_num_base = -1
line_num_offset = 0
@@ -2385,7 +2453,7 @@ def do_diff(lines1: List[Line], lines2: List[Line], config: Config) -> Diff:
)
output = output[config.skip_lines :]
return Diff(lines=output, score=score)
return Diff(lines=output, score=score, max_score=max_score)
def chunk_diff_lines(
@@ -2461,6 +2529,7 @@ def align_diffs(
Text(f"{padding}PREVIOUS ({old_diff.score})"),
),
current_score=new_diff.score,
max_score=new_diff.max_score,
previous_score=old_diff.score,
)
old_chunks = chunk_diff_lines(old_diff.lines)
@@ -2504,6 +2573,7 @@ def align_diffs(
Text(f"{padding}CURRENT ({new_diff.score})"),
),
current_score=new_diff.score,
max_score=new_diff.max_score,
previous_score=None,
)
diff_lines = [(line, line) for line in new_diff.lines]