From ff4a46f35d05783c479cda938b0e8aa89bdb57c2 Mon Sep 17 00:00:00 2001 From: LagoLunatic Date: Sat, 26 Sep 2026 14:24:22 -0400 Subject: [PATCH] debug_map_diff.py: Reorganize code a bit --- tools/utilities/debug_map_diff.py | 143 +++++++++++++++++++----------- 1 file changed, 91 insertions(+), 52 deletions(-) diff --git a/tools/utilities/debug_map_diff.py b/tools/utilities/debug_map_diff.py index 6036f5547..46a5028f7 100755 --- a/tools/utilities/debug_map_diff.py +++ b/tools/utilities/debug_map_diff.py @@ -18,53 +18,63 @@ import argparse from collections import Counter, defaultdict import time -arg_parse = argparse.ArgumentParser() -arg_parse.add_argument("object_name", nargs="?", help="Name of the object to build and diff, e.g. d_a_bridge or d_a_npc_fa1") -arg_parse.add_argument("--all", action='store_true', help="Build and diff all objects") -args = arg_parse.parse_args() -if args.all: - build_all = True -else: - if args.object_name is None: - arg_parse.error("the following arguments are required: object_name (or --all)") - build_all = False - arg_object_name: str = args.object_name - assert not re.search(r"[/\\.]", arg_object_name), "The object name should not contain slashes or dots" +DEBUG_MAPS_ROOT_PATH = Path("orig") / "D44J01" / "files" / "maps" +DECOMP_ROOT_PATH = Path(".") -debug_maps_root_path = Path("orig") / "D44J01" / "files" / "maps" -decomp_root_path = Path(".") +def configure_debug_and_get_all_ninja_outputs(): + retcode = subprocess.call(["python", "configure.py", "--version", "D44J01", "--debug", "--map", "--non-matching"], cwd=DECOMP_ROOT_PATH) + assert retcode == 0, "Failed to configure" + + # Add a short delay between configuring and building to prevent the weird "illegal data in precompiled header" error. + time.sleep(0.5) + + all_ninja_outputs: list[str] = [] + for ninja_target in subprocess.check_output(["ninja", "-t", "targets", "all"]).decode("utf-8").splitlines(): + ninja_output, ninja_rule = ninja_target.split(":", 1) + all_ninja_outputs.append(ninja_output) + + return all_ninja_outputs -retcode = subprocess.call(["python", "configure.py", "--version", "D44J01", "--debug", "--map", "--non-matching"], cwd=decomp_root_path) -assert retcode == 0, "Failed to configure" +def ninja_build_all(): + retcode = subprocess.call(["ninja"], cwd=DECOMP_ROOT_PATH) + assert retcode == 0, "Ninja build call failed" -# Add a short delay between configuring and building to prevent the weird "illegal data in precompiled header" error. -time.sleep(0.5) +def ninja_build_one_map(base_map_path: Path): + retcode = subprocess.call(["ninja", base_map_path.relative_to(DECOMP_ROOT_PATH)], cwd=DECOMP_ROOT_PATH) + assert retcode == 0, "Ninja build call failed" -all_ninja_outputs: list[str] = [] -for ninja_target in subprocess.check_output(["ninja", "-t", "targets", "all"]).decode("utf-8").splitlines(): - ninja_output, ninja_rule = ninja_target.split(":", 1) - all_ninja_outputs.append(ninja_output) - -all_object_names: list[str] = [] -for output_path in all_ninja_outputs: - if not output_path.startswith("build/D44J01/"): - continue - if not output_path.endswith(".o"): - continue - object_name = output_path.rsplit("/", 1)[1].split(".", 1)[0] - all_object_names.append(object_name) +def get_all_object_names_from_ninja_outputs(all_ninja_outputs: list[str]): + all_object_names: list[str] = [] + for output_path in all_ninja_outputs: + if not output_path.startswith("build/D44J01/"): + continue + if not output_path.endswith(".o"): + continue + object_name = output_path.rsplit("/", 1)[1].split(".", 1)[0] + all_object_names.append(object_name) + return all_object_names class Symbol: - def __init__(self, name: str, size: int, sym_type: str | None = None, linkage: str | None = None, stripped: bool | None = None, align: int | None = None): + def __init__( + self, + name: str, + size: int, + section: str, + sym_type: str | None = None, + linkage: str | None = None, + stripped: bool | None = None, + align: int | None = None + ): self.name = name self.size = size + self.section = section self.sym_type = sym_type self.linkage = linkage self.stripped = stripped self.align = align def __repr__(self): - return f"Symbol(name={self.name}, size={self.size}, sym_type={self.sym_type}, linkage={self.linkage}, stripped={self.stripped})" + return f"Symbol(name={self.name}, size={self.size}, section={self.section}, sym_type={self.sym_type}, linkage={self.linkage}, stripped={self.stripped})" @cache def get_symbols_from_linker_map(map_contents: str, missing_tree_and_stripped=False): @@ -147,6 +157,7 @@ def get_symbols_from_linker_map(map_contents: str, missing_tree_and_stripped=Fal symbols: dict[str, dict[str, Symbol]] = defaultdict(dict) unref_dupe_symbol_names_already_added = set() + curr_section_name = None for line in map_lines: symbol_entry_match = re.search(r"^ ([0-9a-f]{8}|UNUSED ) ([0-9a-f]{6}) ([0-9a-f]{8}|\.{8})(?: +(\d+))? (.+?)(?: \(entry of [^)]+\))? \t?(?:(\S+\.a) )?(\S+) ?$", line, re.IGNORECASE) if symbol_entry_match: @@ -177,9 +188,14 @@ def get_symbols_from_linker_map(map_contents: str, missing_tree_and_stripped=Fal symbols[object_name] original_symbol_name = symbol_name - if symbol_name.startswith(".") or symbol_name in ["extab", "extabindex"]: - # e.g. Section symbol (.text) or pool symbol (...data) + if symbol_name.startswith("..."): + # Pool symbol (e.g. ...data) continue + if symbol_name.startswith(".") or symbol_name in ["extab", "extabindex"]: + # Section symbol (e.g. .text) + curr_section_name = symbol_name + continue + assert curr_section_name is not None if re.search(r"^@\d+$", symbol_name): continue if localstatic_match := re.search(r"^([^\s\$]+)\$\d+$", symbol_name): @@ -198,13 +214,13 @@ def get_symbols_from_linker_map(map_contents: str, missing_tree_and_stripped=Fal else: assert stripped or missing_tree_and_stripped, f"Symbol {repr(original_symbol_name)} is missing linkage information in object {repr(object_name)}" symbol_type = linkage = None - symbols[object_name][symbol_name] = Symbol(symbol_name, symbol_size, sym_type=symbol_type, linkage=linkage, stripped=stripped, align=symbol_align) + symbols[object_name][symbol_name] = Symbol(symbol_name, symbol_size, curr_section_name, sym_type=symbol_type, linkage=linkage, stripped=stripped, align=symbol_align) if symbol_name in unref_dupe_symbol_names_to_object_name_to_type_linkage and symbol_name not in unref_dupe_symbol_names_already_added: for other_object_name in unref_dupe_symbol_names_to_object_name_to_type_linkage[symbol_name]: symbol_type, linkage = unref_dupe_symbol_names_to_object_name_to_type_linkage[symbol_name][other_object_name] assert stripped == False, "Shouldn't reach a stripped duplicate here" - symbols[other_object_name][symbol_name] = Symbol(symbol_name, symbol_size, sym_type=symbol_type, linkage=linkage, stripped=stripped, align=symbol_align) + symbols[other_object_name][symbol_name] = Symbol(symbol_name, symbol_size, curr_section_name, sym_type=symbol_type, linkage=linkage, stripped=stripped, align=symbol_align) unref_dupe_symbol_names_already_added.add(symbol_name) return symbols @@ -220,9 +236,9 @@ def should_ignore_missing_symbol(symbol_name: str): return True return False -def diff_debug_map(target_object_name: str, call_ninja: bool, print_size_diffs: bool, print_maybe_fake: bool): - target_map_path_dol = debug_maps_root_path / "frameworkD.map" - target_map_path_rel = debug_maps_root_path / f"{target_object_name}D.map" +def diff_debug_map(target_object_name: str, call_ninja: bool, print_size_diffs: bool, print_maybe_fake: bool, all_ninja_outputs: list[str]): + target_map_path_dol = DEBUG_MAPS_ROOT_PATH / "frameworkD.map" + target_map_path_rel = DEBUG_MAPS_ROOT_PATH / f"{target_object_name}D.map" if target_map_path_rel.exists(): target_is_rel = True target_map_path = target_map_path_rel @@ -233,8 +249,8 @@ def diff_debug_map(target_object_name: str, call_ninja: bool, print_size_diffs: del target_map_path_dol del target_map_path_rel - base_map_path_dol = decomp_root_path / "build" / "D44J01" / "framework.elf.MAP" - base_map_path_rel = decomp_root_path / "build" / "D44J01" / target_object_name / f"{target_object_name}.plf.MAP" + base_map_path_dol = DECOMP_ROOT_PATH / "build" / "D44J01" / "framework.elf.MAP" + base_map_path_rel = DECOMP_ROOT_PATH / "build" / "D44J01" / target_object_name / f"{target_object_name}.plf.MAP" if base_map_path_rel.as_posix() in all_ninja_outputs: base_is_rel = True base_map_path = base_map_path_rel @@ -246,8 +262,7 @@ def diff_debug_map(target_object_name: str, call_ninja: bool, print_size_diffs: del base_map_path_rel if call_ninja: - retcode = subprocess.call(["ninja", base_map_path.relative_to(decomp_root_path)], cwd=decomp_root_path) - assert retcode == 0, "Ninja build call failed" + ninja_build_one_map(base_map_path) target_missing_tree_and_stripped = not target_is_rel @@ -263,6 +278,11 @@ def diff_debug_map(target_object_name: str, call_ninja: bool, print_size_diffs: # print(len(target_symbols), len(base_symbols)) + return target_symbols, base_symbols, all_target_symbols, all_base_symbols, target_missing_tree_and_stripped + +def print_diff_debug_map(target_object_name: str, call_ninja: bool, print_size_diffs: bool, print_maybe_fake: bool, all_ninja_outputs: list[str]): + target_symbols, base_symbols, all_target_symbols, all_base_symbols, target_missing_tree_and_stripped = diff_debug_map(target_object_name, call_ninja, print_size_diffs, print_maybe_fake, all_ninja_outputs) + target_symbol_names_in_previous_objects = set() if target_missing_tree_and_stripped: # This handles the logic for checking if a symbol already appeared earlier on in framework.map. @@ -399,19 +419,20 @@ def diff_debug_map(target_object_name: str, call_ninja: bool, print_size_diffs: return (missing_target_symbols, fake_base_symbols, wrong_linkage_symbols, wrong_align_symbols) -def diff_all_debug_maps(): - retcode = subprocess.call(["ninja"], cwd=decomp_root_path) - assert retcode == 0, "Ninja build call failed" +def diff_all_debug_maps(all_ninja_outputs: list[str]): + ninja_build_all() missing_counts: Counter[str] = Counter() fake_counts: Counter[str] = Counter() wrong_linkage_counts: Counter[str] = Counter() wrong_align_counts: Counter[str] = Counter() + all_object_names = get_all_object_names_from_ninja_outputs(all_ninja_outputs) + for target_object_name in all_object_names: if target_object_name in ["__mem", "exception", "executor"]: continue - missing, fake, wrong_linkage, wrong_align = diff_debug_map(target_object_name, call_ninja=False, print_size_diffs=False, print_maybe_fake=False) + missing, fake, wrong_linkage, wrong_align = print_diff_debug_map(target_object_name, call_ninja=False, print_size_diffs=False, print_maybe_fake=False, all_ninja_outputs=all_ninja_outputs) missing_counts.update(missing) fake_counts.update(fake) wrong_linkage_counts.update(wrong_linkage) @@ -441,8 +462,26 @@ def diff_all_debug_maps(): print(f"Total fake: {sum(fake_counts.values())}") print(f"Total missing: {sum(missing_counts.values())}") -if __name__ == "__main__": - if build_all: - diff_all_debug_maps() +def main(): + arg_parse = argparse.ArgumentParser() + arg_parse.add_argument("object_name", nargs="?", help="Name of the object to build and diff, e.g. d_a_bridge or d_a_npc_fa1") + arg_parse.add_argument("--all", action='store_true', help="Build and diff all objects") + args = arg_parse.parse_args() + if args.all: + build_all = True else: - diff_debug_map(arg_object_name, call_ninja=True, print_size_diffs=True, print_maybe_fake=True) + if args.object_name is None: + arg_parse.error("the following arguments are required: object_name (or --all)") + build_all = False + arg_object_name: str = args.object_name + assert not re.search(r"[/\\.]", arg_object_name), "The object name should not contain slashes or dots" + + all_ninja_outputs = configure_debug_and_get_all_ninja_outputs() + + if build_all: + diff_all_debug_maps(all_ninja_outputs) + else: + print_diff_debug_map(arg_object_name, call_ninja=True, print_size_diffs=True, print_maybe_fake=True, all_ninja_outputs=all_ninja_outputs) + +if __name__ == "__main__": + main()