From 4fdf76cc41f1923f2ad5daffb0c42fa505a4b69c Mon Sep 17 00:00:00 2001 From: LagoLunatic Date: Wed, 8 Jul 2026 13:58:18 -0400 Subject: [PATCH] Several improvements to debug_map_diff.py Support checking linkage diffs, and checking if REL data symbols are missing/misnamed. --- tools/utilities/debug_map_diff.py | 134 +++++++++++++++++++++--------- 1 file changed, 96 insertions(+), 38 deletions(-) diff --git a/tools/utilities/debug_map_diff.py b/tools/utilities/debug_map_diff.py index 678e0ab85..bec1e9866 100755 --- a/tools/utilities/debug_map_diff.py +++ b/tools/utilities/debug_map_diff.py @@ -8,6 +8,7 @@ from pathlib import Path import re import subprocess import argparse +from collections import defaultdict arg_parse = argparse.ArgumentParser() arg_parse.add_argument("object_name", help="Name of the object to compare, e.g. d_a_bridge for a REL, or main/d_a_npc_fa1 for a main.dol object") @@ -35,6 +36,13 @@ assert retcode == 0, "Failed to configure" retcode = subprocess.call(["ninja", base_map_path.relative_to(decomp_root_path)], cwd=decomp_root_path) assert retcode == 0, "Ninja build call failed" +class Symbol: + def __init__(self, name: str, size: int, sym_type: str | None = None, linkage: str | None = None): + self.name = name + self.size = size + self.sym_type = sym_type + self.linkage = linkage + def get_main_symbols(framework_map_contents: str, valid_obj_names = None): symbols = {} matches = re.findall(r"^ [0-9a-f]{8} ([0-9a-f]{6}) (?:[0-9a-f]{8})(?: +\d+)? (.+?)(?: \(entry of [^)]+\))? \t(?:\S+\.a )?([^\s\.]+)\.\S+ ?$", framework_map_contents, re.IGNORECASE | re.MULTILINE) @@ -51,38 +59,63 @@ def get_main_symbols(framework_map_contents: str, valid_obj_names = None): if valid_obj_names is not None and obj_name not in valid_obj_names: continue - symbols[name] = size + symbols[name] = Symbol(name, size) if len(symbols) == 0: raise Exception("Failed to find object matching the given name") return symbols def get_rel_symbols(rel_map_data: str): rel_map_lines = rel_map_data.splitlines() - found_memory_map = False - next_section_index = 0 - section_name_to_section_index = {} - for line in rel_map_lines: - if line.strip() == "Memory map:": - found_memory_map = True - if found_memory_map: - section_match = re.search(r"^ +\.(text|ctors|dtors|rodata|data|bss) [0-9a-f]{8} ([0-9a-f]{8}) [0-9a-f]{8}$", line) - if section_match: - section_name = section_match.group(1) - section_size = int(section_match.group(2), 16) - if section_size > 0: - section_name_to_section_index[section_name] = next_section_index - next_section_index += 1 - if not found_memory_map: - raise Exception("Failed to find memory map") + + line = rel_map_lines.pop(0) + assert line == "Link map of _prolog" + symbol_name_to_type_linkage_obj = {} + while True: + line = rel_map_lines.pop(0) + if line == "": + # End of link map + break + if line.startswith(">>> SYMBOL NOT FOUND: "): + continue + linker_symbol_match = re.search(r"^ +(\d+)\] (\S+) found as linker generated symbol$", line) + if linker_symbol_match: + continue + normal_symbol_match = re.search(r"^ +(\d+)\] (\S+) \((section|func|object|notype),(global|local|weak)\) found in (\S+) $", line) + assert normal_symbol_match is not None, f"Unknown line: {repr(line)}" + link_map_depth = int(normal_symbol_match.group(1)) + symbol_name = normal_symbol_match.group(2) + symbol_type = normal_symbol_match.group(3) + linkage = normal_symbol_match.group(4) + object_name = normal_symbol_match.group(5) + assert symbol_name not in symbol_name_to_type_linkage_obj + symbol_name_to_type_linkage_obj[symbol_name] = (symbol_type, linkage, object_name) + + # found_memory_map = False + # next_section_index = 0 + # section_name_to_section_index = {} + # for line in rel_map_lines: + # if line.strip() == "Memory map:": + # found_memory_map = True + # if found_memory_map: + # section_match = re.search(r"^ +\.(text|ctors|dtors|rodata|data|bss) [0-9a-f]{8} ([0-9a-f]{8}) [0-9a-f]{8}$", line) + # if section_match: + # section_name = section_match.group(1) + # section_size = int(section_match.group(2), 16) + # if section_size > 0: + # section_name_to_section_index[section_name] = next_section_index + # next_section_index += 1 + # if not found_memory_map: + # raise Exception("Failed to find memory map") symbols = {} - current_section_name = None + # current_section_name = None + localstatic_counters = defaultdict(int) for line in rel_map_lines: - section_header_match = re.search(r"^\.(text|ctors|dtors|rodata|data|bss) section layout$", line) - if section_header_match: - current_section_name = section_header_match.group(1) - if current_section_name != "text": - continue + # section_header_match = re.search(r"^\.(text|ctors|dtors|rodata|data|bss) section layout$", line) + # if section_header_match: + # current_section_name = section_header_match.group(1) + # if current_section_name != "text": + # continue symbol_entry_match = re.search(r"^ ([0-9a-f]{8}|UNUSED ) ([0-9a-f]{6}) ([0-9a-f]{8}|\.{8})(?: +\d+)? (.+?)(?: \(entry of [^)]+\))? \t?(\S+)", line, re.IGNORECASE) if symbol_entry_match: symbol_address = symbol_entry_match.group(1) @@ -98,9 +131,21 @@ def get_rel_symbols(rel_map_data: str): symbol_offset = int(symbol_offset, 16) symbol_name = symbol_entry_match.group(4) object_name = symbol_entry_match.group(5) + if object_name in ["executor.o", "global_destructor_chain.o"]: continue - symbols[symbol_name] = symbol_size + if re.search(r"^@\d+$", symbol_name): + continue + if localstatic_match := re.search(r"^([^\s\$]+)\$\d+$", symbol_name): + localstatic_name = localstatic_match.group(1) + localstatic_counters[localstatic_name] += 1 + symbol_name = f"{localstatic_name}${localstatic_counters[localstatic_name]}" + + if symbol_name in symbol_name_to_type_linkage_obj: + symbol_type, linkage, object_name = symbol_name_to_type_linkage_obj[symbol_name] + else: + symbol_type = linkage = object_name = None + symbols[symbol_name] = Symbol(symbol_name, symbol_size, sym_type=symbol_type, linkage=linkage) #print("%08X %s" % (symbol_offset, symbol_name)) #print(rel_symbol_names) @@ -127,18 +172,20 @@ total_missing = 0 total_fake = 0 total_right_size = 0 total_wrong_size = 0 -for symbol_name, target_size in target_symbols.items(): - if target_size == 0: - continue - if symbol_name not in base_symbols: - base_size = 0 - else: - base_size = base_symbols[symbol_name] - size_diff = abs(target_size - base_size) - ratio = size_diff / target_size - if size_diff != 0: - total_wrong_size += 1 - symbol_size_diffs.append((symbol_name, target_size, base_size, ratio)) +total_wrong_linkage = 0 + +for symbol_name, target_symbol in target_symbols.items(): + if target_symbol.size == 0: + continue + if symbol_name not in base_symbols: + base_size = 0 + else: + base_size = base_symbols[symbol_name].size + size_diff = abs(target_symbol.size - base_size) + ratio = size_diff / target_symbol.size + if size_diff != 0: + total_wrong_size += 1 + symbol_size_diffs.append((symbol_name, target_symbol.size, base_size, ratio)) symbol_size_diffs.sort(key=lambda x: x[-1]) @@ -151,13 +198,13 @@ for symbol_name, target_size, base_size, ratio in symbol_size_diffs: elif size_diff == 0: prefix = "GOOD: " total_right_size += 1 - elif ratio < 0.03: + elif ratio < 0.05 and size_diff <= 0x10: prefix = "CLOSE: " else: prefix = "WRONG: " print(prefix + symbol_name, "0x%X" % target_size, "0x%X" % base_size, ratio) -for symbol_name, base_size in base_symbols.items(): +for symbol_name, base_symbol in base_symbols.items(): if symbol_name in target_symbols: continue print("FAKE:", symbol_name) @@ -172,8 +219,19 @@ for symbol_name, target_size, base_size, ratio in symbol_size_diffs: total_missing += 1 print(prefix + symbol_name, "0x%X" % target_size) +for symbol_name, target_symbol in target_symbols.items(): + if target_symbol.size == 0: + continue + if symbol_name not in base_symbols: + continue + base_symbol = base_symbols[symbol_name] + if target_symbol.linkage != base_symbol.linkage: + total_wrong_linkage += 1 + print(f"LINKAGE: {symbol_name} (should be {target_symbol.linkage}, is {base_symbol.linkage})") + print("==================================================") print(f"Total right size: {total_right_size}") print(f"Total wrong size: {total_wrong_size}") print(f"Total fake: {total_fake}") print(f"Total missing: {total_missing}") +print(f"Total wrong linkage: {total_wrong_linkage}")