diff --git a/requirements.txt b/requirements.txt index 9b17a4bfa9..1c05c047c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyelftools>=0.26 # diff watchdog>=0.10.2 argcomplete -python-Levenshtein +Levenshtein cxxfilt # permuter @@ -20,4 +20,4 @@ pycparser toml # map parsing -mapfile-parser>=1.2.1,<2.0.0 +mapfile-parser>=2.3.5,<3.0.0 diff --git a/tools/check_reordering.py b/tools/check_reordering.py new file mode 100755 index 0000000000..03df306381 --- /dev/null +++ b/tools/check_reordering.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: © 2024 ZeldaRET +# SPDX-License-Identifier: MIT + +import colorama +colorama.init() + +import argparse +import sys +import mapfile_parser +from pathlib import Path + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +def mapPathToSource(origName: Path) -> Path: + # Try to map built path to the source path + parts = origName.parts + if parts[0] == "build": + parts = parts[1:] + + path = Path(*parts) + # Assume every file in the asm folder has .s extension, while everything else has .c extension + if path.parts[0] == "asm": + path = path.with_suffix(".s") + else: + path = path.with_suffix(".c") + return path + +def compareMapFiles(mapFileBuild: Path, mapFileExpected: Path, section: str=".bss", reverseCheck: bool=True) -> mapfile_parser.MapsComparisonInfo: + eprint(f"Build mapfile: {mapFileBuild}") + eprint(f"Expected mapfile: {mapFileExpected}") + eprint("") + + if not mapFileBuild.exists(): + eprint(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: mapfile not found at {mapFileBuild}. Did you forget to build the rom?") + exit(1) + + if not mapFileExpected.exists(): + eprint(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: expected mapfile not found at {mapFileExpected}. A mapfile under the 'expected' folder is required") + exit(1) + + buildMap = mapfile_parser.MapFile() + buildMap.readMapFile(mapFileBuild) + if section != "all": + buildMap = buildMap.filterBySectionType(section) + + expectedMap = mapfile_parser.MapFile() + expectedMap.readMapFile(mapFileExpected) + if section != "all": + expectedMap = expectedMap.filterBySectionType(section) + + return buildMap.compareFilesAndSymbols(expectedMap, checkOtherOnSelf=reverseCheck) + + +def printSymbolComparison(comparisonInfo: mapfile_parser.MapsComparisonInfo, printAll = True): + print("Symbol Name,Build Address,Build File,Expected Address,Expected File,Difference,GOOD/BAD/MISSING") + + # If it's bad or missing, don't need to do anything special. + # If it's good, check for if it's in a file with bad or missing stuff, and check if print all is on. If none of these, print it. + + for symbolInfo in comparisonInfo.comparedList: + buildFile = symbolInfo.buildFile.filepath if symbolInfo.buildFile is not None else None + expectedFile = symbolInfo.expectedFile.filepath if symbolInfo.expectedFile is not None else None + + buildFileName = "" + if buildFile is not None: + buildFileName = mapPathToSource(buildFile) + + expectedFileName = "" + if expectedFile is not None: + expectedFileName = mapPathToSource(expectedFile) + + symbolGood = colorama.Fore.RED + "BAD" + colorama.Fore.RESET + if symbolInfo.diff is None: + symbolGood = colorama.Fore.YELLOW + "MISSING" + colorama.Fore.RESET + print(f"{symbolInfo.symbol.name},{symbolInfo.buildAddress:X},{buildFileName},{symbolInfo.expectedAddress:X},{expectedFileName},{symbolInfo.diff},{symbolGood}") + continue + + if symbolInfo.diff == 0: + symbolGood = colorama.Fore.GREEN + "GOOD" + colorama.Fore.RESET + if not buildFile in comparisonInfo.badFiles and not expectedFile in comparisonInfo.badFiles: + if not buildFile in comparisonInfo.badFiles and not expectedFile in comparisonInfo.badFiles: + if not printAll: + continue + + if buildFile != expectedFile: + symbolGood += colorama.Fore.CYAN + " MOVED" + colorama.Fore.RESET + print(f"{symbolInfo.symbol.name},{symbolInfo.buildAddress:X},{buildFileName},{symbolInfo.expectedAddress:X},{expectedFileName},{symbolInfo.diff:X},{symbolGood}") + + +def printFileComparison(comparisonInfo: mapfile_parser.MapsComparisonInfo, fun_allowed: bool = True): + eprint("") + + if len(comparisonInfo.badFiles) != 0: + print(colorama.Fore.RED + " BAD" + colorama.Style.RESET_ALL) + + for file in comparisonInfo.badFiles: + eprint(f"Symbol reordering in {mapPathToSource(file.filepath)}") + eprint("") + + if fun_allowed: + eprint(colorama.Fore.LIGHTWHITE_EX + + " Symbols are REORDERED!!\n" + " Oh! MY GOD!!" + + colorama.Style.RESET_ALL) + eprint("") + + if len(comparisonInfo.missingFiles) != 0: + print(colorama.Fore.YELLOW + " MISSING" + colorama.Style.RESET_ALL) + + for file in comparisonInfo.missingFiles: + eprint(f"Symbols missing from {mapPathToSource(file.filepath)}") + eprint("") + + if fun_allowed: + eprint(colorama.Fore.LIGHTWHITE_EX + " Error, should (not) be in here " + colorama.Style.RESET_ALL) + eprint("") + + eprint("Some files appear to be missing symbols. Have they been renamed or declared as static? You may need to remake 'expected'") + + +def main(): + description = "Check that globally visible symbols has not been reordered." + epilog = """\ + N.B. Since this script reads the map files, it can only see globally visible symbols; in-function static symbols must be examined with other tools. + """ + + parser = argparse.ArgumentParser(description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("-v", "--version", help="MM version to check. Defaults to n64-us", default="n64-us") + parser.add_argument("-a", "--print-all", help="Print all symbols of the section, not just non-matching.", action="store_true") + parser.add_argument("-n", "--no-fun-allowed", help="Remove amusing messages.", action="store_true") + parser.add_argument("-r", "--no-reverse-check", help="Disable looking for symbols on the expected map that are missing on the built map file.", action="store_true") + parser.add_argument("-s", "--section", help="Specify which section should be checked for reordered symbols. Use `all` to check all sections. Defaults to .bss", default=".bss", choices=[".text", ".data", ".rodata", ".bss", "all"]) + args = parser.parse_args() + + mapfilePath = Path(f"build/{args.version}/mm-{args.version}.map") + mapfileExpectedPath = "expected" / mapfilePath + reverseCheck: bool = not args.no_reverse_check + section: str = args.section + + comparisonInfo = compareMapFiles(mapfilePath, mapfileExpectedPath, section, reverseCheck) + printSymbolComparison(comparisonInfo, args.print_all) + + if len(comparisonInfo.badFiles) + len(comparisonInfo.missingFiles) != 0: + printFileComparison(comparisonInfo, not args.no_fun_allowed) + return 1 + + eprint("") + eprint(colorama.Fore.GREEN + " GOOD" + colorama.Style.RESET_ALL) + + if args.no_fun_allowed: + return 0 + + eprint("\n" + colorama.Fore.LIGHTWHITE_EX + + colorama.Back.RED + f" " + colorama.Back.RESET + "\n" + + colorama.Back.RED + f" CONGRATURATIONS! " + colorama.Back.RESET + "\n" + + colorama.Back.RED + "{:^34}".format(f"All Global {section if section != 'all' else 'Symbols'} is correct.") + colorama.Back.RESET + "\n" + + colorama.Back.RED + f" THANK YOU! " + colorama.Back.RESET + "\n" + + colorama.Back.RED + f" You are great decomper! " + colorama.Back.RESET + "\n" + + colorama.Back.RED + f" " + colorama.Style.RESET_ALL ) + + return 0 + +if __name__ == "__main__": + ret = main() + exit(ret) diff --git a/tools/global_bss_check.py b/tools/global_bss_check.py deleted file mode 100755 index 8f04493e0e..0000000000 --- a/tools/global_bss_check.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python3 -import colorama -colorama.init() - -import argparse -import os -import re -import collections - -regex_fileDataEntry = re.compile(r"^\s+(?P
[^\s]+)\s+(?P0x[^\s]+)\s+(?P0x[^\s]+)\s+(?P[^\s]+)$") -regex_bssEntry = re.compile(r"^\s+(?P0x[^\s]+)\s+(?P[^\s]+)$") -regex_label = re.compile(r"^(?PL[0-9A-F]{8})$") - -VarInfo = collections.namedtuple("VarInfo", ["file", "vram"]) - -def parseMapFile(mapPath: str): - with open(mapPath) as f: - mapData = f.read() - startIndex = mapData.find("..makerom") - mapData = mapData[startIndex:] - # print(len(mapData)) - - symbolsDict = collections.OrderedDict() - - inFile = False - currentFile = "" - - mapLines = mapData.split("\n") - for line in mapLines: - if inFile: - if line.startswith(" "): - entryMatch = regex_bssEntry.search(line) - - # Find variable - if entryMatch is not None: - varName = entryMatch["name"] - varVram = int(entryMatch["vram"], 16) - - # Filter out jump table labels - labelMatch = regex_label.search(varName) - if labelMatch is None: - symbolsDict[varName] = VarInfo( currentFile, varVram ) - # print( symbolsDict[varName] ) - - else: - inFile = False - else: - if line.startswith(" .bss "): - inFile = False - entryMatch = regex_fileDataEntry.search(line) - - # Find file - if entryMatch is not None: - name = "/".join(entryMatch["name"].split("/")[1:]) - - # mapfile only contains .o files, so just strip the last character to replace it - # we assume all the .c files are in the src folder, and all others are .s (true for OoT/MM) - if name.split("/")[0] == "src": - name = name[:-1] + "c" - else: - name = name[:-1] + "s" - - size = int(entryMatch["size"], 16) - vram = int(entryMatch["vram"], 16) - - if size > 0: - inFile = True - currentFile = name - - return symbolsDict - - -Compared = collections.namedtuple("Compared", [ "buildAddress", "buildFile", "expectedAddress", "expectedFile", "diff"]) - -def compareMapFiles(mapFileBuild: str, mapFileExpected: str): - badFiles = set() - missingFiles = set() - - print("Build mapfile: " + mapFileBuild, file=os.sys.stderr) - print("Expected mapfile: " + mapFileExpected, file=os.sys.stderr) - print("", file=os.sys.stderr) - - if not os.path.exists(mapFileBuild): - print(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: mapfile not found at {mapFileBuild}. Did you enter the correct path?", file=os.sys.stderr) - exit(1) - - if not os.path.exists(mapFileExpected): - print(f"{colorama.Fore.LIGHTRED_EX}error{colorama.Fore.RESET}: expected mapfile not found at {mapFileExpected}. Is 'expected' missing or in a different folder?", file=os.sys.stderr) - exit(1) - - buildMap = parseMapFile(mapFileBuild) - expectedMap = parseMapFile(mapFileExpected) - - comparedDict = collections.OrderedDict() - - for symbol in buildMap: - if symbol in expectedMap: - comparedDict[symbol] = Compared( buildMap[symbol].vram, buildMap[symbol].file, expectedMap[symbol].vram, expectedMap[symbol].file, buildMap[symbol].vram - expectedMap[symbol].vram ) - if comparedDict[symbol].diff != 0: - badFiles.add(buildMap[symbol].file) - - else: - missingFiles.add(buildMap[symbol].file) - comparedDict[symbol] = Compared( buildMap[symbol].vram, buildMap[symbol].file, -1, "", "Unknown" ) - - for symbol in expectedMap: - if not symbol in buildMap: - missingFiles.add(expectedMap[symbol].file) - comparedDict[symbol] = Compared( -1, "", expectedMap[symbol].vram, expectedMap[symbol].file, "Unknown" ) - - return badFiles, missingFiles, comparedDict - - -def printCsv(badFiles, missingFiles, comparedDict, printAll = True): - print("Symbol Name,Build Address,Build File,Expected Address,Expected File,Difference,GOOD/BAD/MISSING") - - # If it's bad or missing, don't need to do anything special. - # If it's good, check for if it's in a file with bad or missing stuff, and check if print all is on. If none of these, print it. - - for symbol in comparedDict: - symbolInfo = comparedDict[symbol] - symbolGood = colorama.Fore.RED + "BAD" + colorama.Fore.RESET - if type(symbolInfo.diff) != int: - symbolGood = colorama.Fore.YELLOW + "MISSING" + colorama.Fore.RESET - print(f"{symbol},{symbolInfo.buildAddress:X},{symbolInfo.buildFile},{symbolInfo.expectedAddress:X},{symbolInfo.expectedFile},{symbolInfo.diff},{symbolGood}") - continue - - if symbolInfo.diff == 0: - symbolGood = colorama.Fore.GREEN + "GOOD" + colorama.Fore.RESET - if (not symbolInfo.buildFile in badFiles and not symbolInfo.expectedFile in badFiles) and (not symbolInfo.buildFile in badFiles and not symbolInfo.expectedFile in badFiles) and not printAll: - continue - - if symbolInfo.buildFile != symbolInfo.expectedFile: - symbolGood += colorama.Fore.CYAN + " MOVED" + colorama.Fore.RESET - print(f"{symbol},{symbolInfo.buildAddress:X},{symbolInfo.buildFile},{symbolInfo.expectedAddress:X},{symbolInfo.expectedFile},{symbolInfo.diff:X},{symbolGood}") - - -def main(): - description = "Check that globally visible bss has not been reordered." - epilog = """\ - N.B. Since this script reads the map files, it can only see globally visible bss; in-function static bss must be examined with other tools. - """ - - parser = argparse.ArgumentParser(description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument("mapFile", help="Path to a map file.") - parser.add_argument("mapFileExpected", help="Path to the expected map file. Optional, default is 'expected/mapFile'.", nargs="?", default="") - parser.add_argument("-a", "--print-all", help="Print all bss, not just non-matching.", action="store_true") - parser.add_argument("-n", "--no-fun-allowed", help="Remove amusing messages.", action="store_true") - args = parser.parse_args() - - if args.mapFileExpected == "": - args.mapFileExpected = os.path.join("expected", args.mapFile) - - badFiles, missingFiles, comparedDict = compareMapFiles(args.mapFile, args.mapFileExpected) - printCsv(badFiles, missingFiles, comparedDict, args.print_all) - - if len(badFiles) + len(missingFiles) != 0: - print("", file=os.sys.stderr) - - if len(badFiles) != 0: - print(colorama.Fore.RED + " BAD" + colorama.Style.RESET_ALL) - - for file in badFiles: - print(f"bss reordering in {file}", file=os.sys.stderr) - print("", file=os.sys.stderr) - - if not args.no_fun_allowed: - print(colorama.Fore.LIGHTWHITE_EX + - " BSS is REORDERED!!\n" - " Oh! MY GOD!!" - + colorama.Style.RESET_ALL, file=os.sys.stderr) - print("", file=os.sys.stderr) - - if len(missingFiles) != 0: - print(colorama.Fore.YELLOW + " MISSING" + colorama.Style.RESET_ALL) - - for file in missingFiles: - print(f"Symbols missing from {file}", file=os.sys.stderr) - print("", file=os.sys.stderr) - - if not args.no_fun_allowed: - print(colorama.Fore.LIGHTWHITE_EX + " Error, should (not) be in here " + colorama.Style.RESET_ALL, file=os.sys.stderr) - print("", file=os.sys.stderr) - - print("Some files appear to be missing symbols. Have they been renamed or declared as static? You may need to remake 'expected'", file=os.sys.stderr) - - return 1 - - print("", file=os.sys.stderr) - print(colorama.Fore.GREEN + " GOOD" + colorama.Style.RESET_ALL, file=os.sys.stderr) - - if args.no_fun_allowed: - return 0 - - print("\n" + colorama.Fore.LIGHTWHITE_EX + - colorama.Back.RED + " " + colorama.Back.RESET + "\n" + - colorama.Back.RED + " CONGRATURATIONS! " + colorama.Back.RESET + "\n" + - colorama.Back.RED + " All Global BSS is correct. " + colorama.Back.RESET + "\n" + - colorama.Back.RED + " THANK YOU! " + colorama.Back.RESET + "\n" + - colorama.Back.RED + " You are great decomper! " + colorama.Back.RESET + "\n" + - colorama.Back.RED + " " + colorama.Style.RESET_ALL , file=os.sys.stderr) - - return 0 - -if __name__ == "__main__": - ret = main() - exit(ret)