Initial commit

tools/diff.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+import argparse
+from colorama import Fore, Style
+import subprocess
+import sys
+import utils
+
+parser = argparse.ArgumentParser(description="Diff assembly")
+parser.add_argument("function", help="Name of the function to diff. Pass | to get a WIP function", nargs="?", default="|")
+args, unknown = parser.parse_known_args()
+
+find_wip = args.function == "|"
+
+for info in utils.get_functions():
+    addr_end = info.addr + info.size
+
+    if info.name == args.function or info.decomp_name == args.function or (find_wip and info.status == utils.FunctionStatus.Wip):
+        if not info.decomp_name:
+            utils.fail(f"{args.function} has not been decompiled")
+
+        subprocess.call(["asm-differ", "-e", info.decomp_name, "0x%016x" %
+                         info.addr, "0x%016x" % addr_end] + unknown)
+
+        if info.status == utils.FunctionStatus.NonMatching:
+            utils.warn(
+                f"{args.function} is marked as non-matching and possibly NOT functionally equivalent")
+        elif info.status == utils.FunctionStatus.Equivalent:
+            utils.warn(f"{args.function} is marked as functionally equivalent but non-matching")
+
+        if find_wip:
+            print(
+                f"WIP function name: {Style.BRIGHT}{Fore.BLUE}{info.decomp_name}{Style.RESET_ALL}")
+
+        sys.exit(0)
+
+if find_wip:
+    utils.fail("no WIP function")
+
+utils.fail(
+    f"unknown function '{args.function}'\nfor constructors and destructors, list the complete object constructor (C1) or destructor (D1)")

tools/diff_settings.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+def apply(config, args):
+    config['arch'] = 'aarch64'
+    config['baseimg'] = 'data/main.elf'
+    config['myimg'] = 'build/uking.elf'
+    config['source_directories'] = ['src']
+    config['objdump_executable'] = 'aarch64-linux-gnu-objdump'

tools/print_decomp_symbols.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+import argparse
+from colorama import Fore, Style
+import diff_settings
+import subprocess
+import utils
+
+parser = argparse.ArgumentParser(description="Prints build/uking.elf symbols")
+parser.add_argument("--print-undefined", "-u",
+                    help="Print symbols that are undefined", action="store_true")
+parser.add_argument("--print-c2-d2", "-c",
+                    help="Print C2/D2 (base object constructor/destructor) symbols", action="store_true")
+parser.add_argument("--hide-unknown", "-H",
+                    help="Hide symbols that are not present in the original game", action="store_true")
+parser.add_argument("--all", "-a", action="store_true")
+args = parser.parse_args()
+
+
+listed_decomp_symbols = {info.decomp_name for info in utils.get_functions()}
+original_symbols = {info.name for info in utils.get_functions()}
+
+config: dict = dict()
+diff_settings.apply(config, {})
+myimg: str = config["myimg"]
+
+entries = [x.strip().split() for x in subprocess.check_output(["nm", myimg], text=True).split("\n")]
+
+for entry in entries:
+    if len(entry) == 3:
+        addr = int(entry[0], 16)
+        symbol_type: str = entry[1]
+        name = entry[2]
+
+        if (symbol_type == "T" or symbol_type == "W") and (args.all or name not in listed_decomp_symbols):
+            c1_name = name.replace("C2", "C1")
+            is_c2_ctor = "C2" in name and c1_name in listed_decomp_symbols and utils.are_demangled_names_equal(
+                c1_name, name)
+
+            d1_name = name.replace("D2", "D1")
+            is_d2_dtor = "D2" in name and d1_name in listed_decomp_symbols and utils.are_demangled_names_equal(
+                d1_name, name)
+
+            if args.print_c2_d2 or not (is_c2_ctor or is_d2_dtor):
+                color = Fore.YELLOW
+                if name in original_symbols:
+                    color = Fore.RED
+                elif args.hide_unknown:
+                    continue
+                if is_c2_ctor or is_d2_dtor:
+                    color += Style.DIM
+                print(f"{color}UNLISTED {Fore.RESET} {utils.format_symbol_name(name)}")
+
+    elif len(entry) == 2:
+        symbol_type = entry[0]
+        name = entry[1]
+
+        if symbol_type.upper() == "U" and args.print_undefined:
+            print(f"{Fore.CYAN}UNDEFINED{Style.RESET_ALL} {utils.format_symbol_name(name)}")

tools/progress.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+import argparse
+from colorama import Back, Fore, Style
+import utils
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--print-nm", "-n", action="store_true",
+                    help="Print non-matching functions")
+parser.add_argument("--print-eq", "-e", action="store_true",
+                    help="Print non-matching, semantically equivalent functions")
+parser.add_argument("--print-ok", "-m", action="store_true",
+                    help="Print matching functions")
+args = parser.parse_args()
+
+
+num_total = 0
+num_matching = 0
+num_equivalent = 0
+num_nonmatching = 0
+num_wip = 0
+
+num_ai_action_done = 0
+num_ai_ai_done = 0
+num_ai_behavior_done = 0
+num_ai_query_done = 0
+
+num_ai_action = 0
+num_ai_ai = 0
+num_ai_behavior = 0
+num_ai_query = 0
+
+for info in utils.get_functions():
+    num_total += 1
+
+    if info.name.startswith("AI_F_Action_"):
+        num_ai_action += 1
+        num_ai_action_done += bool(info.decomp_name)
+    if info.name.startswith("AI_F_AI_"):
+        num_ai_ai += 1
+        num_ai_ai_done += bool(info.decomp_name)
+    if info.name.startswith("AI_F_Behavior_"):
+        num_ai_behavior += 1
+        num_ai_behavior_done += bool(info.decomp_name)
+    if info.name.startswith("AI_F_Query_"):
+        num_ai_query += 1
+        num_ai_query_done += bool(info.decomp_name)
+
+    if not info.decomp_name:
+        continue
+
+    if info.status == utils.FunctionStatus.NonMatching:
+        num_nonmatching += 1
+        if args.print_nm:
+            print(f"{Fore.RED}NM{Fore.RESET} {utils.format_symbol_name(info.decomp_name)}")
+    elif info.status == utils.FunctionStatus.Equivalent:
+        num_equivalent += 1
+        if args.print_eq:
+            print(f"{Fore.YELLOW}EQ{Fore.RESET} {utils.format_symbol_name(info.decomp_name)}")
+    elif info.status == utils.FunctionStatus.Matching:
+        num_matching += 1
+        if args.print_ok:
+            print(f"{Fore.GREEN}OK{Fore.RESET} {utils.format_symbol_name(info.decomp_name)}")
+    elif info.status == utils.FunctionStatus.Wip:
+        num_wip += 1
+        print(f"{Back.RED}{Style.BRIGHT}{Fore.WHITE} WIP {Style.RESET_ALL} {utils.format_symbol_name(info.decomp_name)}{Style.RESET_ALL}")
+
+print()
+print(f"{num_total} functions")
+print(f"{num_matching + num_equivalent} {Fore.CYAN}matching or equivalent{Fore.RESET} ({round(100 * (num_matching + num_equivalent) / num_total, 3)}%)")
+print(f"{num_matching} {Fore.GREEN}matching{Fore.RESET} ({round(100 * num_matching / num_total, 3)}%)")
+print(f"{num_equivalent} {Fore.YELLOW}equivalent{Fore.RESET} ({round(100 * num_equivalent / num_total, 3)}%)")
+print(f"{num_nonmatching} {Fore.RED}non-matching{Fore.RESET} ({round(100 * num_nonmatching / num_total, 3)}%)")
+
+print()
+print(f"{num_ai_action_done}/{num_ai_action} actions ({round(100 * num_ai_action_done / num_ai_action, 3)}%)")
+print(f"{num_ai_ai_done}/{num_ai_ai} AIs ({round(100 * num_ai_ai_done / num_ai_ai, 3)}%)")
+print(f"{num_ai_behavior_done}/{num_ai_behavior} behaviors ({round(100 * num_ai_behavior_done / num_ai_behavior, 3)}%)")
+print(f"{num_ai_query_done}/{num_ai_query} queries ({round(100 * num_ai_query_done / num_ai_query, 3)}%)")

tools/utils.py

@@ -0,0 +1,73 @@
+from colorama import Fore, Style
+import csv
+import cxxfilt
+import enum
+from pathlib import Path
+import sys
+import typing as tp
+
+
+class FunctionStatus(enum.Enum):
+    Matching = 0
+    Equivalent = 1  # semantically equivalent but not perfectly matching
+    NonMatching = 2
+    Wip = 3
+    NotDecompiled = 4
+
+
+class FunctionInfo(tp.NamedTuple):
+    addr: int  # without the 0x7100000000 base
+    name: str
+    size: int
+    decomp_name: str
+    status: FunctionStatus
+
+
+_markers = {
+    "?": FunctionStatus.Equivalent,
+    "!": FunctionStatus.NonMatching,
+    "|": FunctionStatus.Wip,
+}
+
+
+def parse_function_csv_entry(row) -> FunctionInfo:
+    ea, name, size, decomp_name = row
+    if decomp_name:
+        status = FunctionStatus.Matching
+
+        for marker, new_status in _markers.items():
+            if decomp_name[-1] == marker:
+                status = new_status
+                decomp_name = decomp_name[:-1]
+                break
+    else:
+        status = FunctionStatus.NotDecompiled
+
+    addr = int(ea, 16) - 0x7100000000
+    return FunctionInfo(addr, name, int(size, 0), decomp_name, status)
+
+
+def get_functions() -> tp.Iterable[FunctionInfo]:
+    with (Path(__file__).parent.parent / "data" / "uking_functions.csv").open() as f:
+        for row in csv.reader(f):
+            yield parse_function_csv_entry(row)
+
+
+def format_symbol_name(name: str) -> str:
+    try:
+        return f"{cxxfilt.demangle(name)} {Style.DIM}({name}){Style.RESET_ALL}"
+    except:
+        return name
+
+
+def are_demangled_names_equal(name1: str, name2: str):
+    return cxxfilt.demangle(name1) == cxxfilt.demangle(name2)
+
+
+def warn(msg: str):
+    sys.stderr.write(f"{Fore.MAGENTA}{Style.BRIGHT}warning:{Fore.RESET} {msg}{Style.RESET_ALL}\n")
+
+
+def fail(msg: str):
+    sys.stderr.write(f"{Fore.RED}{Style.BRIGHT}error:{Fore.RESET} {msg}{Style.RESET_ALL}\n")
+    sys.exit(1)