#!/usr/bin/env python3 """Cross-binary function correspondence for the BFM prototype spike (Phase 3.5). Reads the per-function signature dumps produced by ``tools/ghidra_scripts/DumpFunctionSignatures.java`` (``.run/sig..jsonl``) and reports, for each prototype vs retail, how many retail functions have a counterpart at three tiers: exact : identical raw instruction bytes (strict) norm : identical MIPS-normalized stream (same source, relocation-insensitive) seq : identical mnemonic-only sequence (loose; same ops, any regalloc/consts) Counts are reported at two size thresholds -- all functions (>=1 instruction) and "substantial" functions (>= MIN_SUBSTANTIAL instructions) -- so trivial-stub collisions (e.g. ``jr ra; nop``) don't inflate the headline number. The trustworthy output is the set of **1:1 unique h_norm correspondences** (a retail function whose normalized stream matches exactly one proto function and vice-versa); these are written to ``.run/correspondence.-vs-retail.tsv`` for T4/T6. A small anchor sanity-check confirms the normalizer by checking known retail functions (LZSS decoder, game-mode dispatch, loader) map into each proto. Pure stdlib. Read-only. R12: outputs under ``.run/``. """ from __future__ import annotations import json from collections import defaultdict from pathlib import Path RUN = Path(__file__).resolve().parents[1] / ".run" # tools/ -> repo/.run CONFIG = Path(__file__).resolve().parents[1] / "config" # committed symbol records MIN_SUBSTANTIAL = 8 RETAIL = "SLUS_007.26" PROTOS = ["sep8_SLUS_007.26", "aug31_USA_DEMO.EXE"] SHORT = {"sep8_SLUS_007.26": "sep8", "aug31_USA_DEMO.EXE": "demo"} # Known retail anchors (lowercase 0x) for the normalizer sanity check. ANCHORS = { "0x80018730": "LzssDecodeSector", "0x80010b40": "GameModeDispatch", "0x80011144": "DebugMenuHandler", "0x8001971c": "LoaderInitFileTable", "0x80019a24": "CdReadRequest", "0x8002b154": "SaveLoadRoutine", } def load(prog: str) -> list[dict]: p = RUN / f"sig.{prog}.jsonl" rows = [json.loads(line) for line in p.read_text().splitlines() if line.strip()] return [r for r in rows if r["nins"] > 0] # drop GTE-macro stubs / empty def index(rows: list[dict], key: str) -> dict[str, list[dict]]: d: dict[str, list[dict]] = defaultdict(list) for r in rows: d[r[key]].append(r) return d def emit_proto_symbols(proto: str, pairs: list[tuple[dict, dict]]) -> tuple[Path, int]: """Write config/symbols.proto-.txt: retail names mapped onto the proto via 1:1 h_norm correspondence. PROTO-SIDE ONLY, provenance-tagged (R13) -- these are inherited-from-retail labels to make the proto navigable for Phase-6/Gen2, and are UNVERIFIED against the proto until independently confirmed. They are NOT retail symbols and must never feed config/symbols.us.txt. Only meaningful (non-default) retail names are emitted; FUN_/LAB_ defaults are noise and dropped.""" short = SHORT.get(proto, proto) named = [(r, p) for r, p in pairs if not (r["name"].startswith("FUN_") or r["name"].startswith("LAB_"))] named.sort(key=lambda x: int(x[1]["addr"], 16)) lines = [ f"// config/symbols.proto-{short}.txt — retail symbol names mapped onto the {proto} prototype.", "// Phase 3.5: generated by tools/match_protos.py from 1:1 h_norm correspondences (NOT byte-matches).", "// PROVENANCE (R13): each name is INHERITED FROM RETAIL via normalized-instruction equality and is", "// UNVERIFIED against the proto until independently confirmed. Proto-side navigation aid for", "// Phase-6/Gen2 ONLY. These are NOT retail symbols; never merge into config/symbols.us.txt.", f"// count={len(named)}", "", ] for r, p in named: lines.append(f'{r["name"]:32} = 0x{int(p["addr"], 16):08X}; // func <- retail {r["addr"]} (norm-1to1, unverified)') out = CONFIG / f"symbols.proto-{short}.txt" out.write_text("\n".join(lines) + "\n") return out, len(named) def main() -> int: retail = load(RETAIL) print(f"retail {RETAIL}: {len(retail)} functions (nins>0)\n") for proto in PROTOS: prows = load(proto) print(f"===== {proto} vs retail =====") print(f" proto functions (nins>0): {len(prows)}") for tier in ("h_exact", "h_norm", "h_seq"): pset = {r[tier] for r in prows} for thr, label in ((1, "all"), (MIN_SUBSTANTIAL, f">={MIN_SUBSTANTIAL}ins")): rsub = [r for r in retail if r["nins"] >= thr] m = sum(1 for r in rsub if r[tier] in pset) pct = 100 * m / max(1, len(rsub)) print(f" {tier:8} {label:9}: {m:5}/{len(rsub):5} retail funcs matched ({pct:5.1f}%)") # 1:1 unique h_norm correspondences (the trustworthy pairs) rN, pN = index(retail, "h_norm"), index(prows, "h_norm") pairs = [(rs[0], ps[0]) for h, rs in rN.items() if (ps := pN.get(h)) and len(rs) == 1 and len(ps) == 1] sub_pairs = [(r, p) for r, p in pairs if r["nins"] >= MIN_SUBSTANTIAL] print(f" 1:1 unique h_norm correspondences: {len(pairs)} (>= {MIN_SUBSTANTIAL}ins: {len(sub_pairs)})") out = RUN / f"correspondence.{proto}-vs-retail.tsv" with out.open("w") as fh: fh.write("retail_addr\tretail_name\tproto_addr\tproto_name\tnins\ttier\n") for r, p in sorted(pairs, key=lambda x: x[0]["addr"]): fh.write(f'{r["addr"]}\t{r["name"]}\t{p["addr"]}\t{p["name"]}\t{r["nins"]}\tnorm-1to1\n') print(f" -> {out}") sym_out, nsym = emit_proto_symbols(proto, pairs) print(f" -> {sym_out} ({nsym} named retail funcs mapped onto the proto)") # anchor sanity check pN_all, pE_all = index(prows, "h_norm"), index(prows, "h_exact") by_addr = {r["addr"].lower(): r for r in retail} print(" anchor sanity (known retail func -> proto match?):") for a, nm in ANCHORS.items(): r = by_addr.get(a) if not r: print(f" {a} {nm:20}: (not in retail dump)") continue em, nmatch = pE_all.get(r["h_exact"]), pN_all.get(r["h_norm"]) tag = "EXACT" if em else ("norm " if nmatch else " - ") where = (em or nmatch or [{}])[0].get("addr", "") print(f" {a} {nm:20} nins={r['nins']:4} [{tag}] {('-> ' + where) if where else '(no match)'}") print() return 0 if __name__ == "__main__": raise SystemExit(main())