BFM-decomp/tools/match_protos.py

#!/usr/bin/env python3
"""Cross-binary function correspondence for the BFM prototype spike (Phase 3.5).

Reads the per-function signature dumps produced by
``tools/ghidra_scripts/DumpFunctionSignatures.java`` (``.run/sig.<prog>.jsonl``)
and reports, for each prototype vs retail, how many retail functions have a
counterpart at three tiers:

  exact : identical raw instruction bytes (strict)
  norm  : identical MIPS-normalized stream (same source, relocation-insensitive)
  seq   : identical mnemonic-only sequence (loose; same ops, any regalloc/consts)

Counts are reported at two size thresholds -- all functions (>=1 instruction) and
"substantial" functions (>= MIN_SUBSTANTIAL instructions) -- so trivial-stub
collisions (e.g. ``jr ra; nop``) don't inflate the headline number. The trustworthy
output is the set of **1:1 unique h_norm correspondences** (a retail function whose
normalized stream matches exactly one proto function and vice-versa); these are
written to ``.run/correspondence.<proto>-vs-retail.tsv`` for T4/T6.

A small anchor sanity-check confirms the normalizer by checking known retail
functions (LZSS decoder, game-mode dispatch, loader) map into each proto.

Pure stdlib. Read-only. R12: outputs under ``.run/``.
"""
from __future__ import annotations

import json
from collections import defaultdict
from pathlib import Path

RUN = Path(__file__).resolve().parents[1] / ".run"        # tools/ -> repo/.run
CONFIG = Path(__file__).resolve().parents[1] / "config"   # committed symbol records
MIN_SUBSTANTIAL = 8

RETAIL = "SLUS_007.26"
PROTOS = ["sep8_SLUS_007.26", "aug31_USA_DEMO.EXE"]
SHORT = {"sep8_SLUS_007.26": "sep8", "aug31_USA_DEMO.EXE": "demo"}

# Known retail anchors (lowercase 0x) for the normalizer sanity check.
ANCHORS = {
    "0x80018730": "LzssDecodeSector",
    "0x80010b40": "GameModeDispatch",
    "0x80011144": "DebugMenuHandler",
    "0x8001971c": "LoaderInitFileTable",
    "0x80019a24": "CdReadRequest",
    "0x8002b154": "SaveLoadRoutine",
}


def load(prog: str) -> list[dict]:
    p = RUN / f"sig.{prog}.jsonl"
    rows = [json.loads(line) for line in p.read_text().splitlines() if line.strip()]
    return [r for r in rows if r["nins"] > 0]  # drop GTE-macro stubs / empty


def index(rows: list[dict], key: str) -> dict[str, list[dict]]:
    d: dict[str, list[dict]] = defaultdict(list)
    for r in rows:
        d[r[key]].append(r)
    return d


def emit_proto_symbols(proto: str, pairs: list[tuple[dict, dict]]) -> tuple[Path, int]:
    """Write config/symbols.proto-<short>.txt: retail names mapped onto the proto via
    1:1 h_norm correspondence. PROTO-SIDE ONLY, provenance-tagged (R13) -- these are
    inherited-from-retail labels to make the proto navigable for Phase-6/Gen2, and are
    UNVERIFIED against the proto until independently confirmed. They are NOT retail
    symbols and must never feed config/symbols.us.txt. Only meaningful (non-default)
    retail names are emitted; FUN_/LAB_ defaults are noise and dropped."""
    short = SHORT.get(proto, proto)
    named = [(r, p) for r, p in pairs
             if not (r["name"].startswith("FUN_") or r["name"].startswith("LAB_"))]
    named.sort(key=lambda x: int(x[1]["addr"], 16))
    lines = [
        f"// config/symbols.proto-{short}.txt — retail symbol names mapped onto the {proto} prototype.",
        "// Phase 3.5: generated by tools/match_protos.py from 1:1 h_norm correspondences (NOT byte-matches).",
        "// PROVENANCE (R13): each name is INHERITED FROM RETAIL via normalized-instruction equality and is",
        "//   UNVERIFIED against the proto until independently confirmed. Proto-side navigation aid for",
        "//   Phase-6/Gen2 ONLY. These are NOT retail symbols; never merge into config/symbols.us.txt.",
        f"// count={len(named)}",
        "",
    ]
    for r, p in named:
        lines.append(f'{r["name"]:32} = 0x{int(p["addr"], 16):08X}; // func  <- retail {r["addr"]} (norm-1to1, unverified)')
    out = CONFIG / f"symbols.proto-{short}.txt"
    out.write_text("\n".join(lines) + "\n")
    return out, len(named)


def main() -> int:
    retail = load(RETAIL)
    print(f"retail {RETAIL}: {len(retail)} functions (nins>0)\n")

    for proto in PROTOS:
        prows = load(proto)
        print(f"===== {proto} vs retail =====")
        print(f"  proto functions (nins>0): {len(prows)}")
        for tier in ("h_exact", "h_norm", "h_seq"):
            pset = {r[tier] for r in prows}
            for thr, label in ((1, "all"), (MIN_SUBSTANTIAL, f">={MIN_SUBSTANTIAL}ins")):
                rsub = [r for r in retail if r["nins"] >= thr]
                m = sum(1 for r in rsub if r[tier] in pset)
                pct = 100 * m / max(1, len(rsub))
                print(f"    {tier:8} {label:9}: {m:5}/{len(rsub):5} retail funcs matched ({pct:5.1f}%)")

        # 1:1 unique h_norm correspondences (the trustworthy pairs)
        rN, pN = index(retail, "h_norm"), index(prows, "h_norm")
        pairs = [(rs[0], ps[0]) for h, rs in rN.items()
                 if (ps := pN.get(h)) and len(rs) == 1 and len(ps) == 1]
        sub_pairs = [(r, p) for r, p in pairs if r["nins"] >= MIN_SUBSTANTIAL]
        print(f"  1:1 unique h_norm correspondences: {len(pairs)} (>= {MIN_SUBSTANTIAL}ins: {len(sub_pairs)})")

        out = RUN / f"correspondence.{proto}-vs-retail.tsv"
        with out.open("w") as fh:
            fh.write("retail_addr\tretail_name\tproto_addr\tproto_name\tnins\ttier\n")
            for r, p in sorted(pairs, key=lambda x: x[0]["addr"]):
                fh.write(f'{r["addr"]}\t{r["name"]}\t{p["addr"]}\t{p["name"]}\t{r["nins"]}\tnorm-1to1\n')
        print(f"  -> {out}")
        sym_out, nsym = emit_proto_symbols(proto, pairs)
        print(f"  -> {sym_out} ({nsym} named retail funcs mapped onto the proto)")

        # anchor sanity check
        pN_all, pE_all = index(prows, "h_norm"), index(prows, "h_exact")
        by_addr = {r["addr"].lower(): r for r in retail}
        print("  anchor sanity (known retail func -> proto match?):")
        for a, nm in ANCHORS.items():
            r = by_addr.get(a)
            if not r:
                print(f"    {a} {nm:20}: (not in retail dump)")
                continue
            em, nmatch = pE_all.get(r["h_exact"]), pN_all.get(r["h_norm"])
            tag = "EXACT" if em else ("norm " if nmatch else "  -  ")
            where = (em or nmatch or [{}])[0].get("addr", "")
            print(f"    {a} {nm:20} nins={r['nins']:4}  [{tag}] {('-> ' + where) if where else '(no match)'}")
        print()
    return 0


if __name__ == "__main__":
    raise SystemExit(main())