BFM-decomp/tools/dedup_integrate.py

#!/usr/bin/env python3
"""Cross-binary code-share registry validator (Phase 11, G7 "match once, share").

Validates config/dedup.us.yaml — the registry of functions matched ONCE and shared across >=2
duplicate sites/binaries via a source-level body (a macro in src/shared/), instantiated at each
member site so the same bytes land at each member vram.

WHAT THE REGISTRY CLAIMS, AND HOW EACH CLAIM IS CHECKED  (Phase 26-A tooling audit; R32/R33)
--------------------------------------------------------------------------------------------
C1  EQUIVALENCE — "these member vrams hold the same code IN THE ORIGINAL BINARY."
    Checked against .run/sig.<bin>.jsonl, which signs the ORIGINAL bytes. A genuine EXTERNAL
    oracle: it catches a group registered with a wrong hash, or a member vram whose original
    bytes are not in the claimed equivalence class.
    It does NOT — and cannot — detect SOURCE drift: a sig is a property of the ROM, so it is
    immutable with respect to src/. (The pre-audit docstring claimed it did. Source drift is
    caught by the BUILD, `make check`, which is the real byte-gate. P9.)

C2  BANK — "the body is matched once in `source` (named `func`) and INSTANTIATED at every member."
    DERIVED from the build invariant rather than re-parsed (R33): the fleet builds byte-identical,
    and INCLUDE_ASM pastes the ORIGINAL assembly — therefore a member site NOT wrapped in
    INCLUDE_ASM is byte-exact, and one that IS wrapped is NOT banked, whatever the registry says.
      C2a  the `func` token must actually occur in `source`   (no group may name a macro nobody wrote)
      C2b  no member may still be an INCLUDE_ASM stub          (no group may claim work never done)

COVERAGE (R32) — the run reports found-vs-candidates and REFUSES to print a green it did not earn.
    A member whose binary has no signature file is UNVALIDATED for C1, and the run FAILS unless
    --allow-unsigned. Pre-audit this printed [WARN] and still exited 0 — so on a fresh clone, where
    .run/sig.*.jsonl does not exist at all, the gate validated NOTHING and passed.

WHY THIS TOOL IS AUDITED FIRST
------------------------------
It is the last line of `make report` and is fail-closed by design, so a silent skip here is a FALSE
GREEN FROM A GATE and nothing downstream can catch it. The audit found three:
  * C2a was greenlighting 7 groups whose `DEFINE_func_*` macro was never written (1808 claimed,
    1801 defined) — hiding 532 member-instances of real, unbanked work in 4 functions that are
    matched in ov_SC01_077 and still INCLUDE_ASM in the other 133 overlays;
  * a missing sig file degraded silently to "0 validated, 0 failed" and exit 0;
  * the bank claim itself was never checked at all.

This is NOT a fork of psyq_integrate's object-swap: that works only for separate library SUBSEGMENT
stubs; game-code functions are interior to one object per binary, so the linker cannot swap them —
sharing is source-level. The .ld interpose stays the library mechanism.

Usage: tools/dedup_integrate.py [--check] [--binary <alias>] [--allow-unsigned]
                                [--dedup config/dedup.us.yaml]
"""
import argparse, json, pathlib, re, sys

ROOT = pathlib.Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "tools"))
from dup_report import BINARIES  # single source of truth for per-binary .run/sig.<bin>.jsonl paths

TIERS = ("h_exact", "h_norm")

# OVER-APPROXIMATING by design (R32): any C identifier, not just func_<hex> — a curated symbol
# (e.g. listCdBuffer) is a stub too, and a `func_`-only pattern silently misses it.
INCLUDE_ASM_RE = re.compile(r'INCLUDE_ASM\([^)]*,\s*([A-Za-z_]\w*)\s*\)')
SYMBOL_LINE_RE = re.compile(r'^\s*([A-Za-z_]\w*)\s*=\s*(0x[0-9A-Fa-f]+)\s*;')


def _addr(v):
    """Accept a YAML int (0x.. parsed) or a hex string ('0x..' / bare hex) -> int."""
    return v if isinstance(v, int) else int(str(v), 16)


def group_members(g):
    """Yield (binary, vram_int, name) for a group in EITHER form (Phase 15):
       verbose   -> members: [{binary, vram, name}, ...]
       shorthand -> vram: 0xADDR + binaries: [a, b, ...]  (position-locked: same vram/name everywhere;
                    ~20x smaller for fleet-wide shares where every member sits at the identical vaddr)."""
    if g.get("members"):
        for m in g["members"]:
            a = _addr(m["vram"])
            yield m.get("binary"), a, m.get("name", f"func_{a:08X}")
    elif g.get("binaries") is not None and g.get("vram") is not None:
        a = _addr(g["vram"]); nm = f"func_{a:08X}"
        for b in g["binaries"]:
            yield b, a, nm


def _src_paths(binary):
    """The .c files that make up a binary's source (main is the top-level src/*.c)."""
    if binary == "main":
        return sorted((ROOT / "src").glob("*.c"))
    return sorted((ROOT / "src" / binary).glob("*.c"))


def _curated_syms(binary):
    """curated name -> addr, from the symbol files that can name this binary's symbols."""
    out = {}
    for cand in (f"symbols.{binary}.txt", "symbols.resident.txt", "symbols.us.txt"):
        p = ROOT / "config" / cand
        if not p.exists():
            continue
        for line in p.read_text().splitlines():
            m = SYMBOL_LINE_RE.match(line)
            if m:
                out.setdefault(m.group(1), int(m.group(2), 16))
    return out


_STUB_CACHE = {}


def stub_addrs(binary):
    """C2b's oracle: the set of vram addresses this binary still ships as INCLUDE_ASM.

    Resolves ANY C identifier, not just func_<hex> — a curated stub name (listCdBuffer) is a stub.
    An unresolvable stub symbol is a COVERAGE FAILURE, not a shrug: it means this oracle cannot
    answer "is this member banked?" for that address, so we fail loud rather than guess 'banked'."""
    if binary in _STUB_CACHE:
        return _STUB_CACHE[binary]
    paths = _src_paths(binary)
    if not paths:
        sys.exit(f"dedup-check: binary {binary!r} has no source under src/ — cannot verify its bank claims")
    syms = _curated_syms(binary)
    addrs, unresolved = set(), []
    for p in paths:
        for m in INCLUDE_ASM_RE.finditer(p.read_text()):
            nm = m.group(1)
            if re.fullmatch(r'func_[0-9A-Fa-f]{8}', nm):
                addrs.add(int(nm[5:], 16))
            elif nm in syms:
                addrs.add(syms[nm])
            else:
                unresolved.append(f"{p.relative_to(ROOT)}: {nm}")
    if unresolved:
        sys.exit(f"dedup-check: {len(unresolved)} INCLUDE_ASM symbol(s) in {binary} resolve to no "
                 f"address — the bank oracle is blind to them:\n  " + "\n  ".join(unresolved[:10]))
    _STUB_CACHE[binary] = addrs
    return addrs


def _load_sig_index(rel):
    """addr(int) -> sig row for .run/sig.<bin>.jsonl, or None if the file is absent."""
    p = ROOT / rel
    if not p.exists():
        return None
    idx = {}
    for line in p.read_text().splitlines():
        line = line.strip()
        if line:
            r = json.loads(line)
            idx[int(r["addr"], 16)] = r
    return idx


def check(groups, binary_filter=None, allow_unsigned=False):
    """Validate every group's C1 (equivalence, vs the original bytes) and C2 (bank, vs the build
    invariant). Returns the failure count (0 = every claim honest AND fully covered)."""
    sigcache = {}

    def sig_for(binary):
        if binary not in sigcache:
            cfg = BINARIES.get(binary)
            sigcache[binary] = _load_sig_index(cfg["sig"]) if cfg else None
        return sigcache[binary]

    src_cache = {}

    def source_text(rel):
        if rel not in src_cache:
            p = ROOT / rel
            src_cache[rel] = p.read_text() if p.exists() else None
        return src_cache[rel]

    failures = unvalidated = validated = 0
    members_seen = members_c1 = 0

    for g in groups:
        gid = g.get("id", "?")
        members = list(group_members(g))
        if binary_filter and not any(b == binary_filter for b, _, _ in members):
            continue
        if g.get("tier") not in TIERS:
            print(f"[FAIL] {gid}: tier must be one of {TIERS} (got {g.get('tier')!r})"); failures += 1; continue
        if len(members) < 2:
            print(f"[FAIL] {gid}: a share needs >=2 members (got {len(members)})"); failures += 1; continue
        src = g.get("source")
        txt = source_text(src) if src else None
        if txt is None:
            print(f"[FAIL] {gid}: shared source {src!r} missing"); failures += 1; continue

        # ---- C2a: the shared body this group names must actually EXIST in `source` ---------------
        fn = g.get("func")
        if not fn:
            print(f"[FAIL] {gid}: no `func` (the group must name the shared body it claims)")
            failures += 1; continue
        if not re.search(rf'\b{re.escape(fn)}\b', txt):
            print(f"[FAIL] {gid}: `func: {fn}` does NOT occur in {src} — the group claims a shared "
                  f"body that was never written (the share was registered but never propagated)")
            failures += 1; continue

        tier, want = g["tier"], g.get("hash")
        if not want:
            print(f"[FAIL] {gid}: no recorded hash"); failures += 1; continue

        ok = True
        for (b, vram, name) in members:
            members_seen += 1

            # ---- C2b: DERIVED from the build invariant — a member still stubbed is NOT banked ----
            if vram in stub_addrs(b):
                print(f"[FAIL] {gid}: {b}:0x{vram:08x} ({name}) is claimed MATCHED but is still an "
                      f"INCLUDE_ASM stub — the registry is claiming work that was never done")
                failures += 1; ok = False; continue

            # ---- C1: equivalence, against the ORIGINAL bytes ------------------------------------
            idx = sig_for(b)
            if idx is None:
                unvalidated += 1; ok = False
                continue
            row = idx.get(vram)
            if row is None:
                print(f"[FAIL] {gid}: {b}:0x{vram:08x} ({name}) not found in {b} signature")
                failures += 1; ok = False; continue
            got = row.get(tier)
            if got != want:
                print(f"[FAIL] {gid}: {b}:0x{vram:08x} ({name}) {tier}={got} != recorded {want} — SHARE DRIFTED")
                failures += 1; ok = False; continue
            members_c1 += 1

        if ok:
            validated += 1
            uniq = sorted({b for b, _, _ in members})
            disp = ",".join(uniq) if len(uniq) <= 6 else f"{len(uniq)} binaries"
            print(f"[ OK ] {gid}: {len(members)} members [{disp}] share {tier} {want[:12]}… (source {src})")

    # ---- COVERAGE (R32): never print a green this run did not earn ------------------------------
    print(f"dedup-check: {validated} validated, {failures} failed  "
          f"| C1 coverage {members_c1}/{members_seen} members signed")
    if unvalidated:
        msg = (f"dedup-check: {unvalidated} member(s) UNVALIDATED — their binary has no "
               f".run/sig.<bin>.jsonl, so the equivalence claim could not be checked at all. "
               f"Run `make sig-overlays` (or pass --allow-unsigned to accept an unverified registry).")
        if allow_unsigned:
            print(f"[WARN] {msg}")
        else:
            print(f"[FAIL] {msg}")
            failures += unvalidated
    return failures


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--dedup", default="config/dedup.us.yaml")
    ap.add_argument("--binary", default=None, help="validate only groups with a member in this binary")
    ap.add_argument("--check", action="store_true", help="validate (default mode)")
    ap.add_argument("--allow-unsigned", action="store_true",
                    help="downgrade 'member has no sig file' from FAIL to WARN (fresh clone / pre-sig-refresh)")
    ap.add_argument("--apply", action="store_true", help="reserved (shares are source-level; validates only)")
    a = ap.parse_args()

    path = ROOT / a.dedup
    if not path.exists():
        print(f"dedup-check: {a.dedup} absent — no shares registered (build uses INCLUDE_ASM stubs).")
        return
    import yaml  # PyYAML (venv; splat dependency)
    data = yaml.safe_load(path.read_text()) or {}
    groups = data.get("groups") or []
    if a.apply:
        print("dedup_integrate --apply: shares are SOURCE-LEVEL (authored in src/shared/, instantiated "
              "per site); no build-time action — validating instead.")
    if check(groups, a.binary, a.allow_unsigned):
        sys.exit(1)


if __name__ == "__main__":
    main()