BFM-decomp/tools/exemplar_miner.py

#!/usr/bin/env python3
"""exemplar_miner.py — route every residual stub to its lever, rank the reach-134 work (Phase 20 T2).

Consumes the residual census (tools/wall_taxonomy.json: per-stub bucket/nins/mismatch for the canonical
source overlay ov_SC01_077) and computes each stub's FLEET REACH (how many of the 134 overlays carry a
byte-identical copy, the ×N propagation leverage) the same way dedup_propagate does (h_exact over the
per-overlay sigs). It then:
  - routes each bucket to a lever (WAVE / PINS / STRUCT / STUB) — ADVISORY; the byte-gate is the sole
    arbiter (G3/P9), this is a worklist, not a correctness claim,
  - emits the reach-134 WAVE-target list (T6 fuel; ranked reach desc, nins asc),
  - surfaces the PINS/permuter-class reach-134 near-misses (the class-crack candidate pool for T3),
  - tallies the honest STUB residual (ARITY_WALL loose-typing + NONFAITHFUL) and the STRUCT-context band.

Note: wall_taxonomy's `mismatch` is the M2C-DRAFT mismatch (scaffold quality), NOT the hand-match floor —
a STRUCTURAL_MISS with a high mismatch can still hand-match to a 1-instruction residual (e.g. the loop-guard
exemplar func_8012C2D0). So the buckets size the POOLS; the cleanest class-isolating exemplar is a Max pick.

Outputs: docs/exemplar_curriculum.md (human worklist) + .run/exemplar_routing.json (machine, for the wave gen).
Usage: tools/exemplar_miner.py [--source ov_SC01_077] [--census .run/wall_taxonomy.json] [--top 60]
"""
import argparse, json, os, sys, pathlib, datetime

ROOT = pathlib.Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "tools"))
import dedup_propagate as dp  # load_sig, onboarded_overlays (reach == its h_exact computation)
import corpus                 # the derived stub oracle (Phase 26-A) — "still work?" = "still INCLUDE_ASM"

# bucket -> lever. ADVISORY routing (the byte-gate decides truth). Grouped by how we'd actually attack it:
#   WAVE   = m2c/hand-draftable by the Ultracode swarm + §17 toolkit + canon-first gate + fix_arity_callers
#   STRUCT = m2c can't draft until a type is known (fn-ptr tables, jump tables, struct derefs)
#   PINS   = regalloc/schedule near-miss -> register-pins hand-match (permuter grind skipped this phase);
#            the small-mismatch reach-134 ones are the class-crack candidate pool (T3)
#   STUB   = fundamental (loose-typing ARITY_WALL) or non-faithful (GTE/handwritten) -> honest INCLUDE_ASM
LEVER = {
    "STRUCTURAL_MISS": "WAVE", "SIG_FIXABLE_KR": "WAVE", "LEAF_READY": "WAVE",
    "MCOMPILE_arg-arity": "WAVE", "MCOMPILE_undeclared-other": "WAVE", "MCOMPILE_stack-var": "WAVE",
    "MCOMPILE_m2c-incomplete-arg": "WAVE", "DATA_CONFLICT": "WAVE", "OTHER_CONFLICT": "WAVE",
    "MCOMPILE_other": "WAVE", "VOID_VALUE_MISUSE": "WAVE",
    "MCOMPILE_fnptr-call": "STRUCT", "M2C_DECOMP_FAIL": "STRUCT", "MCOMPILE_bad-deref": "STRUCT",
    "MCOMPILE_bad-switch": "STRUCT",
    "PERMUTER_CLASS": "PINS",
    "ARITY_WALL": "STUB", "NONFAITHFUL_DEFER": "STUB", "NO_ASM": "STUB", "M2C_EMPTY": "STUB",
}

# the named class-crack exemplars carried from Phase 18/19 (T3 targets); the m2c bucket understates them.
CLASS_EXEMPLARS = {
    "8012C2D0": "loop-guard operand-order (loop.c get_condition) — T3b",
    "8014F2E0": "store-vs-load scheduling tie-break (sched.c) — T3c",
    "8013C360": "-O0 %lo-folding cluster (address-mode split) — T3a",
}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--source", default="ov_SC01_077")
    ap.add_argument("--census", default=".run/wall_taxonomy.json")
    ap.add_argument("--top", type=int, default=60)
    a = ap.parse_args()

    recs = json.load(open(ROOT / a.census))
    pool = dp.onboarded_overlays()
    sigs = {ov: dp.load_sig(ov) for ov in pool}
    # "Is this census entry still work?" — answered by the INVARIANT (R33), not the dedup registry.
    # registered_addrs() (config/dedup.us.yaml) is a PROXY: a function matched-but-not-registered
    # (banked inline, or matched-but-local) is absent from it, so the old filter kept matched
    # functions in the residual pool (the audit measured ~60% wrong). A function is unmatched iff it
    # is still an INCLUDE_ASM stub in the source — which corpus.stubs derives from the tree.
    live_stub_addrs = set(corpus.stubs(a.source))   # {addr_int: Stub} -> the set of addr ints

    def reach(addr_int):
        h = sigs[a.source].get(addr_int, {}).get("h_exact")
        if not h:
            return 0
        return sum(1 for ov in pool if sigs[ov].get(addr_int, {}).get("h_exact") == h)

    rows = []
    for r in recs:
        ai = int(r["addr"], 16)
        if ai not in live_stub_addrs:       # not a live INCLUDE_ASM stub -> already matched, not a residual
            continue
        lever = LEVER.get(r["bucket"], "WAVE")
        rows.append({"addr": r["addr"].upper(), "nins": r["nins"], "mismatch": r.get("mismatch"),
                     "bucket": r["bucket"], "lever": lever, "reach": reach(ai)})

    # machine output for the wave generator
    os.makedirs(ROOT / ".run", exist_ok=True)
    json.dump({x["addr"]: x for x in rows}, open(ROOT / ".run/exemplar_routing.json", "w"), indent=1)

    # ---- human worklist ----
    by_lever = {}
    for x in rows:
        by_lever.setdefault(x["lever"], []).append(x)
    snap = datetime.datetime.fromtimestamp((ROOT / a.census).stat().st_mtime).strftime("%Y-%m-%d")

    L = ["# BFM residual curriculum / router  (generated by tools/exemplar_miner.py)",
         f"# census: {a.census} (snapshot {snap}, {len(rows)} unregistered residual stubs in {a.source})",
         "# reach = #overlays sharing the byte-identical copy (×N propagation leverage), computed from sigs.",
         "# ROUTING IS ADVISORY — the per-overlay byte-gate is the sole arbiter (G3/P9). `mismatch` is the",
         "# M2C-DRAFT mismatch (scaffold quality), NOT the hand-match floor.", "",
         "## Routing summary (by lever, reach-weighted)", "",
         "| lever | fns | reach-134 | Σ reach | what it is |", "|---|---|---|---|---|"]
    desc = {"WAVE": "Ultracode swarm + §17 toolkit + canon-first gate (the yield engine, T6)",
            "STRUCT": "m2c blocked on a type (fn-ptr/jump tables, struct deref) — needs type context",
            "PINS": "regalloc/schedule near-miss — register-pin hand-match; class-crack pool (T3)",
            "STUB": "fundamental loose-typing (ARITY_WALL) / non-faithful — honest INCLUDE_ASM (P9)"}
    for lev in ["WAVE", "STRUCT", "PINS", "STUB"]:
        g = by_lever.get(lev, [])
        r134 = sum(1 for x in g if x["reach"] >= 134)
        sreach = sum(x["reach"] for x in g)
        L.append(f"| {lev} | {len(g)} | {r134} | {sreach} | {desc[lev]} |")
    L.append("")

    # WAVE pool, reach-134, ranked (T6 fuel)
    wave134 = sorted([x for x in by_lever.get("WAVE", []) if x["reach"] >= 134],
                     key=lambda x: (-(x["reach"]), x["nins"] or 9999))
    L += ["", f"## WAVE pool — reach-134, ranked (T6 fuel; {len(wave134)} fns, smallest first)", "",
          "| addr | nins | bucket | reach |", "|---|---|---|---|"]
    for x in wave134[:a.top]:
        L.append(f"| {x['addr']} | {x['nins']} | {x['bucket']} | {x['reach']} |")
    if len(wave134) > a.top:
        L.append(f"| …+{len(wave134)-a.top} more | | | |")

    # PINS / class-crack candidate pool, reach-134, smallest m2c-mismatch first
    pins134 = sorted([x for x in by_lever.get("PINS", []) if x["reach"] >= 134],
                     key=lambda x: (x["mismatch"] if x["mismatch"] is not None else 999, x["nins"] or 9999))
    L += ["", f"## PINS / permuter-class — reach-134 near-misses ({len(pins134)} fns; class-crack candidate pool)",
          "", "| addr | nins | m2c-mismatch | reach |", "|---|---|---|---|"]
    for x in pins134[:a.top]:
        L.append(f"| {x['addr']} | {x['nins']} | {x['mismatch']} | {x['reach']} |")
    if len(pins134) > a.top:
        L.append(f"| …+{len(pins134)-a.top} more | | | |")

    # the named T3 class exemplars + their census row + reach
    rowmap = {x["addr"]: x for x in rows}
    L += ["", "## Named class-crack exemplars (T3 — the Max spikes)", "",
          "| addr | class | census bucket | nins | reach |", "|---|---|---|---|---|"]
    for addr, note in CLASS_EXEMPLARS.items():
        x = rowmap.get(addr)
        if x:
            L.append(f"| {addr} | {note} | {x['bucket']} | {x['nins']} | {x['reach']} |")
        else:
            L.append(f"| {addr} | {note} | (not a current stub — matched or split) | | |")

    # STUB honesty tally
    stub = by_lever.get("STUB", [])
    from collections import Counter
    sc = Counter(x["bucket"] for x in stub)
    L += ["", "## STUB residual (honest, no lever — P9)", "",
          f"{len(stub)} fns: " + ", ".join(f"{b}={n}" for b, n in sc.most_common()), ""]

    open(ROOT / "docs/exemplar_curriculum.md", "w").write("\n".join(L) + "\n")
    print(f"wrote docs/exemplar_curriculum.md + .run/exemplar_routing.json")
    print(f"  {len(rows)} residual stubs; WAVE {len(by_lever.get('WAVE',[]))} "
          f"(reach-134 {len(wave134)}) | STRUCT {len(by_lever.get('STRUCT',[]))} | "
          f"PINS {len(by_lever.get('PINS',[]))} (reach-134 {len(pins134)}) | STUB {len(stub)}")


if __name__ == "__main__":
    main()