diff --git a/docs/exemplar_curriculum.md b/docs/exemplar_curriculum.md new file mode 100644 index 000000000..942c81492 --- /dev/null +++ b/docs/exemplar_curriculum.md @@ -0,0 +1,144 @@ +# BFM residual curriculum / router (generated by tools/exemplar_miner.py) +# census: .run/wall_taxonomy.json (snapshot 2026-06-19, 835 unregistered residual stubs in ov_SC01_077) +# reach = #overlays sharing the byte-identical copy (×N propagation leverage), computed from sigs. +# ROUTING IS ADVISORY — the per-overlay byte-gate is the sole arbiter (G3/P9). `mismatch` is the +# M2C-DRAFT mismatch (scaffold quality), NOT the hand-match floor. + +## Routing summary (by lever, reach-weighted) + +| lever | fns | reach-134 | Σ reach | what it is | +|---|---|---|---|---| +| WAVE | 472 | 218 | 29468 | Ultracode swarm + §17 toolkit + canon-first gate (the yield engine, T6) | +| STRUCT | 159 | 12 | 1755 | m2c blocked on a type (fn-ptr/jump tables, struct deref) — needs type context | +| PINS | 114 | 45 | 6099 | regalloc/schedule near-miss — register-pin hand-match; class-crack pool (T3) | +| STUB | 90 | 58 | 7804 | fundamental loose-typing (ARITY_WALL) / non-faithful — honest INCLUDE_ASM (P9) | + + +## WAVE pool — reach-134, ranked (T6 fuel; 218 fns, smallest first) + +| addr | nins | bucket | reach | +|---|---|---|---| +| 80174684 | 8 | DATA_CONFLICT | 134 | +| 8014C8C8 | 10 | STRUCTURAL_MISS | 134 | +| 801705C0 | 14 | OTHER_CONFLICT | 134 | +| 80161BE0 | 17 | STRUCTURAL_MISS | 134 | +| 8016E918 | 17 | STRUCTURAL_MISS | 134 | +| 801710DC | 17 | SIG_FIXABLE_KR | 134 | +| 80146A6C | 18 | STRUCTURAL_MISS | 134 | +| 8016B91C | 18 | MCOMPILE_arg-arity | 134 | +| 8012A418 | 19 | STRUCTURAL_MISS | 134 | +| 8012C2D0 | 19 | STRUCTURAL_MISS | 134 | +| 8016E778 | 20 | STRUCTURAL_MISS | 134 | +| 80172C50 | 20 | STRUCTURAL_MISS | 134 | +| 801612B8 | 21 | STRUCTURAL_MISS | 134 | +| 8016EDEC | 21 | STRUCTURAL_MISS | 134 | +| 8016EE40 | 21 | STRUCTURAL_MISS | 134 | +| 8016BF50 | 22 | STRUCTURAL_MISS | 134 | +| 801718AC | 22 | VOID_VALUE_MISUSE | 134 | +| 80149374 | 23 | STRUCTURAL_MISS | 134 | +| 801493D0 | 23 | STRUCTURAL_MISS | 134 | +| 8015C030 | 23 | STRUCTURAL_MISS | 134 | +| 8012A598 | 24 | STRUCTURAL_MISS | 134 | +| 8012A62C | 24 | STRUCTURAL_MISS | 134 | +| 801466F0 | 24 | STRUCTURAL_MISS | 134 | +| 8016191C | 24 | STRUCTURAL_MISS | 134 | +| 8017248C | 24 | STRUCTURAL_MISS | 134 | +| 80157580 | 25 | VOID_VALUE_MISUSE | 134 | +| 80137614 | 26 | SIG_FIXABLE_KR | 134 | +| 8016130C | 26 | STRUCTURAL_MISS | 134 | +| 8013C08C | 27 | STRUCTURAL_MISS | 134 | +| 8013E448 | 27 | STRUCTURAL_MISS | 134 | +| 80168B70 | 27 | MCOMPILE_arg-arity | 134 | +| 8012F274 | 29 | STRUCTURAL_MISS | 134 | +| 80131CF4 | 29 | STRUCTURAL_MISS | 134 | +| 80147364 | 30 | STRUCTURAL_MISS | 134 | +| 801494CC | 30 | MCOMPILE_stack-var | 134 | +| 8016706C | 30 | STRUCTURAL_MISS | 134 | +| 801708B0 | 30 | MCOMPILE_stack-var | 134 | +| 80149450 | 31 | MCOMPILE_stack-var | 134 | +| 80149F2C | 31 | STRUCTURAL_MISS | 134 | +| 8014A51C | 32 | STRUCTURAL_MISS | 134 | +| 8014F3E8 | 32 | SIG_FIXABLE_KR | 134 | +| 80177F84 | 32 | STRUCTURAL_MISS | 134 | +| 8012F038 | 33 | MCOMPILE_stack-var | 134 | +| 80142B2C | 34 | VOID_VALUE_MISUSE | 134 | +| 80144B14 | 34 | STRUCTURAL_MISS | 134 | +| 801495C4 | 34 | MCOMPILE_stack-var | 134 | +| 8014964C | 34 | MCOMPILE_stack-var | 134 | +| 80161A90 | 34 | STRUCTURAL_MISS | 134 | +| 80177AD4 | 34 | STRUCTURAL_MISS | 134 | +| 8012F568 | 35 | STRUCTURAL_MISS | 134 | +| 8012F40C | 36 | STRUCTURAL_MISS | 134 | +| 80161888 | 37 | MCOMPILE_arg-arity | 134 | +| 8012B0B4 | 38 | MCOMPILE_m2c-incomplete-arg | 134 | +| 80131D68 | 38 | STRUCTURAL_MISS | 134 | +| 80147C30 | 38 | MCOMPILE_stack-var | 134 | +| 801778A8 | 38 | STRUCTURAL_MISS | 134 | +| 8014A59C | 39 | STRUCTURAL_MISS | 134 | +| 8012E28C | 40 | STRUCTURAL_MISS | 134 | +| 8014FD54 | 40 | MCOMPILE_stack-var | 134 | +| 80171FFC | 40 | STRUCTURAL_MISS | 134 | +| …+158 more | | | | + +## PINS / permuter-class — reach-134 near-misses (45 fns; class-crack candidate pool) + +| addr | nins | m2c-mismatch | reach | +|---|---|---|---| +| 80153C44 | 12 | 1 | 134 | +| 8015CC0C | 13 | 1 | 134 | +| 8015CF58 | 13 | 1 | 134 | +| 80161208 | 14 | 1 | 134 | +| 801711FC | 15 | 1 | 134 | +| 8015CCD0 | 20 | 1 | 134 | +| 8015F948 | 23 | 1 | 134 | +| 8016F0E4 | 26 | 1 | 134 | +| 80151664 | 35 | 1 | 134 | +| 801522CC | 41 | 1 | 134 | +| 80152C80 | 41 | 1 | 134 | +| 80163764 | 42 | 1 | 134 | +| 80136DFC | 50 | 1 | 134 | +| 80151C54 | 52 | 1 | 134 | +| 80144558 | 53 | 1 | 134 | +| 801446A4 | 54 | 1 | 134 | +| 80155C64 | 67 | 1 | 134 | +| 8014BD60 | 14 | 2 | 134 | +| 80130514 | 46 | 2 | 134 | +| 801683D8 | 22 | 3 | 134 | +| 80157AC8 | 43 | 3 | 134 | +| 80171C64 | 24 | 4 | 134 | +| 80146AFC | 40 | 4 | 134 | +| 80129C40 | 46 | 4 | 134 | +| 8016432C | 57 | 4 | 134 | +| 80131CA8 | 19 | 5 | 134 | +| 80141B90 | 29 | 5 | 134 | +| 80155F80 | 12 | 6 | 134 | +| 801576A8 | 29 | 6 | 134 | +| 80131A34 | 37 | 6 | 134 | +| 8014C168 | 14 | 8 | 134 | +| 80155D70 | 48 | 8 | 134 | +| 8015F380 | 50 | 8 | 134 | +| 80136D08 | 61 | 8 | 134 | +| 801733FC | 25 | 9 | 134 | +| 80155EA4 | 45 | 9 | 134 | +| 80175414 | 16 | 10 | 134 | +| 80151B98 | 47 | 10 | 134 | +| 8014E5B4 | 59 | 10 | 134 | +| 80152500 | 61 | 10 | 134 | +| 8016A08C | 80 | 10 | 134 | +| 80170CF0 | 30 | 11 | 134 | +| 801415C0 | 51 | 11 | 134 | +| 80156600 | 18 | 13 | 134 | +| 80132288 | 97 | 14 | 134 | + +## Named class-crack exemplars (T3 — the Max spikes) + +| addr | class | census bucket | nins | reach | +|---|---|---|---|---| +| 8012C2D0 | loop-guard operand-order (loop.c get_condition) — T3b | STRUCTURAL_MISS | 19 | 134 | +| 8014F2E0 | store-vs-load scheduling tie-break (sched.c) — T3c | STRUCTURAL_MISS | 66 | 134 | +| 8013C360 | -O0 %lo-folding cluster (address-mode split) — T3a | STRUCTURAL_MISS | 45 | 134 | + +## STUB residual (honest, no lever — P9) + +90 fns: ARITY_WALL=74, NONFAITHFUL_DEFER=16 + diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index f31528c35..813ae429b 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -10,8 +10,8 @@ - [x] **T1 — Close the propagation cap (typedef/local-type lift)** · DONE (xHigh) Extended `tools/build_engine_types.py` with `find_typedefs()` + collision/overlap guards; lifted **14 typedefs** to `engine_types.h` (byte-neutral, `ov_SC01_077` stays `d19c9580`); `dedup_propagate --auto-from` propagated **9 reach-134 fns ×134**. **check-all 136/136**, fleet **58.00%→58.35%** (+0.35%), dedup **1485→1494**, 0 NON_MATCHING. **FINDING (R14):** cap was 16; only **9 were type-blocked** (all freed) — the other **7 are callee/data-extern plumbing** (`func_80142C84`, `D_800AE620`…), the §19 declaration-plumbing class, NOT the type-lift → recovery-tooling lever (see Notes). -- [ ] **T2 — Exemplar-miner / residual router** · xHigh - New `tools/exemplar_miner.py` (consumes `wall_taxonomy.json` + `dup_report --cross` reach) → `docs/exemplar_curriculum.md`: routed worklist (waves / pins-permuter / class-crack / stub) + the fresh reach-134 wave-target list. +- [x] **T2 — Exemplar-miner / residual router** · DONE (xHigh) + `tools/exemplar_miner.py` → `docs/exemplar_curriculum.md` + `.run/exemplar_routing.json` (reach computed from sigs like dedup_propagate). **835 residual stubs routed:** WAVE 472 (218 reach-134, T6 fuel) · STRUCT 159 · **PINS 114 (45 reach-134)** · STUB 90 (74 ARITY_WALL + 16 NONFAITHFUL). **Key T3 input:** 17 reach-134 fns at m2c-mismatch=1 = the cleanest regalloc/schedule isolates (likely §17-pins wins + the genuine class residuals). Named exemplars confirmed reach-134. *Caveat: census is the Jun-19 snapshot (pre-Phase-19 waves, ~7% stale); T5/T6 regen gives the authoritative fresh WAVE list.* - [ ] **T3 — gcc class-crack research block** · **Max each** (R17) — the phase hinge - [ ] T3a — `%lo`-folding at `-O0` (most promising; ~10 fns; address-mode/`local-alloc.c`) - [ ] T3b — loop-guard operand-order (`func_8012C2D0`; `loop.c get_condition`) @@ -27,7 +27,7 @@ **Optional / conditional (decide live, else → Phase 21):** `-O0` ×134 rollout (only if T3a cracks `%lo`); the 28 giants (deferred to a focused Phase-21 deep session). ## Current task pointer -→ **T2** (exemplar-miner / residual router). T1 complete + committed. +→ **T3** (gcc class-crack block). Refined opener: attack the **17 reach-134 m2c-mismatch=1 PINS** fns with the existing §17 toolkit (pins/array-decay) — bank ×134 wins + discover what resists; THEN Max gcc-source research on the residuals + the 3 named exemplars (crack-or-cite). T1, T2 complete + committed. **Carry from T1 → recovery tooling (Phase-20 backlog item 2 / T2 router):** the **7 callee/data-plumbing capped fns** (reach-134, matched-but-local): `0x80142B2C 0x801535F4 0x80155E30 0x80157580 0x801576A8 0x80168F40 0x80170B48`. They fail `compiles_standalone` on undeclared callees/data, not types. A `dedup_propagate` macro-extern-injection (or canonical-callee-sig embed, the `gen_harvest_targets` approach) would free them ×134 (~+0.3%). NOT type-lift; do not reopen T1 for them. diff --git a/tools/exemplar_miner.py b/tools/exemplar_miner.py new file mode 100644 index 000000000..d828e10e0 --- /dev/null +++ b/tools/exemplar_miner.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""exemplar_miner.py — route every residual stub to its lever, rank the reach-134 work (Phase 20 T2). + +Consumes the residual census (tools/wall_taxonomy.json: per-stub bucket/nins/mismatch for the canonical +source overlay ov_SC01_077) and computes each stub's FLEET REACH (how many of the 134 overlays carry a +byte-identical copy, the ×N propagation leverage) the same way dedup_propagate does (h_exact over the +per-overlay sigs). It then: + - routes each bucket to a lever (WAVE / PINS / STRUCT / STUB) — ADVISORY; the byte-gate is the sole + arbiter (G3/P9), this is a worklist, not a correctness claim, + - emits the reach-134 WAVE-target list (T6 fuel; ranked reach desc, nins asc), + - surfaces the PINS/permuter-class reach-134 near-misses (the class-crack candidate pool for T3), + - tallies the honest STUB residual (ARITY_WALL loose-typing + NONFAITHFUL) and the STRUCT-context band. + +Note: wall_taxonomy's `mismatch` is the M2C-DRAFT mismatch (scaffold quality), NOT the hand-match floor — +a STRUCTURAL_MISS with a high mismatch can still hand-match to a 1-instruction residual (e.g. the loop-guard +exemplar func_8012C2D0). So the buckets size the POOLS; the cleanest class-isolating exemplar is a Max pick. + +Outputs: docs/exemplar_curriculum.md (human worklist) + .run/exemplar_routing.json (machine, for the wave gen). +Usage: tools/exemplar_miner.py [--source ov_SC01_077] [--census .run/wall_taxonomy.json] [--top 60] +""" +import argparse, json, os, sys, pathlib, datetime + +ROOT = pathlib.Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "tools")) +import dedup_propagate as dp # load_sig, onboarded_overlays, registered_addrs (reach == its computation) + +# bucket -> lever. ADVISORY routing (the byte-gate decides truth). Grouped by how we'd actually attack it: +# WAVE = m2c/hand-draftable by the Ultracode swarm + §17 toolkit + canon-first gate + fix_arity_callers +# STRUCT = m2c can't draft until a type is known (fn-ptr tables, jump tables, struct derefs) +# PINS = regalloc/schedule near-miss -> register-pins hand-match (permuter grind skipped this phase); +# the small-mismatch reach-134 ones are the class-crack candidate pool (T3) +# STUB = fundamental (loose-typing ARITY_WALL) or non-faithful (GTE/handwritten) -> honest INCLUDE_ASM +LEVER = { + "STRUCTURAL_MISS": "WAVE", "SIG_FIXABLE_KR": "WAVE", "LEAF_READY": "WAVE", + "MCOMPILE_arg-arity": "WAVE", "MCOMPILE_undeclared-other": "WAVE", "MCOMPILE_stack-var": "WAVE", + "MCOMPILE_m2c-incomplete-arg": "WAVE", "DATA_CONFLICT": "WAVE", "OTHER_CONFLICT": "WAVE", + "MCOMPILE_other": "WAVE", "VOID_VALUE_MISUSE": "WAVE", + "MCOMPILE_fnptr-call": "STRUCT", "M2C_DECOMP_FAIL": "STRUCT", "MCOMPILE_bad-deref": "STRUCT", + "MCOMPILE_bad-switch": "STRUCT", + "PERMUTER_CLASS": "PINS", + "ARITY_WALL": "STUB", "NONFAITHFUL_DEFER": "STUB", "NO_ASM": "STUB", "M2C_EMPTY": "STUB", +} + +# the named class-crack exemplars carried from Phase 18/19 (T3 targets); the m2c bucket understates them. +CLASS_EXEMPLARS = { + "8012C2D0": "loop-guard operand-order (loop.c get_condition) — T3b", + "8014F2E0": "store-vs-load scheduling tie-break (sched.c) — T3c", + "8013C360": "-O0 %lo-folding cluster (address-mode split) — T3a", +} + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--source", default="ov_SC01_077") + ap.add_argument("--census", default=".run/wall_taxonomy.json") + ap.add_argument("--top", type=int, default=60) + a = ap.parse_args() + + recs = json.load(open(ROOT / a.census)) + pool = dp.onboarded_overlays() + sigs = {ov: dp.load_sig(ov) for ov in pool} + reg = dp.registered_addrs() + + def reach(addr_int): + h = sigs[a.source].get(addr_int, {}).get("h_exact") + if not h: + return 0 + return sum(1 for ov in pool if sigs[ov].get(addr_int, {}).get("h_exact") == h) + + rows = [] + for r in recs: + ai = int(r["addr"], 16) + if ai in reg: # already matched+shared — not a residual + continue + lever = LEVER.get(r["bucket"], "WAVE") + rows.append({"addr": r["addr"].upper(), "nins": r["nins"], "mismatch": r.get("mismatch"), + "bucket": r["bucket"], "lever": lever, "reach": reach(ai)}) + + # machine output for the wave generator + os.makedirs(ROOT / ".run", exist_ok=True) + json.dump({x["addr"]: x for x in rows}, open(ROOT / ".run/exemplar_routing.json", "w"), indent=1) + + # ---- human worklist ---- + by_lever = {} + for x in rows: + by_lever.setdefault(x["lever"], []).append(x) + snap = datetime.datetime.fromtimestamp((ROOT / a.census).stat().st_mtime).strftime("%Y-%m-%d") + + L = ["# BFM residual curriculum / router (generated by tools/exemplar_miner.py)", + f"# census: {a.census} (snapshot {snap}, {len(rows)} unregistered residual stubs in {a.source})", + "# reach = #overlays sharing the byte-identical copy (×N propagation leverage), computed from sigs.", + "# ROUTING IS ADVISORY — the per-overlay byte-gate is the sole arbiter (G3/P9). `mismatch` is the", + "# M2C-DRAFT mismatch (scaffold quality), NOT the hand-match floor.", "", + "## Routing summary (by lever, reach-weighted)", "", + "| lever | fns | reach-134 | Σ reach | what it is |", "|---|---|---|---|---|"] + desc = {"WAVE": "Ultracode swarm + §17 toolkit + canon-first gate (the yield engine, T6)", + "STRUCT": "m2c blocked on a type (fn-ptr/jump tables, struct deref) — needs type context", + "PINS": "regalloc/schedule near-miss — register-pin hand-match; class-crack pool (T3)", + "STUB": "fundamental loose-typing (ARITY_WALL) / non-faithful — honest INCLUDE_ASM (P9)"} + for lev in ["WAVE", "STRUCT", "PINS", "STUB"]: + g = by_lever.get(lev, []) + r134 = sum(1 for x in g if x["reach"] >= 134) + sreach = sum(x["reach"] for x in g) + L.append(f"| {lev} | {len(g)} | {r134} | {sreach} | {desc[lev]} |") + L.append("") + + # WAVE pool, reach-134, ranked (T6 fuel) + wave134 = sorted([x for x in by_lever.get("WAVE", []) if x["reach"] >= 134], + key=lambda x: (-(x["reach"]), x["nins"] or 9999)) + L += ["", f"## WAVE pool — reach-134, ranked (T6 fuel; {len(wave134)} fns, smallest first)", "", + "| addr | nins | bucket | reach |", "|---|---|---|---|"] + for x in wave134[:a.top]: + L.append(f"| {x['addr']} | {x['nins']} | {x['bucket']} | {x['reach']} |") + if len(wave134) > a.top: + L.append(f"| …+{len(wave134)-a.top} more | | | |") + + # PINS / class-crack candidate pool, reach-134, smallest m2c-mismatch first + pins134 = sorted([x for x in by_lever.get("PINS", []) if x["reach"] >= 134], + key=lambda x: (x["mismatch"] if x["mismatch"] is not None else 999, x["nins"] or 9999)) + L += ["", f"## PINS / permuter-class — reach-134 near-misses ({len(pins134)} fns; class-crack candidate pool)", + "", "| addr | nins | m2c-mismatch | reach |", "|---|---|---|---|"] + for x in pins134[:a.top]: + L.append(f"| {x['addr']} | {x['nins']} | {x['mismatch']} | {x['reach']} |") + if len(pins134) > a.top: + L.append(f"| …+{len(pins134)-a.top} more | | | |") + + # the named T3 class exemplars + their census row + reach + rowmap = {x["addr"]: x for x in rows} + L += ["", "## Named class-crack exemplars (T3 — the Max spikes)", "", + "| addr | class | census bucket | nins | reach |", "|---|---|---|---|---|"] + for addr, note in CLASS_EXEMPLARS.items(): + x = rowmap.get(addr) + if x: + L.append(f"| {addr} | {note} | {x['bucket']} | {x['nins']} | {x['reach']} |") + else: + L.append(f"| {addr} | {note} | (not a current stub — matched or split) | | |") + + # STUB honesty tally + stub = by_lever.get("STUB", []) + from collections import Counter + sc = Counter(x["bucket"] for x in stub) + L += ["", "## STUB residual (honest, no lever — P9)", "", + f"{len(stub)} fns: " + ", ".join(f"{b}={n}" for b, n in sc.most_common()), ""] + + open(ROOT / "docs/exemplar_curriculum.md", "w").write("\n".join(L) + "\n") + print(f"wrote docs/exemplar_curriculum.md + .run/exemplar_routing.json") + print(f" {len(rows)} residual stubs; WAVE {len(by_lever.get('WAVE',[]))} " + f"(reach-134 {len(wave134)}) | STRUCT {len(by_lever.get('STRUCT',[]))} | " + f"PINS {len(by_lever.get('PINS',[]))} (reach-134 {len(pins134)}) | STUB {len(stub)}") + + +if __name__ == "__main__": + main()