From 6ed0f77b6a1085deccadb3b2d5cec13f68e768cb Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:26:02 -0600 Subject: [PATCH] =?UTF-8?q?fix(recover):=20gate=20the=20RAW=20draft=20firs?= =?UTF-8?q?t=20=E2=80=94=20macro-externs=20is=20a=20fallback,=20not=20the?= =?UTF-8?q?=20measurement=20(+=20classify=5Ffail=20drops=20orphan=20notes)?= =?UTF-8?q?;=20cookbook=20=C2=A7313?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recover_integration's macro-externs stage rewrote a draft's callee extern to the FLEET macro's signature and then gated only the rewrite. func_ADDR names are per-address, not per-function, so another overlay's 'extern void func_8017C338(void)' replaced this overlay's correct 4-arg decl and manufactured the CC1-FAIL it reported as the draft's failure. The untouched draft banks byte-identical (ov_SC03_012:func_8017BEBC, 246 ins, banked in the previous commit). reconcile_and_gate(draft_rewrite=) now gates raw (pass 1a) then rewrites only what raw refused (pass 1b), records the winning variant per fn, and re-gates that variant in pass 2. harvest_verify.classify_fail kept the 'note:' half of a benign warning pair and labelled a built draft CC1-FAIL with it; notes now drop with their warnings. Negative-controlled over 5 diagnostic shapes — only warning+note-only changed (to the honest no-diagnostic label). R39/R57/R32. Cookbook 920 -> 921 sections, index green. --- docs/cookbook-index.md | 10 ++++-- docs/matching-cookbook.md | 16 ++++++++++ tools/harvest_verify.py | 10 +++++- tools/recover_integration.py | 60 ++++++++++++++++++++++++++++++------ 4 files changed, 83 insertions(+), 13 deletions(-) diff --git a/docs/cookbook-index.md b/docs/cookbook-index.md index 3244c83e8..cf6cfad0a 100644 --- a/docs/cookbook-index.md +++ b/docs/cookbook-index.md @@ -2,7 +2,7 @@ > **Generated by `tools/cookbook_index.py` — do not hand-edit** (R33). Regenerate after adding a cookbook section. > -> `docs/matching-cookbook.md` is ~716 KB / 920 sections. Grepping it blind is how three P30 wave-1 agents each "discovered" an idiom that was already written down. **Start here, then read the section.** A section appears under every symptom it addresses. +> `docs/matching-cookbook.md` is ~716 KB / 921 sections. Grepping it blind is how three P30 wave-1 agents each "discovered" an idiom that was already written down. **Start here, then read the section.** A section appears under every symptom it addresses. **How to use:** name what you SEE in the diff (a stolen delay slot, an extra `la`, a swapped register pair, a `conflicting types` error), find that symptom below, read those sections first. If nothing fits, THEN grind — and add a section when you win. @@ -825,7 +825,7 @@ - **ADDENDUM** — to §199-G — A `default:` LABEL GROUPED ONTO THE LAST CASE REMOVES THE `j default` TAIL, EVEN THOUGH THE 2-NODE HEADER STAYS ALL-POSITIVE L27385 - **§301** — AN INTERNAL `j` CARRIES `R_MIPS_26 .text`: rtu/match_one "MATCH" COULD NOT SEE WHICH LABEL A `j` TAKES — FIXED (`jrel`), AND THE TWO DRAFT SHAPES IT HID (P31 S62 T1; byte-proven 2/2, negative-controlled over 3,297 stubs) L29645 -### build graph, splat & the harness (167) +### build graph, splat & the harness (168) - **§4** — Flag/toolchain gotchas L190 - **Build** — mechanism — per-file opt override (splat resegmentation) L288 @@ -994,8 +994,9 @@ - **§294** — ADDENDA HARVESTED FROM THE S61 GEN0/MAXTOK/MAIN/DEEPSEEK BATCH (waves ab8/ab16/ab24/ab32 · g0a–g0f · m0a · ds1/ds2): 99 candidates, 52 covered, 15 notes → 10 addenda, 29 notes → 5 new laws (§295–§299), 3 refuted L29133 - **§303** — MODULE ISLAND TABLES: DERIVE THE PADS AT BUILD TIME, PEEL NOTHING — THE §154-A/§260 "island-pads"/"island-blocked" WALLS DISSOLVE (P31 S62 T3a; byte-proven md_SC03_076 func_801F0A9C + func_801F0F28, sha 9a165e36…) L29737 - **§305** — "CARVE-REFUSED" AT GATE TIME IS THREE NAMED, DETERMINISTIC CLASSES — NONE OF THEM A CARVE (P31 S62 T3; 28 resolver drafts autopsied 28/28) L29796 +- **§313** — A RECOVERY STAGE THAT REWRITES THE DRAFT IS PART OF THE MEASUREMENT: GATE THE RAW DRAFT FIRST, THE REWRITE ONLY AS A FALLBACK (P31 S65 t5r-recovery; byte-proven func_8017BEBC, 246 ins) L30159 -### process, measurement & doctrine (107) +### process, measurement & doctrine (108) - **§8e** — The jtbl ALIGNMENT LAW + the pad-spec filter — multi-table .rodata spans (Phase 29, byte-proven; `.run/probe_jtbl/verdict.md`) L530 - **§3-The** — mechanism: game-code dedup is SOURCE-LEVEL, not an object swap (R-D1, the key lesson) L926 @@ -1104,6 +1105,7 @@ - **What** — I could not verify L28943 - **§293** — THE "WHOLE-OBJECT GATE NEEDS EVERY SIBLING" LAW, WRITTEN PROPERLY: IT IS THREE LAWS, AND THE LOAD-BEARING ONE IS "THE BASELINE, NOT THE SIBLINGS" (P31 S61; byte-proven on 15 binaries in one night) L29083 - **§294** — ADDENDA HARVESTED FROM THE S61 GEN0/MAXTOK/MAIN/DEEPSEEK BATCH (waves ab8/ab16/ab24/ab32 · g0a–g0f · m0a · ds1/ds2): 99 candidates, 52 covered, 15 notes → 10 addenda, 29 notes → 5 new laws (§295–§299), 3 refuted L29133 +- **§313** — A RECOVERY STAGE THAT REWRITES THE DRAFT IS PART OF THE MEASUREMENT: GATE THE RAW DRAFT FIRST, THE REWRITE ONLY AS A FALLBACK (P31 S65 t5r-recovery; byte-proven func_8017BEBC, 246 ins) L30159 ### (unbucketed — title matched no symptom vocabulary) (288) @@ -2319,6 +2321,7 @@ - **§310** — A TWO-OPERAND `subu`'s DESTINATION TIES ONLY TO THE OPERAND THAT IS BLOCK-LOCAL: SHARE *ONE* SIDE ACROSS BLOCKS TO CHOOSE WHICH OPERAND'S REGISTER IT LANDS IN (P31 S64 t5j-t5m; byte-proven func_8017F578) L30060 - **§311** — A JOIN-BLOCK STORE CAN SWALLOW AN ARM'S LOAD-DELAY `nop`: WHEN ONE ARM'S VALUE COMES FROM A LOAD, DUPLICATE THE STORE INTO BOTH ARMS (P31 S64 t5j-t5m; byte-proven func_8017D7E0) L30091 - **§312** — A BARE RELATIONAL IN AN `if` GIVES THE `slt` NO TARGET: ASSIGN THE COMPARISON INTO THE ALREADY-ALLOCATED VARIABLE TO CHOOSE ITS DESTINATION REGISTER, AT ZERO INSTRUCTION COST (P31 S64 t5o-t5r; byte-proven func_80180DCC) L30133 +- **§313** — A RECOVERY STAGE THAT REWRITES THE DRAFT IS PART OF THE MEASUREMENT: GATE THE RAW DRAFT FIRST, THE REWRITE ONLY AS A FALLBACK (P31 S65 t5r-recovery; byte-proven func_8017BEBC, 246 ins) L30159 --- @@ -3251,3 +3254,4 @@ Notes routinely quote that as a section id. This table resolves it. Grep bait: ` | L30060 | §310 | A TWO-OPERAND `subu`'s DESTINATION TIES ONLY TO THE OPERAND THAT IS BLOCK-LOCAL: SHARE *ON | | L30091 | §311 | A JOIN-BLOCK STORE CAN SWALLOW AN ARM'S LOAD-DELAY `nop`: WHEN ONE ARM'S VALUE COMES FROM | | L30133 | §312 | A BARE RELATIONAL IN AN `if` GIVES THE `slt` NO TARGET: ASSIGN THE COMPARISON INTO THE ALR | +| L30159 | §313 | A RECOVERY STAGE THAT REWRITES THE DRAFT IS PART OF THE MEASUREMENT: GATE THE RAW DRAFT FI | diff --git a/docs/matching-cookbook.md b/docs/matching-cookbook.md index 70709f5c3..0c3f903d2 100644 --- a/docs/matching-cookbook.md +++ b/docs/matching-cookbook.md @@ -30155,3 +30155,19 @@ The two spellings cost the **same instruction count** — a bare relational has **BYTE EVIDENCE.** `func_80180DCC`, 66 ins, all quotes from `match_one --json` on the pinned triple. v0 (backlog draft, `register s16 s1` pin + call-site `(s16)` casts) → closeness 9, `STRENGTH/sll!=sra`. Unpinning `s1` → closeness 6, same sig (block 2 clean, block 1's `sll/sra/slt/bnez` still wrong). v6 (pin `register s32 v1 __asm__("$3")`, hoist `v1 = (s16)s1; v0 = (s16)v0;` as statements ahead of a still-bare `if (v0 < v1)`) → **6 → 2**: `{"status":"near","closeness":2,"residual":[[34,"0043182a slt v1,v0,v1","0043102a slt $v0, $v0, $v1"],[35,"14600013 bnez v1,…","14400013 bnez $v0, .L80180EA8"]],"verdict":{"klass":"REGALLOC-PERM","sig":"REGALLOC-PERM/$v1>$v0","detail":{"map":{"$v1":"$v0"}}}}`. v7, the **only** change being `if (v0 < v1) {…}` → `v0 = (v0 < v1); if (v0 != 0) {…}` → **2 → 0**: `{"status":"match","closeness":0,"residual":[],"verdict":{"klass":"MATCH"}}`. **BOUND.** (1) n = 1. (2) Only the **v6 → v7** delta is solo-proven (§266): the pins and the cast hoist arrived in compound edits and are co-requisites of the shape, not separately ablated — the pin is what makes "the variable's hard register" a nameable thing here, and an unpinned local would only move the dest to whatever that local got. (3) Routing: on a REGALLOC-PERM whose single wrong register is the compare's **DEST**, try this first — one statement, zero bytes; go to §164-39's fence or its addendum's transposition only when the wrong register is one of the compare's **INPUTS**. (4) Says nothing about compare-against-constant shape (`slti`/`sltiu` immediate-vs-register is §194-L's 2-D lookup) or about signedness (§280). + +## §313 — A RECOVERY STAGE THAT REWRITES THE DRAFT IS PART OF THE MEASUREMENT: GATE THE RAW DRAFT FIRST, THE REWRITE ONLY AS A FALLBACK (P31 S65 t5r-recovery; byte-proven func_8017BEBC, 246 ins) + +*(Tooling law, not a codegen idiom — the §65/§121 family. §121 established `macro-externs`: a draft's `extern` for a `DEFINE_func_*`-defined callee must match the macro's own definition head, because a guessed `extern int f();` collides with the macro's real `void f(s32)` and poisons the whole-binary build. This entry does not retract §121; it bounds it. R57 is the governing rule — an instrument's own write path is part of the instrument.)* + +**THE TELL.** `recover_integration.py --probe-only` classifies a stranded draft `MATCH` in its real TU (`blocker_probe` compiles the draft where it will live and compares the function's bytes), yet the driver's own pass reports `banked 0/1` and the classified ledger names a blocker. The two disagree because they are not measuring the same text: the probe reads the draft, the driver rewrites it and then measures the rewrite. + +**THE MECHANISM — PROVEN, NOT INFERRED.** `func_ADDR` names are **per-address, not per-function**: the same address name exists in many overlays and denotes DIFFERENT code in each. `macro_def_sig_map()` is fleet-wide, so `macro-externs` can install another overlay's signature over a correct local one. Measured: the ov_SC03_012 draft of `func_8017BEBC` declared its callee `extern int func_8017C338(short *, short *, short *, int);` — correct for this overlay, and what the original asm's call site emits. The stage replaced it with the fleet macro's head, `extern void func_8017C338(void);`, and cc1 then refused the 4-argument call. The driver reported that as the DRAFT's failure and moved on. + +**THE COST.** A byte-perfect 246-instruction body sat unbanked, and the raw text was never gated once — `stage_drafts` copies into the run dir, `macro-externs` mutates in place, and every later pass reads the mutated copy. Spliced by hand, the untouched draft built `sha1 e9483e61… == config/check.ov_SC03_012.sha (BYTE-IDENTICAL)` on the first try. `gate_stage --drafts ` banked it: `{"drafts":1,"banked":1,…}`. + +**THE FIX (shipped).** `reconcile_and_gate(..., draft_rewrite=)` gates in a ladder — **pass 1a** the raw drafts, **pass 1b** only what 1a refused, with the rewrite applied — and records per-fn which variant won so pass 2 re-gates that one. Cost is one extra gate round, and only when something failed raw. Same shape as §53's carve-law: a 0% from a tool that changed its own input is not evidence about the input. + +**THE SECOND DEFECT IT EXPOSED (also shipped).** `harvest_verify.classify_fail`'s §58 red-herring guard drops `warning:` lines but kept their orphaned `note:` half, so the label read `CC1-FAIL: …/engine_core.h:57282: note: this is the location of the previous definition` — a note whose antecedent was the benign fleet-wide `"DEFINE_func_80181538" redefined` warning, on a draft that had in fact built. Notes are now dropped with their warnings; the honest `CC1-FAIL(no-diagnostic)` label surfaces the gap instead (R32). Negative-controlled over five diagnostic shapes (real error + note, plumbing conflict, gcc-2.7.2 no-`error`-word hard error, `undefined reference`, warning+note only): only the last changed. + +**BOUND.** (1) n = 1 for the bank; the same raw-first gate over the other three CC1-FAIL strandeds of the same wave banked 0 — they are genuine near-misses, so this is a real-but-narrow class, not a general "the rewrite is wrong" verdict. (2) `macro-externs` keeps its §121 warrant: it is now a fallback, and it still fires for the drafts a raw gate refuses. (3) The deeper hazard — a fleet-wide signature map applied to a per-overlay name (R48's collision class) — is NOT fixed here, only ordered around. A per-binary "is this `DEFINE_` actually instantiated in THIS TU?" guard is the real repair; it is unbuilt because it is a new refusal-check and needs an R39 negative control over every draft the stage ever helped. diff --git a/tools/harvest_verify.py b/tools/harvest_verify.py index 97f048ce4..996a982ea 100644 --- a/tools/harvest_verify.py +++ b/tools/harvest_verify.py @@ -147,7 +147,15 @@ def classify_fail(got_sha): # useless here, splice individually and read real cc1 stderr" as a manual step. Classify on # NON-warning lines only, and when nothing but warnings matched, surface the real error # instead of guessing (R32: report the gap, do not paper over it). - lines = [ln for ln in _last_err.splitlines() if 'warning:' not in ln] + # A `note:` line is NEVER a cause — gcc emits it only as the SECOND half of a diagnostic pair, + # and when the first half is a benign warning the note is orphaned by the filter above and then + # wins the label. Measured P31 S65: a byte-perfect 246-ins draft was labelled + # CC1-FAIL: …/engine_core.h:57282: note: this is the location of the previous definition + # whose antecedent was the fleet-wide benign `"DEFINE_func_80181538" redefined` WARNING — the + # draft had in fact built and banked whole-binary-identical. Same §58 red-herring family: drop + # the notes with their warnings, and let the real error (or the honest gap) carry the label. + lines = [ln for ln in _last_err.splitlines() + if 'warning:' not in ln and not re.search(r':\s*note:', ln)] for ln in lines: if _PLUMBING.search(ln): return 'PLUMBING: ' + _squeeze(ln) diff --git a/tools/recover_integration.py b/tools/recover_integration.py index 97e88a70b..8ab90e427 100644 --- a/tools/recover_integration.py +++ b/tools/recover_integration.py @@ -290,11 +290,14 @@ def main(): smap = stub_map(a.binary) - def reconcile_and_gate(targets, propagate, commit): + def reconcile_and_gate(targets, propagate, commit, draft_rewrite=True): """no-proto the targets' conflicting caller decls, then gate each SPLIT-file group separately — harvest_verify substitutes into ONE --src/--asm-subdir per call, so _after/_a/_o0 drafts must be gated against their own split (§39). run_gate self-filters the drafts dir to each split's stubs; - its per-group propagate is idempotent (dedup_propagate skips registered addrs).""" + its per-group propagate is idempotent (dedup_propagate skips registered addrs). + + `draft_rewrite=False` suppresses the DRAFT-TEXT stages (macro-externs) so the drafter's own + text is what the gate judges — see the raw-first ladder at PASS 1a (P31 S65).""" before = git_dirty() open(listf, "w").write("\n".join(targets) + "\n") @@ -305,7 +308,7 @@ def main(): raise SystemExit(f"[recover] fix_arity_callers failed: {(r.stderr or r.stdout)[-300:]}") print(" " + (r.stdout.strip().splitlines()[-1] if r.stdout.strip() else "(fix_arity_callers: no output)")) - if "macro-externs" in stages: # P31 T6 — §121: a draft's decl of a DEFINE_-defined callee + if "macro-externs" in stages and draft_rewrite: # P31 T6 — §121: a draft's decl of a DEFINE_-defined callee # must match the macro's OWN definition head (a guessed `extern int f();` collides with # the macro's real `void f(s32)` — measured: ONE such draft poisoned an entire probe # group's whole-binary builds). Draft-text only; drafts with no DEFINE_ callee untouched. @@ -402,11 +405,49 @@ def main(): "propagated": propagated, "tier": tier} # ---- PASS 1: reconcile+gate ALL candidates (no propagate) to find the bankable set. - print("[recover] pass 1 — find bankable set") - s1 = reconcile_and_gate(fns, propagate=False, commit=False) - banked = sorted(s1.get("verified", [])) - print(f"[recover] pass 1 banked {len(banked)}/{len(fns)}") - restore() # exact — undo pass-1's caller edits AND the substituted defs + # + # 1a gates the RAW draft text; 1b re-gates ONLY what 1a refused, with the draft-text rewrite + # (macro-externs) applied. Order matters and was inverted until P31 S65: macro-externs rewrites + # a draft's callee decl to the `DEFINE_` macro's own head, but a same-named func_ADDR in ANOTHER + # overlay is a DIFFERENT function — so the "fix" can install a wrong signature (measured: + # `extern int func_8017C338(short*,short*,short*,int)` -> `extern void func_8017C338(void)` on a + # byte-perfect 246-ins draft of func_8017BEBC, which then CC1-FAILed and was reported as the + # DRAFT's failure; raw banked whole-binary-identical on the first try). A stage that mutates the + # thing it is measuring must never be the only variant gated (R57). + raw_drafts = {fn: open(os.path.join(REPO, dd, fn + ".c")).read() + for fn in fns if os.path.exists(os.path.join(REPO, dd, fn + ".c"))} + + def put_draft(fn, text): + open(os.path.join(REPO, dd, fn + ".c"), "w").write(text) + + print("[recover] pass 1a — gate the RAW drafts (no draft-text rewrite)") + s1 = reconcile_and_gate(fns, propagate=False, commit=False, draft_rewrite=False) + variant = {fn: "raw" for fn in sorted(s1.get("verified", []))} + print(f"[recover] pass 1a banked {len(variant)}/{len(fns)} raw") + restore() # exact — undo pass-1a's caller edits AND the substituted defs + + rest = [fn for fn in fns if fn not in variant] + if rest and "macro-externs" in stages: + for fn in rest: # 1a left the text untouched; be explicit anyway + if fn in raw_drafts: + put_draft(fn, raw_drafts[fn]) + print(f"[recover] pass 1b — macro-externs rewrite, {len(rest)} draft(s) the raw gate refused") + s1b = reconcile_and_gate(rest, propagate=False, commit=False, draft_rewrite=True) + for fn in sorted(s1b.get("verified", [])): + variant[fn] = "macro-externs" + print(f"[recover] pass 1b banked " + f"{sum(v == 'macro-externs' for v in variant.values())}/{len(rest)} rewritten") + restore() + elif rest: + print(f"[recover] pass 1b skipped ({len(rest)} unbanked; macro-externs not in --stages)") + + banked = sorted(variant) + print(f"[recover] pass 1 banked {len(banked)}/{len(fns)} " + f"(raw {sum(v == 'raw' for v in variant.values())}, " + f"macro-externs {sum(v == 'macro-externs' for v in variant.values())})") + for fn, how in sorted(variant.items()): # pass 2 must re-gate each winner's OWN variant + if how == "raw" and fn in raw_drafts: + put_draft(fn, raw_drafts[fn]) if not banked: print(json.dumps({"banked": [], "propagated": 0, "fleet": s1.get("fleet_pct")})); return @@ -418,7 +459,8 @@ def main(): for p in glob.glob(os.path.join(REPO, dd, "*.c")): if os.path.basename(p)[:-2] not in banked: os.remove(p) - s2 = reconcile_and_gate(banked, propagate=propagate, commit=a.commit) + # variants are already baked into dd/.c by pass 1 — never re-rewrite here. + s2 = reconcile_and_gate(banked, propagate=propagate, commit=a.commit, draft_rewrite=False) banked2 = sorted(s2.get("verified", [])) print(f"[recover] pass 2 banked {len(banked2)}/{len(banked)} propagated groups +{s2.get('propagated')} " f"fleet {s2.get('fleet_pct')}%")