#!/usr/bin/env python3 """Phase-28 T4: EXTEND the existing dedup registry to newly-onboarded binaries. THE GAP THIS FILLS (byte-measured, P28 T3-A). `tools/new_overlay.sh` produces a byte-clean binary that is NOT a citizen of the shared-body ecosystem: its `.c` includes only `common.h` (never `../shared/engine_core.h`), so not one shared engine body can reach it, and it appears in ZERO of `config/dedup.us.yaml`'s groups. The 4 SC07 overlays the Phase-27 disc audit onboarded therefore sat at ~80 matched / ~2,400 stubs while their siblings were ~2,150 matched — and 1,689 registry groups still read "134 binaries", never 138. WHY THE EXISTING TOOL CANNOT DO IT. `dedup_propagate` is built for CRACK -> AUTHOR MACRO -> INSTANTIATE: `--auto-from` scans a source overlay's INLINE DEFS (it planned only 11 fns here, skipping the ~1,600 whose bodies are ALREADY `DEFINE_func_*` macros in engine_core.h), and `--addr` dies with "no source overlay has it matched" because no overlay holds an inline def — every one of the 134 uses the macro. Extending an existing MACRO-BACKED group to a new binary is a different operation, and nothing implemented it. WHY IT IS SAFE (the correctness argument — this is a byte-gate feeder, so it must be explicit). `h_exact` is the SHA1 of RAW INSTRUCTION BYTES (tools/sig_image.py). Two instances sharing an h_exact are therefore identical *including* their `jal`/`lui`/`%lo` reloc immediates — same callee addresses, same data addresses, same symbols. So the body that compiles byte-identically at one member compiles byte-identically at the other, with no remap at all. (This is exactly why dup_report calls h_exact "guaranteed byte-match" and h_norm "candidate-only".) The whole-binary byte-gate remains the sole arbiter (G3/P9): a wrong instantiation changes the bytes and fails SHA1. A bug here can make this tool FAIL TO BANK; it cannot make it falsely bank. REUSE, DON'T REBUILD (R33). This owns only the SET COMPUTATION and the REGISTRY EDIT. The splice and the gate are `tools/harvest_verify.py` verbatim — which already derives each stub's home TU from the corpus oracle, chunks + bisects, and reverts on failure. h_exact members are byte-identical by construction, so the happy path is ONE build per binary (not one per function). tools/dedup_extend.py --binaries ov_SC07_006,ov_SC07_007,ov_SC07_010,ov_SC07_011 [--check-only] [--chunk N] [--limit N] """ import argparse import glob import json import os import re import subprocess import sys import yaml sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import corpus import family_hseq # has_mid_jr — the ONE jr oracle (R33) import family_remap as FR # stream_words/nins_of REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PY = sys.executable REGISTRY = "config/dedup.us.yaml" SHARED_INCLUDE = '#include "../shared/engine_core.h"' EXTEND_DIR = ".run/extend" def sig_hashes(binary): """{vram_int: h_exact} from the binary's sig — the ORIGINAL bytes, independent of splat.""" out = {} p = os.path.join(REPO, f".run/sig.{binary}.jsonl") for line in open(p): d = json.loads(line) out[int(d["addr"], 16)] = d["h_exact"] return out def load_groups(): doc = yaml.safe_load(open(os.path.join(REPO, REGISTRY))) groups = doc if isinstance(doc, list) else doc.get("groups", doc) if not isinstance(groups, list): sys.exit(f"dedup_extend: unexpected {REGISTRY} shape: {type(groups).__name__}") return doc, groups def add_members_surgical(additions): """Append binaries to each group's `binaries: [...]` list by TEXT EDIT, in place. NEVER `yaml.safe_dump` this file. The first cut of this tool round-tripped it through safe_dump and silently destroyed BOTH of the things a human needs from it (H5 — "never silently drop comments on a rewrite"): * all 47 comment lines — including the curated Phase-11 header explaining WHY the share is source-level (the linker cannot excise bytes interior to an object) — dumped to nothing; * every `vram: 0x80162FF4` re-serialized as `vram: 2148937716` (PyYAML parses YAML-1.1 hex to int, and dumps int as decimal), making 1,832 entries unreadable. It was invisible to every gate: dedup-check passed 1840/0 and check-all stayed 140/140, because `_addr()` accepts both forms — the data was fine and the DOCUMENT was ruined. A formatting-destructive write that all your oracles call green is exactly the class this project keeps re-learning: the gate measures bytes, not intent. `additions` = {group_id: [binary, ...]}. Idempotent: a binary already listed is skipped. """ p = os.path.join(REPO, REGISTRY) lines = open(p).read().splitlines(keepends=True) cur, n = None, 0 for i, ln in enumerate(lines): m = re.match(r"^\s*-?\s*id:\s*(\S+)\s*$", ln) if m: cur = m.group(1) continue if cur and cur in additions and re.match(r"^\s*binaries:\s*\[", ln): add = [b for b in additions[cur] if re.search(rf"\b{re.escape(b)}\b", ln) is None] if add: lines[i] = ln.rstrip("\n").rstrip()[:-1].rstrip() + ", " + ", ".join(add) + "]\n" n += len(add) cur = None open(p, "w").write("".join(lines)) return n def _addr(v): return int(v, 16) if isinstance(v, str) else int(v) def plan_for(binary, groups): """[(group, vram, macro)] for every h_exact group this binary could join, plus the skip tally. A group is extendable into `binary` iff: it is h_exact tier, the binary is not already a member, the binary has a LIVE STUB at the group's vram, that stub's original bytes hash to the group's recorded h_exact (the C1 equivalence the registry itself asserts), and it is NOT a jr/switch function (see below). THE jr GUARD (cookbook §53). A function with a mid-body `jr` on a non-$ra register dispatches through a compiler-generated jump table the LINKER must place at the sibling's exact address — a per-sibling `jtbl_carve` + interleave (`jtbl_family_bank.py`). This tool has no carve step and a newly-onboarded overlay has no `_jr_*` split at all, so a jr body instantiated here would leave its table unplaced and the gate would (correctly) reject it. The guard is PREVENTIVE: it skips and REPORTS rather than feeding the gate drafts that cannot pass (R32 — a skip is only honest if it is counted and named). On the SC07 set it currently skips ZERO: no jr function is in the extendable set. ⚠️ R14 — WHAT THIS GUARD IS *NOT*. It does NOT explain the 12 DIFFs the first run produced (`func_80162FF4/801630C4/80163194/8016325C`, the same 4 in every overlay, 12 of 6457 = 0.19%). I first assumed those were the jr class because ov_SC01_077 hosts them in `_jr_8017A4AC.c` / `_jr_80182268.c` — a natural read, and WRONG: `has_mid_jr` is **False** for all four (33-52 ins, no jump table). They merely LIVE in a carved jr-REGION split (the carve region for `func_8017A4AC` sweeps in every function in its address range), which says nothing about them. Their DIFF cause is UNDIAGNOSED and logged for follow-up; they are correctly left as stubs by the gate. Hosting file != function class — do not infer one from the other. Detector reused verbatim from `family_hseq.has_mid_jr` (R33 — one oracle, not two). """ sig = sig_hashes(binary) stubs = corpus.stubs(binary) out, skips = [], {} for g in groups: if g.get("tier") != "h_exact": continue if "vram" not in g or "binaries" not in g: skips["verbose-form"] = skips.get("verbose-form", 0) + 1 continue # not position-locked if binary in g["binaries"]: continue vram = _addr(g["vram"]) if vram not in stubs: continue # not a live stub here (matched already, or absent) if sig.get(vram) != g["hash"]: continue # DIFFERENT CODE at the same vram — the point of the check words = FR.stream_words(binary, vram, FR.nins_of(binary, vram)) if words and family_hseq.has_mid_jr(words): skips["jr-needs-carve"] = skips.get("jr-needs-carve", 0) + 1 continue # §53: route via jtbl_family_bank, not here out.append((g, vram, g["func"])) return out, skips def ensure_include(binary, apply=True): """Add the shared-header include if absent. Byte-neutral: engine_core.h is only `#define`s, so including it without instantiating emits no code (proven: ov_SC07_006 built 7ca772be with the include + one macro). Returns True if the file was changed.""" p = os.path.join(REPO, f"src/{binary}/{binary}.c") t = open(p).read() if SHARED_INCLUDE in t: return False if apply: t = t.replace('#include "common.h"', '#include "common.h"\n' + SHARED_INCLUDE, 1) open(p, "w").write(t) return True def write_drafts(binary, plan): d = os.path.join(REPO, EXTEND_DIR, binary) subprocess.run(["rm", "-rf", d], check=False) os.makedirs(d, exist_ok=True) for g, vram, macro in plan: fn = f"func_{vram:08X}" body = f"{macro}() /* dedup: shared engine-core @0x{vram:08x} (src/shared) */\n" open(os.path.join(d, fn + ".c"), "w").write(body) return d def gate(binary, drafts_dir, chunk, ladder=True): """The whole-binary byte-gate is the sole arbiter (G3/P9). Returns the banked fn set. LADDER (S46): this called `harvest_verify` VERBATIM, which is the bare gate with no recovery. Measured cost of that: 0 of 142 extendable groups banked, and the classified reasons were 118 PLUMBING / 21 CC1-FAIL / 3 DIFF — i.e. ~1 in 50 was a real byte divergence and everything else was a declaration conflict in the TARGET TU (`conflicting types for memcpy / ApplyMatrixSV / D_800AE620 / func_8014C568`). `gate_stage` wraps the same gate in exactly the reconcile ladder those need (canon_resident_calls -> cast_call_sites -> sig_unify -> harvest_verify), and it carries the §156 reconcile LEDGER so a failed candidate cannot leave a fleet-shared edit behind. Measured on the same failure class in the S46 wave residue: 6 of 19 PLUMBING recovered (~32%). GATE_NO_ARITY=1 is set for the child: gate_stage's arity pre-pass writes the fleet-shared engine_core.h BEFORE the gate, and on a failing draft that edit can survive — the F1 defect that broke 141 of 213 binaries in S45 and made every later gate report `near`. The ladder's other rungs are draft-local and safe. """ good = open(os.path.join(REPO, f"config/check.{binary}.sha")).read().split()[0] vout = f".run/extend_verified.{binary}.txt" fout = f".run/extend_failed.{binary}.txt" if ladder: cmd = [PY, "tools/gate_stage.py", "--binary", binary, "--drafts", os.path.relpath(drafts_dir, REPO), "--no-propagate", "--verified-out", vout, "--failed-out", fout] else: cmd = [PY, "tools/harvest_verify.py", "--binary", binary, "--out", f"build/{binary}/{binary}", "--good-sha", good, "--drafts", os.path.relpath(drafts_dir, REPO), "--chunk", str(chunk), "--verified-out", vout, "--failed-out", fout] env = dict(os.environ, GATE_NO_ARITY="1") subprocess.run(cmd, cwd=REPO, timeout=7200, env=env) p = os.path.join(REPO, vout) return set(open(p).read().split()) if os.path.exists(p) else set() def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--binaries", required=True, help="comma list of newly-onboarded binaries to wire in") ap.add_argument("--chunk", type=int, default=512, help="drafts per build (h_exact is byte-identical by construction, so the happy " "path is ONE build; harvest_verify bisects on failure). default 512") ap.add_argument("--limit", type=int, default=0, help="cap #groups per binary (0 = all)") ap.add_argument("--check-only", action="store_true", help="print the plan, touch nothing") a = ap.parse_args() os.chdir(REPO) doc, groups = load_groups() targets = [b.strip() for b in a.binaries.split(",") if b.strip()] dirty = subprocess.run("git status --porcelain -- config/ src/", shell=True, capture_output=True, text=True).stdout.strip() if dirty and not a.check_only: sys.exit("dedup_extend: config/ or src/ is dirty — commit first so a failed gate reverts " "cleanly.\n" + dirty[:400]) total_banked, total_planned = 0, 0 for b in targets: plan, skips = plan_for(b, groups) if a.limit: plan = plan[:a.limit] total_planned += len(plan) print(f"[{b}] {len(plan)} extendable h_exact group(s); " f"include {'ABSENT -> add' if ensure_include(b, apply=False) else 'present'}" + (f"; skipped {skips}" if skips else "")) if a.check_only or not plan: continue added_include = ensure_include(b) # True IFF this run inserted the line d = write_drafts(b, plan) banked = gate(b, d, a.chunk) print(f"[{b}] BANKED {len(banked)} / {len(plan)}") total_banked += len(banked) if not banked: # ONLY undo what THIS run did (§61/§63). The revert is an INVERSE TRANSFORM, not a # snapshot restore, so firing it on a binary that ALREADY had the include strips a # load-bearing line: every `DEFINE_func_*()` in that overlay stops resolving. It was # unconditional until Phase 29 SESSION-19, where a 0-banked run over 135 already-wired # binaries removed the include from all 135 at once. Invisible to every byte-gate (R34) # because the damage lands AFTER the last gate has run. if added_include: ensure_include_revert(b) continue adds = {g["id"]: [b] for g, vram, _ in plan # registry: only what the GATE accepted (P9) if f"func_{vram:08X}" in banked} n = add_members_surgical(adds) # TEXT edit — never safe_dump (see the docstring) print(f"[{b}] registry: +{n} membership(s)") print(f"\n=== dedup_extend: banked {total_banked} / {total_planned} planned " f"across {len(targets)} binaries ===") if a.check_only: print("(--check-only: no files touched)") def ensure_include_revert(binary): p = os.path.join(REPO, f"src/{binary}/{binary}.c") t = open(p).read().replace('#include "common.h"\n' + SHARED_INCLUDE, '#include "common.h"', 1) open(p, "w").write(t) if __name__ == "__main__": main()