From 70a99aa9f7ffd1d7572dd04db759828a8c1b59fa Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:30:03 -0600 Subject: [PATCH] =?UTF-8?q?fix(phase-26a):=20A3=20=E2=80=94=20the=20byte-g?= =?UTF-8?q?ate=20could=20only=20see=20ONE=20translation=20unit=20(96.6%=20?= =?UTF-8?q?of=20stubs=20unreachable)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit harvest_verify is the sole arbiter (G3/P9) and has never accepted a wrong match. It also could not REACH most of the work: it scanned the single file the caller passed as --src and silently dropped every draft whose stub lived elsewhere. An overlay's source spans up to 14 .c files, so: open overlay stubs it could not see : 56,742 of 58,717 (96.6%) ov_SC01_077 reachable by the gate : 13 of 264 (4.9%) -> 264 of 264 (100%) THREE of the six callers passed no --src at all (orchestrator.py, grinder.py, idiom_hunt.py) and so inherited gate_stage's `src = src or f"src/{binary}/{binary}.c"` default. For grinder.py that means 1,290 of its own 1,298 QUEUED FUNCTIONS COULD NEVER BANK, however good the permuter's output was. => Phase-22's "the permuter's fuel is exhausted" was never a safe conclusion. Re-test (A12). gate_stage knew the right answer and then handed the gate the wrong file: its negative control ALREADY globs every split .c to build bin_stubs. The default is now removed; --src is passed only when a caller deliberately restricts the gate to one TU. WHAT CHANGED, PRECISELY: only the SPLICE LOCATION. Each draft is now spliced into whichever TU actually holds its stub, derived from tools/corpus.py. Every TU links into the same image, so ONE `make build BINARY=` still gates them all — correct AND strictly fewer builds than the per-split re-gate it replaces. SAFETY (this is the byte-gate, so the argument is explicit): the VERDICT is untouched — `make build` + SHA1 == the locked hash. INCLUDE_ASM pastes the ORIGINAL assembly, so a wrong draft always changes the bytes and always fails SHA1. A bug in the splice can therefore make the tool FAIL TO BANK; it CANNOT make it falsely bank. The failure mode is conservative by construction. VERIFIED end-to-end (2 real builds, tree clean before and after): * discovery: 264 live stubs across 12 TUs (was: only those in the single --src file) * IDENTITY known-answer test: 3 drafts whose stubs live in THREE DIFFERENT split TUs (_jr_801734BC, _after, _jr_8012ACE0) — all discovered, spliced into their own files, built, SHA-matched, committed, restored. Final SHA d19c9580 BYTE-IDENTICAL. Under the old code all three were silently dropped as "not stubbed". * `git checkout -- src/` recovers, exactly as the docstring promises. Also derived rather than defaulted: --good-sha now reads config/check..sha (a caller that passed --binary but forgot --good-sha used to gate an overlay against RESIDENT's SHA), and match_one_closeness resolves the asm subdir PER FUNCTION — one subdir for a whole batch is the same single-TU bug, and pointing match_one at the wrong one scores a draft against a DIFFERENT function's asm, producing a phantom non-zero closeness that lands in the backlog and feeds reserved_walls(). No committed source or config changed, so no build artifact can have moved; the byte-gate was exercised twice and returned BYTE-IDENTICAL both times. --- tools/gate_stage.py | 40 ++++++++++++++---- tools/harvest_verify.py | 93 +++++++++++++++++++++++++++++++---------- 2 files changed, 104 insertions(+), 29 deletions(-) diff --git a/tools/gate_stage.py b/tools/gate_stage.py index a3de5a48f4..bb8d787698 100644 --- a/tools/gate_stage.py +++ b/tools/gate_stage.py @@ -27,6 +27,8 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import backlog REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(REPO, 'tools')) +import corpus # the derived corpus oracle (Phase 26-A) PY = ".venv/bin/python" # ov_SC01_077 defaults (the canonical harvest binary) OV = "ov_SC01_077" @@ -69,9 +71,14 @@ def _gate1(binary, src, asm, out, good_sha, d, verified_out=None, failed_out=Non binaries in parallel without cross-reading each other's results.""" vo = verified_out or ".run/harvest_verified.txt" fo = failed_out or ".run/harvest_failed.txt" - sh([PY, "tools/harvest_verify.py", "--binary", binary, "--src", src, "--asm-subdir", asm, - "--out", out, "--good-sha", good_sha, "--drafts", d, "--chunk", "1", - "--verified-out", vo, "--failed-out", fo], timeout=7200) + # --src is passed ONLY when a caller explicitly restricts the gate to one TU. Omitted, the gate + # derives each draft's home TU from the tree and splices it there (Phase 26-A) — every TU links + # into the same image, so one `make build` still gates them all. + cmd = [PY, "tools/harvest_verify.py", "--binary", binary] + if src: + cmd += ["--src", src] + sh(cmd + ["--out", out, "--good-sha", good_sha, "--drafts", d, "--chunk", "1", + "--verified-out", vo, "--failed-out", fo], timeout=7200) vp = os.path.join(REPO, vo) return [w for w in (open(vp).read().split() if os.path.exists(vp) else []) if w.startswith("func_")] @@ -92,8 +99,20 @@ def _dedup_group_count(): len(re.findall(r"^[A-Za-z]", open(p).read(), re.M)) -def match_one_closeness(fn, cpath, asm): - """('match',0) | ('near', n) | ('fail', None) via match_one (relocation-masked).""" +def match_one_closeness(fn, cpath, asm, binary=None): + """('match',0) | ('near', n) | ('fail', None) via match_one (relocation-masked). + + The asm subdir is derived PER FUNCTION from the stub that names it (Phase 26-A). One subdir for + a whole batch is the same single-TU bug: an overlay has TWELVE, and pointing match_one at the + wrong one scores a draft against a DIFFERENT function's asm — a phantom non-zero closeness that + then lands in the backlog as a matching failure and feeds reserved_walls().""" + if binary: + st = corpus.stubs(binary) + hit = next((s for s in st.values() if s.symbol == fn), None) + if hit is not None: + asm = hit.asm_dir + if not asm: + return ("fail", None) try: r = sh([PY, "tools/match_one.py", fn, "--c", cpath, "--asm-subdir", asm], timeout=180) except subprocess.TimeoutExpired: @@ -136,8 +155,13 @@ def _run_gate_locked(drafts, binary, src, asm, out, good_sha, propagate, source_ # 0/222 Bug-B). good_sha is normalized to the BARE hash: config/check..sha is sha1sum format # " ", but harvest_verify compares it against a bare sha1() — passing the whole line # never matches, so banking is 0 for EVERY binary incl. ov_SC01_077 (the 0/12 Bug-A). - src = src or f"src/{binary}/{binary}.c" - asm = asm or f"asm/{binary}/nonmatchings/{binary}" + # src/asm are NOT defaulted (Phase 26-A audit, HIGH). harvest_verify now DERIVES each draft's + # home TU from the tree, so inventing a default here would silently PIN the gate to the main .c. + # That default is exactly what capped orchestrator.py / grinder.py / idiom_hunt.py — the three + # callers that pass no src — to 13 of ov_SC01_077's 264 stubs (4.9%), and made 1,290 of the + # grinder's own 1,298 queued functions UNBANKABLE however good the permuter's output was. + # Note the negative control below already globs every split .c: this function knew the right + # answer and then handed the gate the wrong file. Pass src ONLY to deliberately restrict to one TU. out = out or f"build/{binary}/{binary}" good_sha = (good_sha or _check_sha(binary) or DEF_SHA).split()[0] draft_fns = sorted(os.path.basename(p)[:-2] for p in @@ -215,7 +239,7 @@ def _run_gate_locked(drafts, binary, src, asm, out, good_sha, propagate, source_ for fn in [f for f in draft_fns if f not in verified]: cpath = os.path.join(REPO, d, fn + ".c") body = open(cpath).read() if os.path.exists(cpath) else "" - kind, close = match_one_closeness(fn, cpath, asm) if body else ("fail", None) + kind, close = match_one_closeness(fn, cpath, asm, binary) if body else ("fail", None) meta = _manifest_class(fn) cm = re.search(r"//\s*@class:\s*(.+)", body) sm = re.search(r"//\s*@stuck:\s*(.+)", body) diff --git a/tools/harvest_verify.py b/tools/harvest_verify.py index b60a3e29b0..d567692796 100644 --- a/tools/harvest_verify.py +++ b/tools/harvest_verify.py @@ -14,14 +14,20 @@ Resident defaults; pass flags for another binary. python3 tools/harvest_verify.py # resident python3 tools/harvest_verify.py --chunk 6 """ -import subprocess, glob, os, re, hashlib, argparse +import subprocess, glob, os, re, sys, hashlib, argparse + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import corpus # the derived corpus oracle (Phase 26-A) — a draft's home TU is a FACT of the tree ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--binary', default='resident') -ap.add_argument('--src', default='src/resident/resident.c') -ap.add_argument('--asm-subdir', default='asm/resident/nonmatchings/resident') -ap.add_argument('--out', default='build/resident/resident') -ap.add_argument('--good-sha', default='8e17e02ff8954d07c979449198f7e1645046b353') +ap.add_argument('--src', default=None, + help='DEPRECATED and ignored for stub LOCATION. A draft is spliced into whichever TU ' + 'actually holds its stub, derived from the tree. Kept only as an optional FILTER: ' + 'if given, only drafts whose stub lives in this file are considered.') +ap.add_argument('--asm-subdir', default=None, help='DEPRECATED and ignored (derived per stub).') +ap.add_argument('--out', default=None, help='built image path (derived from --binary if omitted)') +ap.add_argument('--good-sha', default=None, help='locked SHA1 (read from config/check..sha if omitted)') ap.add_argument('--drafts', default='.run/drafts') ap.add_argument('--chunk', type=int, default=8) ap.add_argument('--verified-out', default='.run/harvest_verified.txt', @@ -29,7 +35,18 @@ ap.add_argument('--verified-out', default='.run/harvest_verified.txt', ap.add_argument('--failed-out', default='.run/harvest_failed.txt') a = ap.parse_args() -STUB = 'INCLUDE_ASM("' + a.asm_subdir + '", {fn});' # {fn} filled per function +# ---- derive the build target + the locked SHA from the binary, not from a caller's flag ---------- +# A caller that passed --binary but forgot --good-sha used to gate an overlay against RESIDENT's SHA +# (conservative — everything fails — but baffling). The Makefile and config/check..sha already +# state these facts; do not keep a second copy. +_OUT = {'main': 'build/us/SLUS_007.26', 'resident': 'build/resident/resident'} +if not a.out: + a.out = _OUT.get(a.binary, f'build/{a.binary}/{a.binary}') +if not a.good_sha: + p = f'config/check.{"us" if a.binary == "main" else a.binary}.sha' + if not os.path.exists(p): + sys.exit(f'harvest_verify: no --good-sha and no {p} — refusing to gate against an unknown SHA') + a.good_sha = open(p).read().split()[0] # Some drafts inline `typedef unsigned char u8;` etc. ("self-contained") -> when placed in a # .c that already #includes common.h, gcc-2.7.2 (C89) errors on the redefinition. Strip those @@ -54,13 +71,30 @@ def build(): return sha1(a.out) -# --- load drafts for functions still stubbed in the current source --- -src0 = open(a.src).read() -stubbed = set(re.findall(r'INCLUDE_ASM\("' + re.escape(a.asm_subdir) + r'", (func_[0-9A-Fa-f]+|DsMix)\)', src0)) -drafts = {} +# --- locate every live stub, in EVERY TU of the binary (Phase 26-A, HIGH) ------------------------ +# This used to scan the ONE file the caller passed as --src, and silently drop every draft whose +# stub lived elsewhere. An overlay's source is spread over up to 14 .c files, so 56,742 of the +# fleet's 58,717 open stubs (96.6%) were unreachable, and THREE of the six callers passed no --src +# at all — including grinder.py, which meant 1,290 of the grinder's own 1,298 queued functions +# COULD NEVER BANK, however good the permuter's output was. ("The permuter's fuel is exhausted" +# was therefore never a safe conclusion.) +# +# Every TU links into the same image, so one `make build BINARY=` still gates them all — this +# is both correct AND strictly fewer builds than the per-split re-gate it replaces. +# +# SAFETY (G3/P9 — this is the byte-gate, so the argument must be explicit): the verdict is unchanged +# (`make build` + SHA1 == the locked hash). Only the SPLICE LOCATION is derived rather than guessed. +# INCLUDE_ASM pastes the ORIGINAL assembly, so a wrong draft always changes the bytes and always +# fails SHA1. A bug here can therefore make the tool FAIL TO BANK; it cannot make it falsely bank. +_stubs = {s.symbol: s for s in corpus.stubs(a.binary).values()} +if a.src: # optional filter, not a location oracle + _stubs = {n: s for n, s in _stubs.items() if s.path == a.src} + +drafts, not_stubbed = {}, [] for cf in sorted(glob.glob(a.drafts + '/*.c')): fn = os.path.basename(cf)[:-2] - if fn not in stubbed: + if fn not in _stubs: + not_stubbed.append(fn) # REPORTED, never silently dropped (R32) continue conf = 'medium' cp = cf[:-2] + '.conf' @@ -72,32 +106,49 @@ for cf in sorted(glob.glob(a.drafts + '/*.c')): order = {'high': 0, 'medium': 1, 'low': 2} items = sorted(drafts, key=lambda fn: (order[drafts[fn]['conf']], len(drafts[fn]['c']))) -print('drafts to verify: %d (high=%d medium=%d low=%d) of %d stubbed' % ( +print('drafts to verify: %d (high=%d medium=%d low=%d) of %d live stubs across %d TUs' % ( len(items), sum(drafts[f]['conf'] == 'high' for f in items), sum(drafts[f]['conf'] == 'medium' for f in items), sum(drafts[f]['conf'] == 'low' for f in items), - len(stubbed))) + len(_stubs), len({s.path for s in _stubs.values()}))) +if not_stubbed: + print(' (%d draft(s) skipped — not a live stub in %s: %s%s)' % ( + len(not_stubbed), a.binary, ' '.join(not_stubbed[:6]), + ' …' if len(not_stubbed) > 6 else '')) -baseline = src0 # known byte-identical source state (accumulates verified) +# baseline is now PER-FILE: {path: text}. Only the TUs we actually touch are tracked. +_touched = sorted({_stubs[fn].path for fn in items}) +baseline = {p: open(p).read() for p in _touched} verified, failed = [], [] +def _stub_line(fn): + s = _stubs[fn] + return 'INCLUDE_ASM("%s", %s);' % (s.asm_dir, s.symbol) + + def render(fns): - s = baseline + """{path: text} with each draft spliced into the TU that actually holds its stub.""" + out = dict(baseline) for fn in fns: - line = STUB.format(fn=fn) - if line not in s: + p, line = _stubs[fn].path, _stub_line(fn) + if line not in out.get(p, ''): return None # stub missing -> skip (don't let a no-op build false-pass) - s = s.replace(line, drafts[fn]['c'], 1) - return s + out[p] = out[p].replace(line, drafts[fn]['c'], 1) + return out + + +def _write(state): + for p, text in state.items(): + open(p, 'w').write(text) def attempt(fns): s = render(fns) if s is None: return False - open(a.src, 'w').write(s) + _write(s) return build() == a.good_sha @@ -122,7 +173,7 @@ while i < len(items): failed.append(fn); print(' - %s (%s)' % (fn, drafts[fn]['conf'])) # restore the accumulated verified state and confirm the binary is byte-identical -open(a.src, 'w').write(baseline) +_write(baseline) final = build() print('\n=== RESULT ===') print('verified %d / failed %d ; final SHA %s (%s)' % (