mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-10-01 07:40:42 -04:00
fix(phase-26a): A3 — the byte-gate could only see ONE translation unit (96.6% of stubs unreachable)
harvest_verify is the sole arbiter (G3/P9) and has never accepted a wrong match. It also could not
REACH most of the work: it scanned the single file the caller passed as --src and silently dropped
every draft whose stub lived elsewhere. An overlay's source spans up to 14 .c files, so:
open overlay stubs it could not see : 56,742 of 58,717 (96.6%)
ov_SC01_077 reachable by the gate : 13 of 264 (4.9%) -> 264 of 264 (100%)
THREE of the six callers passed no --src at all (orchestrator.py, grinder.py, idiom_hunt.py) and so
inherited gate_stage's `src = src or f"src/{binary}/{binary}.c"` default. For grinder.py that means
1,290 of its own 1,298 QUEUED FUNCTIONS COULD NEVER BANK, however good the permuter's output was.
=> Phase-22's "the permuter's fuel is exhausted" was never a safe conclusion. Re-test (A12).
gate_stage knew the right answer and then handed the gate the wrong file: its negative control
ALREADY globs every split .c to build bin_stubs. The default is now removed; --src is passed only
when a caller deliberately restricts the gate to one TU.
WHAT CHANGED, PRECISELY: only the SPLICE LOCATION. Each draft is now spliced into whichever TU
actually holds its stub, derived from tools/corpus.py. Every TU links into the same image, so ONE
`make build BINARY=<bin>` still gates them all — correct AND strictly fewer builds than the
per-split re-gate it replaces.
SAFETY (this is the byte-gate, so the argument is explicit): the VERDICT is untouched — `make build`
+ SHA1 == the locked hash. INCLUDE_ASM pastes the ORIGINAL assembly, so a wrong draft always changes
the bytes and always fails SHA1. A bug in the splice can therefore make the tool FAIL TO BANK; it
CANNOT make it falsely bank. The failure mode is conservative by construction.
VERIFIED end-to-end (2 real builds, tree clean before and after):
* discovery: 264 live stubs across 12 TUs (was: only those in the single --src file)
* IDENTITY known-answer test: 3 drafts whose stubs live in THREE DIFFERENT split TUs
(_jr_801734BC, _after, _jr_8012ACE0) — all discovered, spliced into their own files, built,
SHA-matched, committed, restored. Final SHA d19c9580 BYTE-IDENTICAL. Under the old code all
three were silently dropped as "not stubbed".
* `git checkout -- src/` recovers, exactly as the docstring promises.
Also derived rather than defaulted: --good-sha now reads config/check.<bin>.sha (a caller that
passed --binary but forgot --good-sha used to gate an overlay against RESIDENT's SHA), and
match_one_closeness resolves the asm subdir PER FUNCTION — one subdir for a whole batch is the same
single-TU bug, and pointing match_one at the wrong one scores a draft against a DIFFERENT function's
asm, producing a phantom non-zero closeness that lands in the backlog and feeds reserved_walls().
No committed source or config changed, so no build artifact can have moved; the byte-gate was
exercised twice and returned BYTE-IDENTICAL both times.
This commit is contained in:
Drew T
+32
-8
@@ -27,6 +27,8 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import backlog
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, os.path.join(REPO, 'tools'))
|
||||
import corpus # the derived corpus oracle (Phase 26-A)
|
||||
PY = ".venv/bin/python"
|
||||
# ov_SC01_077 defaults (the canonical harvest binary)
|
||||
OV = "ov_SC01_077"
|
||||
@@ -69,9 +71,14 @@ def _gate1(binary, src, asm, out, good_sha, d, verified_out=None, failed_out=Non
|
||||
binaries in parallel without cross-reading each other's results."""
|
||||
vo = verified_out or ".run/harvest_verified.txt"
|
||||
fo = failed_out or ".run/harvest_failed.txt"
|
||||
sh([PY, "tools/harvest_verify.py", "--binary", binary, "--src", src, "--asm-subdir", asm,
|
||||
"--out", out, "--good-sha", good_sha, "--drafts", d, "--chunk", "1",
|
||||
"--verified-out", vo, "--failed-out", fo], timeout=7200)
|
||||
# --src is passed ONLY when a caller explicitly restricts the gate to one TU. Omitted, the gate
|
||||
# derives each draft's home TU from the tree and splices it there (Phase 26-A) — every TU links
|
||||
# into the same image, so one `make build` still gates them all.
|
||||
cmd = [PY, "tools/harvest_verify.py", "--binary", binary]
|
||||
if src:
|
||||
cmd += ["--src", src]
|
||||
sh(cmd + ["--out", out, "--good-sha", good_sha, "--drafts", d, "--chunk", "1",
|
||||
"--verified-out", vo, "--failed-out", fo], timeout=7200)
|
||||
vp = os.path.join(REPO, vo)
|
||||
return [w for w in (open(vp).read().split() if os.path.exists(vp) else []) if w.startswith("func_")]
|
||||
|
||||
@@ -92,8 +99,20 @@ def _dedup_group_count():
|
||||
len(re.findall(r"^[A-Za-z]", open(p).read(), re.M))
|
||||
|
||||
|
||||
def match_one_closeness(fn, cpath, asm):
|
||||
"""('match',0) | ('near', n) | ('fail', None) via match_one (relocation-masked)."""
|
||||
def match_one_closeness(fn, cpath, asm, binary=None):
|
||||
"""('match',0) | ('near', n) | ('fail', None) via match_one (relocation-masked).
|
||||
|
||||
The asm subdir is derived PER FUNCTION from the stub that names it (Phase 26-A). One subdir for
|
||||
a whole batch is the same single-TU bug: an overlay has TWELVE, and pointing match_one at the
|
||||
wrong one scores a draft against a DIFFERENT function's asm — a phantom non-zero closeness that
|
||||
then lands in the backlog as a matching failure and feeds reserved_walls()."""
|
||||
if binary:
|
||||
st = corpus.stubs(binary)
|
||||
hit = next((s for s in st.values() if s.symbol == fn), None)
|
||||
if hit is not None:
|
||||
asm = hit.asm_dir
|
||||
if not asm:
|
||||
return ("fail", None)
|
||||
try:
|
||||
r = sh([PY, "tools/match_one.py", fn, "--c", cpath, "--asm-subdir", asm], timeout=180)
|
||||
except subprocess.TimeoutExpired:
|
||||
@@ -136,8 +155,13 @@ def _run_gate_locked(drafts, binary, src, asm, out, good_sha, propagate, source_
|
||||
# 0/222 Bug-B). good_sha is normalized to the BARE hash: config/check.<bin>.sha is sha1sum format
|
||||
# "<sha> <name>", but harvest_verify compares it against a bare sha1() — passing the whole line
|
||||
# never matches, so banking is 0 for EVERY binary incl. ov_SC01_077 (the 0/12 Bug-A).
|
||||
src = src or f"src/{binary}/{binary}.c"
|
||||
asm = asm or f"asm/{binary}/nonmatchings/{binary}"
|
||||
# src/asm are NOT defaulted (Phase 26-A audit, HIGH). harvest_verify now DERIVES each draft's
|
||||
# home TU from the tree, so inventing a default here would silently PIN the gate to the main .c.
|
||||
# That default is exactly what capped orchestrator.py / grinder.py / idiom_hunt.py — the three
|
||||
# callers that pass no src — to 13 of ov_SC01_077's 264 stubs (4.9%), and made 1,290 of the
|
||||
# grinder's own 1,298 queued functions UNBANKABLE however good the permuter's output was.
|
||||
# Note the negative control below already globs every split .c: this function knew the right
|
||||
# answer and then handed the gate the wrong file. Pass src ONLY to deliberately restrict to one TU.
|
||||
out = out or f"build/{binary}/{binary}"
|
||||
good_sha = (good_sha or _check_sha(binary) or DEF_SHA).split()[0]
|
||||
draft_fns = sorted(os.path.basename(p)[:-2] for p in
|
||||
@@ -215,7 +239,7 @@ def _run_gate_locked(drafts, binary, src, asm, out, good_sha, propagate, source_
|
||||
for fn in [f for f in draft_fns if f not in verified]:
|
||||
cpath = os.path.join(REPO, d, fn + ".c")
|
||||
body = open(cpath).read() if os.path.exists(cpath) else ""
|
||||
kind, close = match_one_closeness(fn, cpath, asm) if body else ("fail", None)
|
||||
kind, close = match_one_closeness(fn, cpath, asm, binary) if body else ("fail", None)
|
||||
meta = _manifest_class(fn)
|
||||
cm = re.search(r"//\s*@class:\s*(.+)", body)
|
||||
sm = re.search(r"//\s*@stuck:\s*(.+)", body)
|
||||
|
||||
+72
-21
@@ -14,14 +14,20 @@ Resident defaults; pass flags for another binary.
|
||||
python3 tools/harvest_verify.py # resident
|
||||
python3 tools/harvest_verify.py --chunk 6
|
||||
"""
|
||||
import subprocess, glob, os, re, hashlib, argparse
|
||||
import subprocess, glob, os, re, sys, hashlib, argparse
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import corpus # the derived corpus oracle (Phase 26-A) — a draft's home TU is a FACT of the tree
|
||||
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument('--binary', default='resident')
|
||||
ap.add_argument('--src', default='src/resident/resident.c')
|
||||
ap.add_argument('--asm-subdir', default='asm/resident/nonmatchings/resident')
|
||||
ap.add_argument('--out', default='build/resident/resident')
|
||||
ap.add_argument('--good-sha', default='8e17e02ff8954d07c979449198f7e1645046b353')
|
||||
ap.add_argument('--src', default=None,
|
||||
help='DEPRECATED and ignored for stub LOCATION. A draft is spliced into whichever TU '
|
||||
'actually holds its stub, derived from the tree. Kept only as an optional FILTER: '
|
||||
'if given, only drafts whose stub lives in this file are considered.')
|
||||
ap.add_argument('--asm-subdir', default=None, help='DEPRECATED and ignored (derived per stub).')
|
||||
ap.add_argument('--out', default=None, help='built image path (derived from --binary if omitted)')
|
||||
ap.add_argument('--good-sha', default=None, help='locked SHA1 (read from config/check.<bin>.sha if omitted)')
|
||||
ap.add_argument('--drafts', default='.run/drafts')
|
||||
ap.add_argument('--chunk', type=int, default=8)
|
||||
ap.add_argument('--verified-out', default='.run/harvest_verified.txt',
|
||||
@@ -29,7 +35,18 @@ ap.add_argument('--verified-out', default='.run/harvest_verified.txt',
|
||||
ap.add_argument('--failed-out', default='.run/harvest_failed.txt')
|
||||
a = ap.parse_args()
|
||||
|
||||
STUB = 'INCLUDE_ASM("' + a.asm_subdir + '", {fn});' # {fn} filled per function
|
||||
# ---- derive the build target + the locked SHA from the binary, not from a caller's flag ----------
|
||||
# A caller that passed --binary but forgot --good-sha used to gate an overlay against RESIDENT's SHA
|
||||
# (conservative — everything fails — but baffling). The Makefile and config/check.<bin>.sha already
|
||||
# state these facts; do not keep a second copy.
|
||||
_OUT = {'main': 'build/us/SLUS_007.26', 'resident': 'build/resident/resident'}
|
||||
if not a.out:
|
||||
a.out = _OUT.get(a.binary, f'build/{a.binary}/{a.binary}')
|
||||
if not a.good_sha:
|
||||
p = f'config/check.{"us" if a.binary == "main" else a.binary}.sha'
|
||||
if not os.path.exists(p):
|
||||
sys.exit(f'harvest_verify: no --good-sha and no {p} — refusing to gate against an unknown SHA')
|
||||
a.good_sha = open(p).read().split()[0]
|
||||
|
||||
# Some drafts inline `typedef unsigned char u8;` etc. ("self-contained") -> when placed in a
|
||||
# .c that already #includes common.h, gcc-2.7.2 (C89) errors on the redefinition. Strip those
|
||||
@@ -54,13 +71,30 @@ def build():
|
||||
return sha1(a.out)
|
||||
|
||||
|
||||
# --- load drafts for functions still stubbed in the current source ---
|
||||
src0 = open(a.src).read()
|
||||
stubbed = set(re.findall(r'INCLUDE_ASM\("' + re.escape(a.asm_subdir) + r'", (func_[0-9A-Fa-f]+|DsMix)\)', src0))
|
||||
drafts = {}
|
||||
# --- locate every live stub, in EVERY TU of the binary (Phase 26-A, HIGH) ------------------------
|
||||
# This used to scan the ONE file the caller passed as --src, and silently drop every draft whose
|
||||
# stub lived elsewhere. An overlay's source is spread over up to 14 .c files, so 56,742 of the
|
||||
# fleet's 58,717 open stubs (96.6%) were unreachable, and THREE of the six callers passed no --src
|
||||
# at all — including grinder.py, which meant 1,290 of the grinder's own 1,298 queued functions
|
||||
# COULD NEVER BANK, however good the permuter's output was. ("The permuter's fuel is exhausted"
|
||||
# was therefore never a safe conclusion.)
|
||||
#
|
||||
# Every TU links into the same image, so one `make build BINARY=<bin>` still gates them all — this
|
||||
# is both correct AND strictly fewer builds than the per-split re-gate it replaces.
|
||||
#
|
||||
# SAFETY (G3/P9 — this is the byte-gate, so the argument must be explicit): the verdict is unchanged
|
||||
# (`make build` + SHA1 == the locked hash). Only the SPLICE LOCATION is derived rather than guessed.
|
||||
# INCLUDE_ASM pastes the ORIGINAL assembly, so a wrong draft always changes the bytes and always
|
||||
# fails SHA1. A bug here can therefore make the tool FAIL TO BANK; it cannot make it falsely bank.
|
||||
_stubs = {s.symbol: s for s in corpus.stubs(a.binary).values()}
|
||||
if a.src: # optional filter, not a location oracle
|
||||
_stubs = {n: s for n, s in _stubs.items() if s.path == a.src}
|
||||
|
||||
drafts, not_stubbed = {}, []
|
||||
for cf in sorted(glob.glob(a.drafts + '/*.c')):
|
||||
fn = os.path.basename(cf)[:-2]
|
||||
if fn not in stubbed:
|
||||
if fn not in _stubs:
|
||||
not_stubbed.append(fn) # REPORTED, never silently dropped (R32)
|
||||
continue
|
||||
conf = 'medium'
|
||||
cp = cf[:-2] + '.conf'
|
||||
@@ -72,32 +106,49 @@ for cf in sorted(glob.glob(a.drafts + '/*.c')):
|
||||
|
||||
order = {'high': 0, 'medium': 1, 'low': 2}
|
||||
items = sorted(drafts, key=lambda fn: (order[drafts[fn]['conf']], len(drafts[fn]['c'])))
|
||||
print('drafts to verify: %d (high=%d medium=%d low=%d) of %d stubbed' % (
|
||||
print('drafts to verify: %d (high=%d medium=%d low=%d) of %d live stubs across %d TUs' % (
|
||||
len(items),
|
||||
sum(drafts[f]['conf'] == 'high' for f in items),
|
||||
sum(drafts[f]['conf'] == 'medium' for f in items),
|
||||
sum(drafts[f]['conf'] == 'low' for f in items),
|
||||
len(stubbed)))
|
||||
len(_stubs), len({s.path for s in _stubs.values()})))
|
||||
if not_stubbed:
|
||||
print(' (%d draft(s) skipped — not a live stub in %s: %s%s)' % (
|
||||
len(not_stubbed), a.binary, ' '.join(not_stubbed[:6]),
|
||||
' …' if len(not_stubbed) > 6 else ''))
|
||||
|
||||
baseline = src0 # known byte-identical source state (accumulates verified)
|
||||
# baseline is now PER-FILE: {path: text}. Only the TUs we actually touch are tracked.
|
||||
_touched = sorted({_stubs[fn].path for fn in items})
|
||||
baseline = {p: open(p).read() for p in _touched}
|
||||
verified, failed = [], []
|
||||
|
||||
|
||||
def _stub_line(fn):
|
||||
s = _stubs[fn]
|
||||
return 'INCLUDE_ASM("%s", %s);' % (s.asm_dir, s.symbol)
|
||||
|
||||
|
||||
def render(fns):
|
||||
s = baseline
|
||||
"""{path: text} with each draft spliced into the TU that actually holds its stub."""
|
||||
out = dict(baseline)
|
||||
for fn in fns:
|
||||
line = STUB.format(fn=fn)
|
||||
if line not in s:
|
||||
p, line = _stubs[fn].path, _stub_line(fn)
|
||||
if line not in out.get(p, ''):
|
||||
return None # stub missing -> skip (don't let a no-op build false-pass)
|
||||
s = s.replace(line, drafts[fn]['c'], 1)
|
||||
return s
|
||||
out[p] = out[p].replace(line, drafts[fn]['c'], 1)
|
||||
return out
|
||||
|
||||
|
||||
def _write(state):
|
||||
for p, text in state.items():
|
||||
open(p, 'w').write(text)
|
||||
|
||||
|
||||
def attempt(fns):
|
||||
s = render(fns)
|
||||
if s is None:
|
||||
return False
|
||||
open(a.src, 'w').write(s)
|
||||
_write(s)
|
||||
return build() == a.good_sha
|
||||
|
||||
|
||||
@@ -122,7 +173,7 @@ while i < len(items):
|
||||
failed.append(fn); print(' - %s (%s)' % (fn, drafts[fn]['conf']))
|
||||
|
||||
# restore the accumulated verified state and confirm the binary is byte-identical
|
||||
open(a.src, 'w').write(baseline)
|
||||
_write(baseline)
|
||||
final = build()
|
||||
print('\n=== RESULT ===')
|
||||
print('verified %d / failed %d ; final SHA %s (%s)' % (
|
||||
|
||||
Reference in New Issue
Block a user