mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-10-01 23:52:03 -04:00
Drew T
c513e1fbbd
- BANK: the stored S71 closeness-0 draft spliced into src/resident/resident_jr_800D128C.c; jtbl_carve --func carved jtbl_80113FB8 (119 entries, 1 pad word trimmed) + jtbl_80114198 into [0x451c0, .rodata, resident_jr_800D128C] + [0x453c4, data, tail3]; JTBL_PADS 0,4; make extract + make build BINARY=resident -j8 rc 0, sha 8e17e02ff8954d07c979449198f7e1645046b353 == check (R53). pads_audit ok/ok; interleave_check ALIGNED n=5; verbatim_check --strict 5==5. Resident stubs 2 -> 1 (func_800D06E8 remains). - WHY THE GATE SAID DIFF (parallel_gate banked 0/DIFF on an rtu_match MATCH): jtbl_carve.set_overlays_var regenerated resident_JTBL_INTERLEAVE from the carve set and DROPPED the resident's `--pre hdr.rodata.o` (§8f leading-rodata sandwich); make extract refused (ld_interleave: hdr.rodata.o would be parked with .text), the build linked the STALE script (249,252 differing bytes from file offset 0x4), and harvest_verify._jtbl_prep_one never read the post-carve extract's exit code (R49/R61). - FIXES (R35/R40/R57): jtbl_carve._merge_pre carries an existing --pre forward (idempotent; overlays unchanged, 4-shape unit control); harvest_verify refuses loudly on a failed post-carve extract and restores the snapshot (CARVE refusal, NOT a draft verdict); interleave_check's anchor accepts a leading --pre (was a false DRIFT n=0 on the resident; control ov_SC02_017 ALIGNED n=44 unchanged). - cookbook §498 (+ the stale-asm-after-a-failed-extract sequencing law); SETUP rows for all three
835 lines
50 KiB
Python
835 lines
50 KiB
Python
#!/usr/bin/env python3
|
|
"""Byte-gate drafted matching-C against a byte-locked binary, one draft at a time.
|
|
|
|
Reads .run/drafts/<fn>.c (self-contained C: externs + the function definition) produced
|
|
by the resident-harvest-draft workflow, and for each function still stubbed as INCLUDE_ASM:
|
|
substitutes the draft for that stub in the binary's source .c, rebuilds, and KEEPS it only
|
|
if the binary stays byte-identical (`make build` SHA1 == the locked SHA). Non-matches and
|
|
compile/link failures revert to the INCLUDE_ASM stub. The build is the sole truth (G3/P9):
|
|
a wrong draft can never be accepted. Chunk-with-bisection keeps it fast on high-hit batches.
|
|
|
|
The source .c is git-committed, so `git checkout` always recovers if interrupted.
|
|
Resident defaults; pass flags for another binary.
|
|
|
|
⚠️ RUN THIS AS A SCRIPT — NEVER `import` IT. The whole gate lives at module level (deliberately
|
|
NOT refactored into main(): re-indenting the project's most load-bearing gate is riskier than the
|
|
hazard). A bare `import harvest_verify` used to PARSE argv and RUN A FULL BUILD (found the hard way
|
|
in P30 S43 — the import alone ran a 13-draft resident gate); since P31 T0 an import-guard below
|
|
RAISES immediately instead, so the failure is loud and side-effect-free. To test a helper here,
|
|
exec that function's source (ast.get_source_segment) instead of importing the module.
|
|
|
|
python3 tools/harvest_verify.py # resident
|
|
python3 tools/harvest_verify.py --chunk 6
|
|
"""
|
|
import subprocess, glob, os, re, sys, hashlib, argparse, shutil
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
PY = '.venv/bin/python'
|
|
|
|
import corpus # the derived corpus oracle (Phase 26-A) — a draft's home TU is a FACT of the tree
|
|
import cdecl # the C-declaration oracle (Phase 26-A) — the per-TU typedef strip-set (T4)
|
|
import draft_prechecks as DP
|
|
|
|
# P31 T0 import guard: the gate runs at module level by design (see docstring); importing must
|
|
# fail LOUD before argv is parsed or any build runs, never execute a gate as a side effect.
|
|
if __name__ != "__main__":
|
|
raise RuntimeError(
|
|
"harvest_verify.py is a CLI gate, not a library — run it as a subprocess "
|
|
"(its whole pipeline executes at module level; see the docstring)")
|
|
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument('--binary', default='resident')
|
|
ap.add_argument('--src', default=None,
|
|
help='DEPRECATED and ignored for stub LOCATION. A draft is spliced into whichever TU '
|
|
'actually holds its stub, derived from the tree. Kept only as an optional FILTER: '
|
|
'if given, only drafts whose stub lives in this file are considered.')
|
|
ap.add_argument('--asm-subdir', default=None, help='DEPRECATED and ignored (derived per stub).')
|
|
ap.add_argument('--out', default=None, help='built image path (derived from --binary if omitted)')
|
|
ap.add_argument('--good-sha', default=None, help='locked SHA1 (read from config/check.<bin>.sha if omitted)')
|
|
ap.add_argument('--drafts', default='.run/drafts')
|
|
ap.add_argument('--chunk', type=int, default=8)
|
|
ap.add_argument('--verified-out', default='.run/harvest_verified.txt',
|
|
help='per-worker result path (bulk_harvest gates distinct binaries in parallel)')
|
|
ap.add_argument('--failed-out', default='.run/harvest_failed.txt')
|
|
a = ap.parse_args()
|
|
|
|
# ---- derive the build target + the locked SHA from the binary, not from a caller's flag ----------
|
|
# A caller that passed --binary but forgot --good-sha used to gate an overlay against RESIDENT's SHA
|
|
# (conservative — everything fails — but baffling). The Makefile and config/check.<bin>.sha already
|
|
# state these facts; do not keep a second copy.
|
|
_OUT = {'main': 'build/us/SLUS_007.26', 'resident': 'build/resident/resident'}
|
|
if not a.out:
|
|
a.out = _OUT.get(a.binary, f'build/{a.binary}/{a.binary}')
|
|
if not a.good_sha:
|
|
p = f'config/check.{"us" if a.binary == "main" else a.binary}.sha'
|
|
if not os.path.exists(p):
|
|
sys.exit(f'harvest_verify: no --good-sha and no {p} — refusing to gate against an unknown SHA')
|
|
a.good_sha = open(p).read().split()[0]
|
|
|
|
# A self-contained draft inlines its own typedefs (`typedef unsigned char u8;`, and often a struct
|
|
# type like `typedef struct {...} Blk16;`). Spliced into a TU that already provides those names
|
|
# (common.h + engine_types.h), gcc-2.7.2/C89 REJECTS the redefinition — a plumbing error, not a byte
|
|
# mismatch. Strip exactly the names the TARGET TU provides, per-TU (cdecl.typedef_names + the T4
|
|
# primitive). The old `_TD` here was scalar-only (no M2C_UNK, no struct typedefs) and dropped 39
|
|
# still-open functions this way; the strip-set is now derived from the tree, not hand-listed.
|
|
_JOBS = int(os.environ.get('BFM_BUILD_JOBS') or (os.cpu_count() or 8))
|
|
|
|
|
|
def sha1(path):
|
|
return hashlib.sha1(open(path, 'rb').read()).hexdigest() if os.path.exists(path) else None
|
|
|
|
|
|
# Distinguish a DECLARATION/PLUMBING failure (byte-correct C the front end rejects — recoverable by
|
|
# strip/reconcile) from a genuine codegen miss. The build's stderr was being discarded, so a draft
|
|
# that failed to COMPILE was recorded identically to one that compiled to the wrong bytes — the exact
|
|
# "a compiler wall that is really a plumbing error" the 26-A audit exists to end (R32). We surface it.
|
|
_PLUMBING = re.compile(
|
|
r'conflicting types|redefinition of|redeclar|parse error before|storage size|'
|
|
r'undefined reference|prototype declaration', re.I)
|
|
# make's own failure summary — a wrapper around the diagnostic, never the diagnostic (see classify_fail)
|
|
_MAKE_WRAP = re.compile(r'^\s*make(\[\d+\])?:\s*\*\*\*')
|
|
# A DIAGNOSTIC IS A POSITION, NOT A VOCABULARY: `<file>:<line>: <text>` from cc1, or
|
|
# `{standard input}:<line>: <text>` from the assembler. gcc-2.7.2 prefixes hard errors with
|
|
# nothing at all, so matching on the word `error` misses them entirely (see classify() below).
|
|
_SRC_DIAG = re.compile(r'^(?:\{standard input\}|[^\s:]+):\d+:\s')
|
|
_last_err = ''
|
|
_last_sha = None
|
|
_NO_BUILD = object() # attempt() short-circuited (stub already spliced) — no build ran
|
|
|
|
|
|
def build():
|
|
"""make build BINARY=<bin>; return the output SHA1, or None on compile/link failure
|
|
(remove the output first so a stale file can't masquerade as a passing build). Stashes the
|
|
build's stderr + sha in _last_err/_last_sha so a single-draft failure can be CLASSIFIED."""
|
|
global _last_err, _last_sha
|
|
if os.path.exists(a.out):
|
|
os.remove(a.out)
|
|
# -j: THE INNER LOOP OF EVERY GATE. A binary is ~35 objects and the build was SERIAL — measured
|
|
# 7.18 s serial vs 1.18 s at -j16 on ov_SC03_010, byte-identical and equal to the locked SHA
|
|
# (6.1x). `JOBS` in the Makefile is parallelism ACROSS binaries (xargs -P); this is parallelism
|
|
# WITHIN one binary and was simply never passed. docs/SETUP.md:416 already documents
|
|
# `make -j$(nproc) build` as the sanctioned form.
|
|
# SAFE BY CONSTRUCTION: this build feeds a locked-SHA comparison, so a bad parallel build FAILS
|
|
# the gate rather than banking wrong bytes — the error direction is a false NEGATIVE, never a
|
|
# false bank (G3/P9 remains the sole arbiter).
|
|
p = subprocess.run(['make', '-j%d' % _JOBS, 'build', 'BINARY=' + a.binary],
|
|
capture_output=True, text=True)
|
|
_last_err = (p.stderr or '') + (p.stdout or '')
|
|
_last_sha = sha1(a.out)
|
|
return _last_sha
|
|
|
|
|
|
def _squeeze(ln, n=170, head=58):
|
|
"""Shorten a compiler/linker diagnostic WITHOUT losing its payload.
|
|
|
|
`<path>:<pos>: <message>` — the message names the symbol, so keep the tail; keep a short head
|
|
so the file is still identifiable. Left-truncation threw the symbol away (see the note in
|
|
classify_fail)."""
|
|
ln = ln.strip()
|
|
if len(ln) <= n:
|
|
return ln
|
|
return ln[:head].rstrip() + ' … ' + ln[-(n - head - 3):].lstrip()
|
|
|
|
|
|
def classify_fail(got_sha):
|
|
"""Why did this single draft fail? Only meaningful right after a 1-draft attempt().
|
|
* no build ran (stub already spliced) -> SKIP
|
|
* built but wrong bytes (sha present, != good) -> DIFF (a genuine codegen residual)
|
|
* did not build, stderr matches a decl conflict -> PLUMBING:...(recoverable, not a wall)
|
|
* did not build, other -> CC1-FAIL (needs a look)"""
|
|
if got_sha is _NO_BUILD:
|
|
return 'SKIP'
|
|
if got_sha is not None:
|
|
return 'DIFF'
|
|
# TRUNCATE FROM THE MIDDLE, NEVER FROM THE LEFT (P30 S38). These diagnostics are
|
|
# `<long path>:<position>: <message>`, and the MESSAGE names the symbol — which is the whole
|
|
# routing value of the label. A left-truncating `[:90]` spent its budget on the path and cut the
|
|
# payload mid-token: a real failure was labelled
|
|
# PLUMBING: src/ov_SC03_001/ov_SC03_001_jr_8017AE2C.c:(.text+0xc090): undefined reference to `func_801
|
|
# — the symbol name, the one thing that picks the lever, severed at four hex digits. Keeping
|
|
# both ends costs nothing and makes the class actionable. (Same family as §58 below and §136a:
|
|
# a label that cannot distinguish its inputs carries no information.)
|
|
# §58 RED-HERRING GUARD (2026-07-22). The build log is full of BENIGN WARNINGS — chiefly
|
|
# `warning: conflicting types for built-in function 'memcpy'`, which fires from an unrelated
|
|
# TU position on essentially every overlay build. The old code searched the whole stderr and
|
|
# returned the FIRST match, so that warning won: an 11-draft re-probe returned the SAME
|
|
# 'PLUMBING: …conflicting types for built-in fu' label for 8 of 8 failures, across four
|
|
# genuinely different causes. A label that is identical for every input carries no
|
|
# information, and it is worse than none — the cookbook had to record "the gate label is
|
|
# useless here, splice individually and read real cc1 stderr" as a manual step. Classify on
|
|
# NON-warning lines only, and when nothing but warnings matched, surface the real error
|
|
# instead of guessing (R32: report the gap, do not paper over it).
|
|
# A `note:` line is NEVER a cause — gcc emits it only as the SECOND half of a diagnostic pair,
|
|
# and when the first half is a benign warning the note is orphaned by the filter above and then
|
|
# wins the label. Measured P31 S65: a byte-perfect 246-ins draft was labelled
|
|
# CC1-FAIL: …/engine_core.h:57282: note: this is the location of the previous definition
|
|
# whose antecedent was the fleet-wide benign `"DEFINE_func_80181538" redefined` WARNING — the
|
|
# draft had in fact built and banked whole-binary-identical. Same §58 red-herring family: drop
|
|
# the notes with their warnings, and let the real error (or the honest gap) carry the label.
|
|
lines = [ln for ln in _last_err.splitlines()
|
|
if 'warning:' not in ln and not re.search(r':\s*note:', ln)]
|
|
for ln in lines:
|
|
if _PLUMBING.search(ln):
|
|
return 'PLUMBING: ' + _squeeze(ln)
|
|
errs = [ln for ln in lines if re.search(r'\berror\b|\bError \d', ln)]
|
|
# `make: *** [Makefile:N: build/src/<ov>/<tu>.o] Error N` is make's SUMMARY of a failure, never a
|
|
# diagnosis of it — and because make prints it LAST, `errs[-1]` picked it every time. Measured
|
|
# P30 S43 over the committed .classified.txt corpus: ~3,000 of ~4,000 CC1-FAIL labels were that
|
|
# wrapper, carrying nothing but the TU name that the record already stores. Each one therefore
|
|
# cost a manual splice-and-rebuild to learn what cc1 actually said (done 3x in S42 alone). Same
|
|
# defect as the §58 warning red-herring directly above — a label identical for every input is
|
|
# worse than none — so the guard has to exclude make's own lines too, not just warnings.
|
|
# Prefer the FIRST real diagnostic: cc1 cascades, so error #1 is the root cause and error #N is
|
|
# usually its aftershock. Fall back to the wrapper only when nothing else exists (R32: surface
|
|
# the gap, never paper over it).
|
|
real = [ln for ln in errs if not _MAKE_WRAP.match(ln)]
|
|
if real:
|
|
return 'CC1-FAIL: ' + _squeeze(real[0])
|
|
# GCC-2.7.2 EMITS NO `error:` PREFIX ON HARD ERRORS — the filter above requires the *word*
|
|
# `error`, and this compiler's diagnostics simply do not carry it (P30 S47, byte-witnessed):
|
|
# src/…/ov_SC02_037_jr_8013B83C.c:447: multiple storage classes in declaration of `tail_…'
|
|
# src/…/ov_SC02_037_jr_80159A20.c:1147: storage class specified for parameter `D_8011FD10'
|
|
# src/…/ov_SC06_025_jr_8012ACE0.c:2217: `tbl_D_80187044' undeclared (first use this function)
|
|
# None of those survive `\berror\b`, so `errs` held nothing but make's `Error 33` wrapper and
|
|
# every hard error was labelled `no-diagnostic` — "the compiler failed and we cannot see why".
|
|
# Consequence measured this session: 132 siblings of func_80132018 were classified that way by a
|
|
# single missing declaration, which reads as a codegen wall and gets a family deprioritised.
|
|
# Three separate cheap faults hid behind this label in one day, and rtu_match had the same
|
|
# blindness repaired at T0(b) — the fix was never propagated here.
|
|
# Classify on POSITION, not vocabulary: `<file>:<line>: <text>` (and the assembler's
|
|
# `{standard input}:<line>:`) IS a diagnostic. Context lines (`…: In function 'f':`) carry no
|
|
# `:<line>:` and are correctly skipped.
|
|
diag = [ln for ln in lines
|
|
if _SRC_DIAG.match(ln.strip()) and 'warning' not in ln.lower()]
|
|
if diag:
|
|
return 'CC1-FAIL: ' + _squeeze(diag[0])
|
|
if errs:
|
|
return 'CC1-FAIL(no-diagnostic): ' + _squeeze(errs[-1])
|
|
return 'CC1-FAIL'
|
|
|
|
|
|
# --- locate every live stub, in EVERY TU of the binary (Phase 26-A, HIGH) ------------------------
|
|
# This used to scan the ONE file the caller passed as --src, and silently drop every draft whose
|
|
# stub lived elsewhere. An overlay's source is spread over up to 14 .c files, so 56,742 of the
|
|
# fleet's 58,717 open stubs (96.6%) were unreachable, and THREE of the six callers passed no --src
|
|
# at all — including grinder.py, which meant 1,290 of the grinder's own 1,298 queued functions
|
|
# COULD NEVER BANK, however good the permuter's output was. ("The permuter's fuel is exhausted"
|
|
# was therefore never a safe conclusion.)
|
|
#
|
|
# Every TU links into the same image, so one `make build BINARY=<bin>` still gates them all — this
|
|
# is both correct AND strictly fewer builds than the per-split re-gate it replaces.
|
|
#
|
|
# SAFETY (G3/P9 — this is the byte-gate, so the argument must be explicit): the verdict is unchanged
|
|
# (`make build` + SHA1 == the locked hash). Only the SPLICE LOCATION is derived rather than guessed.
|
|
# INCLUDE_ASM pastes the ORIGINAL assembly, so a wrong draft always changes the bytes and always
|
|
# fails SHA1. A bug here can therefore make the tool FAIL TO BANK; it cannot make it falsely bank.
|
|
_stubs = {s.symbol: s for s in corpus.stubs(a.binary).values()}
|
|
if a.src: # optional filter, not a location oracle
|
|
_stubs = {n: s for n, s in _stubs.items() if s.path == a.src}
|
|
|
|
drafts, not_stubbed = {}, []
|
|
for cf in sorted(glob.glob(a.drafts + '/*.c')):
|
|
fn = os.path.basename(cf)[:-2]
|
|
if fn not in _stubs:
|
|
not_stubbed.append(fn) # REPORTED, never silently dropped (R32)
|
|
continue
|
|
conf = 'medium'
|
|
cp = cf[:-2] + '.conf'
|
|
if os.path.exists(cp):
|
|
w = open(cp).read().strip().lower().split()
|
|
if w and w[0] in ('high', 'medium', 'low'):
|
|
conf = w[0]
|
|
# Strip the typedefs the DRAFT's OWN target TU already provides (per-TU strip-set, T4). Computed
|
|
# against the baseline tree (before any splice); cdecl.typedef_names is cached per TU.
|
|
# ABOVE= IS LOAD-BEARING (P31 S74). The strip-set must be what is in scope AT THE SPLICE
|
|
# POINT, not everything the TU declares anywhere: md_SC07_004 typedefs `M4_801AE734` at :10174
|
|
# while func_801ADA10's stub is at :9524, so the un-scoped set stripped the draft's own copy of
|
|
# a type the spliced body still needs -> `parse error before 'D_801B0838'`, logged as PLUMBING
|
|
# and indistinguishable from a real decl conflict. cdecl.typedef_names already takes `above`
|
|
# for exactly this; the gate simply never passed it.
|
|
provided = cdecl.typedef_names(_stubs[fn].path, above=fn)
|
|
_txt = cdecl.strip_provided_typedefs(open(cf).read(), provided)
|
|
# A DRAFT THAT IS THE TARGET'S OWN ASSEMBLY IS A NO-OP THAT PASSES FOR FREE (P31 S76).
|
|
# A §265 verbatim body assembles to the bytes it was copied from, so match_one prints MATCH and
|
|
# this gate goes BYTE-IDENTICAL -- both truthfully -- while nothing is decompiled. gate_main
|
|
# gained this refusal earlier the same session after 9 functions round-tripped
|
|
# verbatim -> stub -> verbatim and "banked" with progress.py moving by exactly zero; the same
|
|
# wave then produced verbatim submissions for md_MAIN_003 and ov_SC06_010, which reach the
|
|
# tree through THIS gate, not that one. One defect, two doors (R43; the sibling-provisioner
|
|
# lesson of §442/S74 -- a fix made in one of two paths is a fix in neither).
|
|
if DP.is_verbatim_asm_draft(_txt, fn):
|
|
print(' SKIP %s: draft is the target\'s own asm in a file-scope __asm__ (a §265 verbatim '
|
|
'body), not a decompile — it would pass this gate for free and bank nothing' % fn)
|
|
continue
|
|
drafts[fn] = {'c': _txt, 'conf': conf}
|
|
|
|
order = {'high': 0, 'medium': 1, 'low': 2}
|
|
items = sorted(drafts, key=lambda fn: (order[drafts[fn]['conf']], len(drafts[fn]['c'])))
|
|
print('drafts to verify: %d (high=%d medium=%d low=%d) of %d live stubs across %d TUs' % (
|
|
len(items),
|
|
sum(drafts[f]['conf'] == 'high' for f in items),
|
|
sum(drafts[f]['conf'] == 'medium' for f in items),
|
|
sum(drafts[f]['conf'] == 'low' for f in items),
|
|
len(_stubs), len({s.path for s in _stubs.values()})))
|
|
if not_stubbed:
|
|
print(' (%d draft(s) skipped — not a live stub in %s: %s%s)' % (
|
|
len(not_stubbed), a.binary, ' '.join(not_stubbed[:6]),
|
|
' …' if len(not_stubbed) > 6 else ''))
|
|
|
|
# baseline is now PER-FILE: {path: text}. Only the TUs we actually touch are tracked.
|
|
_touched = sorted({_stubs[fn].path for fn in items})
|
|
baseline = {p: open(p).read() for p in _touched}
|
|
verified, failed = [], []
|
|
|
|
|
|
# --------------------------------------------------------------------------------------------
|
|
# jtbl prep (Phase-29 Task-14 stage 4) — the carve MUST FOLLOW THE SPLICE
|
|
# --------------------------------------------------------------------------------------------
|
|
# A draft whose function owns a switch jump table cannot link into a jtbl-CARVED TU until that
|
|
# table has its own contiguous .rodata carve. gate_stage tried this as a batch pre-pass and it
|
|
# does not work, for a reason that is the whole point of doing it here instead:
|
|
#
|
|
# THE NON-CONTIGUITY IS ONLY DETECTABLE ONCE THE BODY IS IN THE OBJECT. While the function is
|
|
# still INCLUDE_ASM, `jtbl_carve` reports SUCCESS and yields a spec that fails when the body
|
|
# lands. Byte-witnessed both ways on func_80135A4C: spliced -> "NON-CONTIGUOUS 0xaa810 and
|
|
# 0xaa920"; unspliced -> "prepared 1/1", then a byte-DIFF at the gate.
|
|
#
|
|
# So we splice TEMPORARILY, ask jtbl_carve, isolate if it refuses, then un-splice and let the
|
|
# normal gate loop do the real splice. Isolation MOVES the stub's TU, so the stub map and the
|
|
# baseline are re-derived afterwards (they are keyed by path).
|
|
_JTBL_RE = re.compile(r"jtbl_[0-9A-Fa-f]{8}")
|
|
_ISO_WALLS = ("NON-CONTIGUOUS", "do not fit the span")
|
|
# §154-A leading-island refusals (both spellings jtbl_carve emits: apply()'s migrated-table
|
|
# message and build_carve's below-the-data-region message). Handled by the §260 island split
|
|
# branch in _jtbl_prep_one, byte-proven on md_SC03_076/func_801F218C (sha 9a165e36…).
|
|
_ISLAND_WALLS = ("LEADING-ISLAND", "leading .rodata island")
|
|
|
|
|
|
def _sh(cmd, timeout=1800):
|
|
return subprocess.run(cmd, capture_output=True, text=True, cwd=REPO, timeout=timeout)
|
|
|
|
|
|
def _fn_has_jtbl(fn):
|
|
st = _stubs.get(fn)
|
|
if st is None:
|
|
return False
|
|
try:
|
|
return bool(_JTBL_RE.search(open(os.path.join(REPO, st.asm_path)).read()))
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def _reload_corpus():
|
|
"""Re-derive the stub map + baseline after an isolation moved a stub to a new TU.
|
|
|
|
THE `--src` FILTER MUST NOT APPLY TO THE WORKING SET HERE (P30 S38, byte-witnessed).
|
|
Following a carved stub to its NEW TU is this function's entire stated purpose — and the old
|
|
code then re-applied `--src` (`x.path == a.src`) to the reloaded map, which DELETED exactly the
|
|
function just carved. The consequences chained all the way out to a false wall:
|
|
_stubs loses fn -> render() raises KeyError(fn) -> the exception is UNCAUGHT, so
|
|
_jtbl_restore(snap) never runs -> the carve is STRANDED in config/ + src/ -> and the
|
|
caller (s6f_gate.py) greps stdout for VERIFIED:/FAILED: lines, finds neither, and books a
|
|
SILENT SKIP.
|
|
Net effect: 10 of wave 6's 16 drafts — 9 of them claiming MATCH — reported as nothing at all,
|
|
while every later group in the same gate ran against a tree the earlier crashes had mutated.
|
|
Line 136 already labels `--src` "an optional filter, not a location oracle"; this is the place
|
|
that was treating it as one.
|
|
|
|
So: keep the filter for the ambient stub map (it is what makes `len(_stubs)` mean "stubs in the
|
|
TU under test"), but NEVER let it drop a draft we are actively verifying, wherever it now lives.
|
|
That also repairs `_touched`/`baseline`, which are derived from this map — a carved fn missing
|
|
from `_stubs` left its new TU unbaselined, so the revert path could not restore it either."""
|
|
global _stubs, baseline, _touched
|
|
for f in (corpus.stubs, corpus.sig, corpus.symbols, corpus.src_files):
|
|
if hasattr(f, 'cache_clear'):
|
|
f.cache_clear()
|
|
_stubs = {x.symbol: x for x in corpus.stubs(a.binary).values()}
|
|
if a.src:
|
|
working = set(drafts)
|
|
_stubs = {n: x for n, x in _stubs.items() if x.path == a.src or n in working}
|
|
missing = [fn for fn in items if fn not in _stubs]
|
|
if missing: # R32: a vanished working stub is a DEFECT, loudly
|
|
print(' [reload] !! %d working stub(s) not in the corpus after reload: %s'
|
|
% (len(missing), ' '.join(missing[:6])))
|
|
_touched = sorted({_stubs[fn].path for fn in items if fn in _stubs})
|
|
baseline = {q: open(q).read() for q in _touched}
|
|
|
|
|
|
_MK_PATH = os.path.join(REPO, 'config/overlays.mk')
|
|
_MK_LOCK = os.path.join(REPO, '.run', 'auto', 'overlays_mk.lock')
|
|
|
|
|
|
def _mk_block_spans(txt, binary):
|
|
"""EVERY (start, end) span whose `# --- <binary> …` header names this binary, in file order.
|
|
|
|
ALL of them, not the first (P31 S75). This was `_mk_block_span`, singular: it took the FIRST
|
|
matching header and ran to the next `# --- `, so a binary whose carve state spans more than one
|
|
block was HALF-snapshotted and, on revert, silently HALF-restored.
|
|
|
|
Measured: `resident` is the only such binary today (1 of 142 with a block) and it has exactly
|
|
two — `§8e jtbl pad spec` and `§8f leading-rodata sandwich`. A rejected gate of
|
|
`resident:func_800D06E8` left the file with a 4th `JTBL_PADS` entry AND without
|
|
`--pre hdr.rodata.o`; the revert put back block 1 and never saw block 2, so the binary would not
|
|
build at all (`consumed 3 rodata jump table(s) but 4 pad spec(s) given`) — with `src/` perfectly
|
|
clean, so `git status src/` said nothing was wrong. Shared COMMITTED carve state, damaged by a
|
|
gate that REJECTED the draft (R51/R52).
|
|
|
|
The singular form was also a silent narrowing in the classic shape: it returned a TRUE span for a
|
|
scope smaller than the caller believed, and nothing compared the two (R32)."""
|
|
out = []
|
|
for m in re.finditer(r'^# --- %s\b.*$' % re.escape(binary), txt, re.M):
|
|
nxt = re.search(r'^# --- ', txt[m.end():], re.M)
|
|
out.append((m.start(), (m.end() + nxt.start()) if nxt else len(txt)))
|
|
return out
|
|
|
|
|
|
def _mk_block(binary):
|
|
"""This binary's carve state as a LIST of block texts, or None when it has no block.
|
|
|
|
None (not []) for the no-block case, so the caller's `is not None` guard keeps its old meaning:
|
|
71 of the 213 binaries have no block at all and must not be snapshotted or warned about."""
|
|
try:
|
|
txt = open(_MK_PATH).read()
|
|
except OSError:
|
|
return None
|
|
blocks = [txt[a:b] for a, b in _mk_block_spans(txt, binary)]
|
|
return blocks or None
|
|
|
|
|
|
def _mk_block_restore(binary, blocks):
|
|
"""Restore this binary's block(s) in the CURRENT overlays.mk under the shared lock (other
|
|
binaries' blocks are left exactly as they are now).
|
|
|
|
Accepts the list `_mk_block` returns; a bare string is still honoured so an older snapshot
|
|
(or a caller that kept one) restores as a single block rather than being mis-iterated."""
|
|
import fcntl
|
|
if isinstance(blocks, str):
|
|
blocks = [blocks]
|
|
os.makedirs(os.path.dirname(_MK_LOCK), exist_ok=True)
|
|
with open(_MK_LOCK, 'w') as lk:
|
|
fcntl.flock(lk, fcntl.LOCK_EX)
|
|
txt = open(_MK_PATH).read()
|
|
spans = _mk_block_spans(txt, binary)
|
|
if not spans:
|
|
print(' [jtbl] !! overlays.mk block for %s not found — block restore skipped' % binary)
|
|
return
|
|
if len(spans) == len(blocks):
|
|
new = txt
|
|
for (a, b), blk in zip(reversed(spans), reversed(blocks)): # tail-first: spans stay valid
|
|
new = new[:a] + blk + new[b:]
|
|
else:
|
|
# The carve added or removed a header. Collapse to the snapshot: put every snapshot block
|
|
# where the first one was and drop the rest, so no half-state survives.
|
|
print(' [jtbl] overlays.mk %s: %d block(s) now vs %d in the snapshot — collapsing to the '
|
|
'snapshot' % (binary, len(spans), len(blocks)))
|
|
new = txt
|
|
for a, b in reversed(spans[1:]):
|
|
new = new[:a] + new[b:]
|
|
a, b = spans[0]
|
|
new = new[:a] + ''.join(blocks) + new[b:]
|
|
if new != txt:
|
|
open(_MK_PATH, 'w').write(new)
|
|
# R32 — ASSERT THE RESTORE LANDED. The defect this replaces was a reported success.
|
|
back = open(_MK_PATH).read()
|
|
if [back[a:b] for a, b in _mk_block_spans(back, binary)] != list(blocks):
|
|
print(' [jtbl] !! overlays.mk restore for %s did NOT reproduce the snapshot — the carve '
|
|
'state is now UNKNOWN; `git diff config/overlays.mk` before building' % binary)
|
|
|
|
|
|
def _jtbl_snapshot():
|
|
"""Text of EVERY file a carve/isolation may rewrite — config AND the binary's sources.
|
|
|
|
SOURCES ARE NOT OPTIONAL (byte-witnessed 2026-07-22). `jr_isolate_all` repartitions a code
|
|
object by writing region 0 back over the ORIGINAL `src/<ov>/<nm>.c` (truncated to just that
|
|
region) and emitting the rest as new `_jr_<lo>.c` files. An undo that restores only config/
|
|
and deletes the new region files therefore leaves the original TU PERMANENTLY TRUNCATED —
|
|
its stubs are gone, and nothing regenerates them (splat does not rewrite a committed
|
|
overlay .c). Measured on an 11-draft re-probe: live stubs fell 419 -> 414 -> 406 -> 395 as
|
|
successive rejected drafts each ate a TU, ending in `undefined reference to func_80191C50`.
|
|
|
|
Worse, it is INVISIBLE to the gate that caused it: the incremental build keeps linking the
|
|
stale objects (§42b), so `make build` stays green while a CLEAN rebuild fails. That is the
|
|
mechanism behind the "139/140, twice" reading that became the §61c blocker — the tree was
|
|
being eaten by the very undo meant to protect it.
|
|
|
|
So snapshot the whole source set, and undo by RESTORE, never by an inverse transform (§61)."""
|
|
out = {}
|
|
for q in (os.path.join(REPO, 'config/splat.%s.yaml' % a.binary),):
|
|
try:
|
|
out[q] = open(q).read()
|
|
except OSError:
|
|
pass
|
|
# overlays.mk is SHARED by every binary's gate running in parallel (sweep_parallel -j N). A
|
|
# whole-file snapshot restored later resurrects other binaries' lines as they stood at snapshot
|
|
# time (P31 S62: ov_MAIN_012's rejected carve line came back after that binary had undone it,
|
|
# then gate_and_commit blanket-committed the drift). Snapshot ONLY this binary's block and
|
|
# restore it block-wise under the mk lock against the CURRENT file (`_mk_block`).
|
|
blk = _mk_block(a.binary)
|
|
if blk is not None:
|
|
out['__mk_block__'] = blk
|
|
for q in glob.glob(os.path.join(REPO, 'src/%s/*.c' % a.binary)):
|
|
try:
|
|
out[q] = open(q).read()
|
|
except OSError:
|
|
pass
|
|
return out
|
|
|
|
|
|
def _jtbl_restore(snap):
|
|
"""Undo a carve/isolation EXACTLY: restore the config text and delete only the region files
|
|
this attempt created. Then re-extract and re-derive the stub map.
|
|
|
|
WHY THIS EXISTS (byte-proven 2026-07-21). A carve left behind by a draft the gate REJECTED has
|
|
no owner — the function is still INCLUDE_ASM — so `jr_inventory`'s 1:1 ownership assertion
|
|
(R32/R33) then refuses EVERY later isolation in that overlay:
|
|
committed .rodata carve ownership is not 1:1 … [('UNOWNED','0x801d288c')]
|
|
0x801d288c is func_801299C8's table: prepped, gate-rejected, carve stranded. The assertion is
|
|
CORRECT and caught it; the defect was leaving the carve behind. One failed draft poisoned the
|
|
whole overlay for the rest of the batch (4 isolate-FAILs downstream)."""
|
|
for q, txt in snap.items():
|
|
if q == '__mk_block__':
|
|
_mk_block_restore(a.binary, txt)
|
|
continue
|
|
open(q, 'w').write(txt)
|
|
# Remove every source file the attempt CREATED. Derived from the snapshot (which holds the
|
|
# exact pre-attempt file set), not from the `_jr_*` name shape — an isolation may emit a
|
|
# region whose name that glob does not predict, and a leftover .c is picked up by the OBJS
|
|
# glob at the next parse (R33: derive from the recorded state, do not re-guess it).
|
|
for rf in set(glob.glob(os.path.join(REPO, 'src/%s/*.c' % a.binary))) - set(snap):
|
|
try:
|
|
os.remove(rf)
|
|
except OSError:
|
|
pass
|
|
# R32/§93: CHECK the recovery's own exit status. `_sh` does not raise, so a failed re-extract here
|
|
# left the generated build inputs describing the abandoned isolation while config/ read clean —
|
|
# a recovery that silently did not recover, which is strictly worse than not attempting one.
|
|
if _sh(['make', '--no-print-directory', 'extract', 'BINARY=%s' % a.binary]).returncode:
|
|
print(' [jtbl] !! RESTORE INCOMPLETE: re-extract FAILED after undoing the carve — the tree '
|
|
'is NOT back at baseline; stop and `git checkout -- src/ config/` before trusting any '
|
|
'later verdict in this run')
|
|
_reload_corpus()
|
|
|
|
|
|
def _unsplice_body(fn, st, orig_txt):
|
|
"""Restore fn's INCLUDE_ASM stub after a prep that left the body spliced.
|
|
|
|
The isolation may have MOVED the body into a freshly-created region file, so the stub cannot
|
|
simply be written back to the path it came from — that would also undo the partition. Locate
|
|
the file that actually holds the body now and put back the stub line for THAT subseg (the
|
|
`asm/<bin>/nonmatchings/<subseg>` convention, subseg == the region file's basename). If nothing
|
|
moved, this restores the original file verbatim."""
|
|
body = drafts[fn]['c']
|
|
for cf in sorted(glob.glob(os.path.join(REPO, 'src/%s/*.c' % a.binary))):
|
|
t = open(cf).read()
|
|
if body in t:
|
|
sub = os.path.basename(cf)[:-2]
|
|
stub = 'INCLUDE_ASM("asm/%s/nonmatchings/%s", %s);' % (a.binary, sub, fn)
|
|
open(cf, 'w').write(t.replace(body, stub, 1))
|
|
return True
|
|
# body not found: nothing was spliced anywhere (carve failed before the splice landed) —
|
|
# restore the original text so the caller's snapshot logic sees an untouched tree.
|
|
if st is not None and orig_txt is not None:
|
|
open(st.path, 'w').write(orig_txt)
|
|
return True
|
|
return False
|
|
|
|
|
|
def _jtbl_prep_one(fn):
|
|
"""Prep ONE table-bearing draft's carve. Returns (ok, snapshot)."""
|
|
if not _fn_has_jtbl(fn):
|
|
return True, None
|
|
snap = _jtbl_snapshot()
|
|
done = []
|
|
for fn in [fn]:
|
|
st = _stubs.get(fn)
|
|
if st is None:
|
|
continue
|
|
line = _stub_line(fn)
|
|
txt = open(st.path).read()
|
|
if line not in txt:
|
|
continue
|
|
open(st.path, 'w').write(txt.replace(line, drafts[fn]['c'], 1)) # TEMPORARY splice
|
|
r = _sh([PY, 'tools/jtbl_carve.py', a.binary, '--func', fn])
|
|
out = (r.stdout or '') + (r.stderr or '')
|
|
if r.returncode and any(w in out for w in _ISO_WALLS):
|
|
# ISOLATE WITH THE BODY STILL SPLICED (byte-proven 2026-07-22, func_80135888).
|
|
# This used to un-splice first. `jr_isolate_all` accumulates each object's file-scope
|
|
# decls as the new region's `ambient` set, so partitioning around an INCLUDE_ASM stub
|
|
# gives the region a DIFFERENT decl context than the one the draft's body needs — and
|
|
# the gate then produced a byte-DIFF rather than a compile error, which is exactly why
|
|
# a batch of these read as "9/10 compile, 0 bank" and looked like a codegen wall.
|
|
# §61b's law (THE CARVE MUST FOLLOW THE SPLICE) extends one step: SO MUST THE
|
|
# ISOLATION. The un-splice was there only because the tool is stub-centric and a
|
|
# spliced function is no longer in corpus.stubs — a plumbing problem, handled below.
|
|
if _sh([PY, 'tools/jr_isolate_all.py', a.binary, '--only', fn]).returncode:
|
|
print(' [jtbl] isolate FAILED %s' % fn); continue
|
|
if _sh(['make', '--no-print-directory', 'extract', 'BINARY=%s' % a.binary]).returncode:
|
|
print(' [jtbl] extract-after-isolate FAILED %s' % fn); continue
|
|
r = _sh([PY, 'tools/jtbl_carve.py', a.binary, '--func', fn])
|
|
elif r.returncode and any(w in out for w in _ISLAND_WALLS) and a.binary.startswith('md_'):
|
|
# MODULE ISLAND TABLES NEED NO CARVE AT ALL (P31 S62 T3a, cookbook §303). The Makefile
|
|
# runs jtbl_rodata_pads --derive for every md_* object: the pads (lead/trailing zero
|
|
# words) are derived at build time from the retail island + the emission stream, so a
|
|
# matched body's table lands in place inside the main TU object — interior tables
|
|
# included (the old §260 peel was END-adjacent only). Byte-proven: md_SC03_076
|
|
# func_801F0A9C (9 entries + trailing pad) + func_801F0F28, sha 9a165e36…. The gate's
|
|
# only job here is to NOT refuse: leave the splice, skip isolate/split, let the build judge.
|
|
print(' [jtbl] module island table %s: handled by jtbl_rodata_pads --derive at build time' % fn)
|
|
r = subprocess.CompletedProcess(args=[], returncode=0)
|
|
elif r.returncode and any(w in out for w in _ISLAND_WALLS):
|
|
# THE §260 ISLAND SPLIT, AT GATE TIME (P31 S59). An md_* module binds its rodata
|
|
# island at offset 0 to its own code subseg; a table in that island cannot be tail-
|
|
# carved (jtbl_carve refuses with the messages matched above). The byte-proven fix is
|
|
# ONE inserted config line binding the fn's table span to the fn's OWN isolated
|
|
# object (cookbook §260, sha 9a165e36… on md_SC03_076/func_801F218C).
|
|
# ORDER MATTERS twice over:
|
|
# * isolate FIRST, insert SECOND — jr_inventory asserts every .rodata carve
|
|
# resolves to exactly one owner (R32), and the fresh line's owner would look like
|
|
# nobody until the fn's C is in the tree; running the isolation before the line
|
|
# exists keeps the assertion out of the loop.
|
|
# * isolate WITH THE BODY STILL SPLICED — same §61b reason as the branch above:
|
|
# jr_isolate_all carries each region's decl context, and partitioning around the
|
|
# stub gives the region a different environment than the draft's body needs.
|
|
# Only an END-ADJACENT table can take the split (the island is a stack; the probe
|
|
# inside --island-split enforces it and names the blocking owner otherwise), so a
|
|
# blocked member fails here LOUDLY and becomes CARVE-REFUSED — the honest verdict.
|
|
if _sh([PY, 'tools/jr_isolate_all.py', a.binary, '--only', fn]).returncode:
|
|
print(' [jtbl] island-isolate FAILED %s' % fn); continue
|
|
spl = _sh([PY, 'tools/jtbl_carve.py', a.binary, '--island-split', '--func', fn])
|
|
if spl.returncode:
|
|
last = ((spl.stdout or '') + (spl.stderr or '')).strip().splitlines()[-1:] or ['']
|
|
# §322b FALLBACK (P31 S70). The island-split probe can refuse because the function is
|
|
# a TAIL case rather than an island-end one — and it NAMES the right route in its own
|
|
# refusal: "... is 'tail', not 'island-end' — table(s) in the data tail — standard §8a
|
|
# carve at gate time". Booking that as CARVE-REFUSED records a verdict about the ROUTE
|
|
# WE CHOSE, not about the function: the first jtbl_carve refusal merely mentioned a
|
|
# leading island, which sent it down this branch, and no branch ever tried the carve
|
|
# the tool actually asked for. The isolate above has already run, so the standard
|
|
# route is just re-extract + re-carve — the tail of the _ISO_WALLS branch.
|
|
if "not 'island-end'" in last[0] or 'standard' in last[0]:
|
|
print(' [jtbl] island-split says TAIL -> falling back to the standard §8a carve %s'
|
|
% fn)
|
|
if _sh(['make', '--no-print-directory', 'extract',
|
|
'BINARY=%s' % a.binary]).returncode:
|
|
print(' [jtbl] extract-after-isolate FAILED %s' % fn); continue
|
|
r = _sh([PY, 'tools/jtbl_carve.py', a.binary, '--func', fn])
|
|
if r.returncode:
|
|
tail_ = ((r.stdout or '') + (r.stderr or '')).strip().splitlines()[-1:] or ['']
|
|
print(' [jtbl] standard carve ALSO refused %s: %s' % (fn, tail_[0][:160]))
|
|
continue
|
|
else:
|
|
print(' [jtbl] island-split REFUSED %s: %s' % (fn, last[0][:160])); continue
|
|
if _sh(['make', '--no-print-directory', 'extract', 'BINARY=%s' % a.binary]).returncode:
|
|
print(' [jtbl] extract-after-island-split FAILED %s' % fn); continue
|
|
r = _sh([PY, 'tools/jtbl_carve.py', a.binary, '--func', fn])
|
|
# ALWAYS un-splice — but the isolation may have MOVED the body to a new region file, so
|
|
# find where it actually is now and restore the stub line for THAT subseg. The gate then
|
|
# re-splices identical text, so the build it verifies is the one proven above.
|
|
if not _unsplice_body(fn, st, txt):
|
|
print(' [jtbl] could not un-splice %s (body not found)' % fn); continue
|
|
if r.returncode:
|
|
last = ((r.stdout or '') + (r.stderr or '')).strip().splitlines()[-1:] or ['']
|
|
print(' [jtbl] carve FAILED %s: %s' % (fn, last[0][:120])); continue
|
|
done.append(fn)
|
|
if not done:
|
|
_jtbl_restore(snap)
|
|
return False, None
|
|
# R49/R61 (P32 T1a, cookbook §498): the post-carve re-extract's exit code was IGNORED. When the carve
|
|
# writes a layout ld_interleave refuses (measured: a regenerated JTBL_INTERLEAVE that dropped the
|
|
# resident's `--pre hdr.rodata.o`), `make extract` fails, the build then links the STALE script, and
|
|
# the draft is booked as DIFF — an instrument failure wearing a codegen verdict. Refuse loudly instead.
|
|
_rx = _sh(['make', '--no-print-directory', 'extract', 'BINARY=%s' % a.binary])
|
|
if _rx.returncode:
|
|
_last = ((_rx.stdout or '') + (_rx.stderr or '')).strip().splitlines()[-3:] or ['']
|
|
print(' [jtbl] !! extract-after-carve FAILED %s (rc %d) — carve config restored; this is a CARVE '
|
|
'refusal, NOT a draft verdict: %s' % (done[0], _rx.returncode, ' | '.join(l[:100] for l in _last)))
|
|
_jtbl_restore(snap)
|
|
if _sh(['make', '--no-print-directory', 'extract', 'BINARY=%s' % a.binary]).returncode:
|
|
print(' [jtbl] !! RESTORE INCOMPLETE: re-extract FAILED after undoing the carve')
|
|
return False, None
|
|
_reload_corpus()
|
|
print(' [jtbl] carved %s' % done[0])
|
|
return True, snap
|
|
|
|
|
|
def _jtbl_reconcile(fn):
|
|
"""Post-carve draft reconcile (Phase-29). After _jtbl_prep_one carves the table, fn's stub lives
|
|
in a split TU carrying a §8b decl layer — canonical externs for the data/callees fn's body
|
|
touches (jr_isolate_all accumulates each earlier region's file-scope decls as the new region's
|
|
ambient set). The draft declares those SAME symbols for its own byte-match, and where the two
|
|
disagree the whole-binary gate fails to COMPILE (`conflicting types for D_XXXX / func_XXXX`), not
|
|
a byte miss — the exact class that made a batch of loose-typed jtbl families read as "carves, 0
|
|
banks" and look like a codegen wall.
|
|
|
|
The chain that fixes it ALREADY exists and takes --src-file: cast_call_sites (§17a-1, the callee
|
|
fn-ptr cast) + reconcile_tu (§8d, conform the draft's DATA decls to what THIS TU declares, casting
|
|
every use so the access is byte-identical). gate_stage runs it, but only PRE-carve and against the
|
|
default src/<ov>/<ov>.c — never fn's real TU, because for a jtbl fn that TU is this split file,
|
|
which did not exist when the ladder ran (gate_stage.py:271 deleted its batch jtbl stage for exactly
|
|
this reason: "harvest_verify owns the splice, so the prep belongs there"). So run the chain HERE,
|
|
against the carved TU, and update the draft in place. Draft-only rewrite (no shared-state edit, so
|
|
no §61 undo needed); the whole-binary gate stays the sole arbiter (G3/P9)."""
|
|
st = _stubs.get(fn)
|
|
if st is None:
|
|
return
|
|
tu = os.path.relpath(st.path, REPO)
|
|
base = os.path.join(REPO, '.run', '%s_jtbl_rc' % a.binary)
|
|
shutil.rmtree(base, ignore_errors=True)
|
|
cur = os.path.join(base, 'in')
|
|
os.makedirs(cur, exist_ok=True)
|
|
open(os.path.join(cur, fn + '.c'), 'w').write(drafts[fn]['c'])
|
|
for tool, suf in (('cast_call_sites.py', '-cast'), ('reconcile_tu.py', '-rc')):
|
|
out = cur + suf
|
|
shutil.rmtree(out, ignore_errors=True)
|
|
_sh([PY, 'tools/' + tool, '--overlay', a.binary, '--in', cur, '--out', out,
|
|
'--src-file', tu])
|
|
if os.path.isdir(out) and os.path.exists(os.path.join(out, fn + '.c')):
|
|
cur = out
|
|
reconciled = os.path.join(cur, fn + '.c')
|
|
if os.path.exists(reconciled):
|
|
drafts[fn]['c'] = open(reconciled).read()
|
|
|
|
|
|
def _stub_line(fn):
|
|
s = _stubs[fn]
|
|
return 'INCLUDE_ASM("%s", %s);' % (s.asm_dir, s.symbol)
|
|
|
|
|
|
def render(fns):
|
|
"""{path: text} with each draft spliced into the TU that actually holds its stub."""
|
|
out = dict(baseline)
|
|
for fn in fns:
|
|
p, line = _stubs[fn].path, _stub_line(fn)
|
|
if line not in out.get(p, ''):
|
|
return None # stub missing -> skip (don't let a no-op build false-pass)
|
|
out[p] = out[p].replace(line, drafts[fn]['c'], 1)
|
|
return out
|
|
|
|
|
|
def _write(state):
|
|
for p, text in state.items():
|
|
open(p, 'w').write(text)
|
|
|
|
|
|
def attempt(fns):
|
|
global _last_sha
|
|
s = render(fns)
|
|
if s is None:
|
|
_last_sha = _NO_BUILD # no build ran -> classify_fail reports SKIP, not a false CC1-FAIL
|
|
return False
|
|
_write(s)
|
|
return build() == a.good_sha
|
|
|
|
|
|
def commit(fns):
|
|
global baseline
|
|
baseline = render(fns)
|
|
verified.extend(fns)
|
|
|
|
|
|
i = 0
|
|
while i < len(items):
|
|
chunk = items[i:i + a.chunk]
|
|
i += a.chunk
|
|
# per-chunk jtbl prep (chunk==1 is the prescribed default, so this is per-function): carve the
|
|
# table, and if the gate then REJECTS the draft, undo the carve — a stranded carve has no owner
|
|
# and poisons every later isolation in the overlay (see _jtbl_restore).
|
|
_jsnap = None
|
|
if len(chunk) == 1:
|
|
_ok, _jsnap = _jtbl_prep_one(chunk[0])
|
|
if not _ok:
|
|
# THE CARVE WAS REFUSED — do NOT build this draft (Phase 29 SESSION-22, byte-witnessed).
|
|
# `_ok` was computed and ignored here, so a table-bearing draft whose carve jtbl_carve
|
|
# REFUSED (§59(3): a non-contiguous same-subseg table) was spliced and built anyway. It
|
|
# cannot link without its carve, so the build fails with `Error 33` — and that failure was
|
|
# then recorded as **CC1-FAIL**, i.e. a codegen-flavoured verdict, for what is purely a
|
|
# carve-plumbing wall. Worse, the tail of the batch inherited the broken object: one
|
|
# refused carve produced FOUR CC1-FAILs on the same `ov_SC01_077_o0.o`, three of them
|
|
# against drafts that were never even diagnosed, and the run ended unable to rebuild the
|
|
# binary at all (`final SHA None`). Skip it with a NAMED class instead: honest, cheaper by
|
|
# one build, and it leaves the tree where the next draft can be judged on its own merits.
|
|
fn = chunk[0]
|
|
failed.append((fn, 'CARVE-REFUSED'))
|
|
print(' - %s (%s) [CARVE-REFUSED: jtbl_carve declined the table; §59(3) plumbing, '
|
|
'NOT a codegen verdict]' % (fn, drafts[fn]['conf']))
|
|
continue
|
|
if _jsnap is not None: # a jtbl carve happened -> reconcile the draft vs the CARVED TU
|
|
_jtbl_reconcile(chunk[0])
|
|
if attempt(chunk):
|
|
commit(chunk)
|
|
print(' + chunk(%d): %s' % (len(chunk), ' '.join(chunk)))
|
|
elif len(chunk) == 1:
|
|
if _jsnap is not None:
|
|
_jtbl_restore(_jsnap) # stranded-carve + truncated-TU undo (R32/§61)
|
|
# AND PUT THE SPLICE BACK (Phase 29 SESSION-22). `attempt()` writes the candidate render and
|
|
# does not restore on failure — it relied on the NEXT attempt's render to overwrite it. That
|
|
# left the tree dirty BETWEEN drafts, and `_jtbl_snapshot()` snapshots the tree AS IT FINDS
|
|
# IT: a later draft's carve therefore captured an earlier FAILED draft's splice, and its undo
|
|
# faithfully RE-APPLIED it — after the final `_write(baseline)` had already run. Net effect: a
|
|
# run that verified nothing still ended with a spliced body on disk and could not rebuild the
|
|
# binary (`final SHA None`), which reads exactly like a byte regression and is not one.
|
|
# The invariant is one line: the tree is at `baseline` except while a draft is under test.
|
|
_write(baseline)
|
|
# ATOMIC CHUNK — do NOT bisect (Phase-28 T6). The old code fell into the loop below and
|
|
# re-ran attempt([fn]) on the SAME single element against the SAME baseline: a byte-identical
|
|
# DUPLICATE build. classify_fail reads _last_sha/_last_err, which the failed attempt(chunk)
|
|
# above ALREADY set — so the re-attempt bought nothing but a second cc1+maspsx+as+ld. On the
|
|
# gate's hot path (--chunk 1, the prescribed default: chunked failures mis-attribute innocent
|
|
# neighbours, cookbook:1568) that was 1.35 builds/draft at the measured 65% bank rate; now 1.0
|
|
# — ~26% fewer builds, every wave, for one branch. The gate's verdict is unchanged.
|
|
fn = chunk[0]
|
|
klass = classify_fail(_last_sha)
|
|
failed.append((fn, klass))
|
|
print(' - %s (%s) [%s]' % (fn, drafts[fn]['conf'], klass))
|
|
else:
|
|
for fn in chunk: # bisect: isolate the matches from the misses, CLASSIFY the misses
|
|
if attempt([fn]):
|
|
commit([fn]); print(' + %s' % fn)
|
|
else:
|
|
klass = classify_fail(_last_sha) # DIFF (real codegen) vs PLUMBING (recoverable) vs CC1-FAIL
|
|
failed.append((fn, klass))
|
|
print(' - %s (%s) [%s]' % (fn, drafts[fn]['conf'], klass))
|
|
_write(baseline) # same invariant as the atomic branch above
|
|
|
|
# restore the accumulated verified state and confirm the binary is byte-identical
|
|
_write(baseline)
|
|
final = build()
|
|
# failure breakdown by class — a PLUMBING count > 0 means "recoverable, not a compiler wall" (T4)
|
|
from collections import Counter
|
|
_klass = Counter(k.split(':')[0] for _, k in failed)
|
|
print('\n=== RESULT ===')
|
|
print('verified %d / failed %d ; final SHA %s (%s)' % (
|
|
len(verified), len(failed), final,
|
|
'BYTE-IDENTICAL' if final == a.good_sha else '*** MISMATCH — investigate ***'))
|
|
if _klass:
|
|
print(' failed by class:', ' '.join('%s=%d' % (k, n) for k, n in sorted(_klass.items())))
|
|
print('VERIFIED:', ' '.join(verified) or '(none)')
|
|
print('FAILED :', ' '.join(fn for fn, _ in failed) or '(none)')
|
|
if verified:
|
|
# §404 (P31 S70): "verified" means THE BYTE-GATE ACCEPTED IT, not that it is banked. Run standalone
|
|
# this tool leaves the INCLUDE_ASM stub in place; `gate_stage.py` is the entrypoint that persists,
|
|
# propagates and commits. S70 read `verified 1 / VERIFIED: func_8002B0B4` here, committed straight
|
|
# afterwards, and banked NOTHING — the stub is still at src/800.c:18341 and a bank was reported to
|
|
# the owner that never existed. Say it at the point of confusion, not in a doc nobody re-reads.
|
|
print(' NOTE: "verified" = the byte-gate ACCEPTED these drafts. If you invoked harvest_verify\n'
|
|
' directly, they are NOT banked — re-run through `tools/gate_stage.py --drafts <dir>\n'
|
|
' --binary %s [--commit]` to persist. Confirm with corpus.stubs(), not this line.'
|
|
% a.binary)
|
|
# R32 — assert the gate LEFT THE TREE where it found it (plus whatever it banked). `_write(baseline)`
|
|
# above restores the files render() manages; a carve/isolation can touch files it does not. At 0
|
|
# verified the tracked diff must be EMPTY, and any residue is a failed draft still spliced in — which
|
|
# the next run would silently gate on top of. Name the files; do not make the operator go looking.
|
|
_dirty = [l[3:] for l in _sh(['git', 'status', '--porcelain', '--', 'src/%s' % a.binary,
|
|
'config']).stdout.splitlines()]
|
|
if _dirty and not verified:
|
|
print(' !! TREE NOT CLEAN at 0 verified — residue from a failed draft/carve, NOT a result:')
|
|
for q in _dirty[:12]:
|
|
print(' %s' % q)
|
|
print(' recover with: git checkout -- src/%s config' % a.binary)
|
|
open(a.verified_out, 'w').write('\n'.join(verified) + '\n')
|
|
# failed_out stays NAMES-only (backward-compatible for existing consumers); the class goes to a sidecar
|
|
open(a.failed_out, 'w').write('\n'.join(fn for fn, _ in failed) + '\n')
|
|
open(a.failed_out.rsplit('.', 1)[0] + '.classified.txt', 'w').write(
|
|
'\n'.join('%s\t%s' % (fn, k) for fn, k in failed) + '\n')
|