#!/usr/bin/env python3 """Gate a BATCH of main-EXE drafts the only way main can be gated: a CLEAN rebuild. WHY THIS EXISTS (P31, 2026-08-15). `gate_lane`/`gate_stage` build INCREMENTALLY. That is fine for overlays, but main's `make extract` runs the EXE-only `psyq_integrate` + `ld_interleave` steps which REWRITE THE LINKER SCRIPT. An incremental build after a source change re-runs that on an already-rewritten `.ld` and produces a FALSE DIFF -- exactly the trap R22's own rationale describes. This cost the campaign a night: 4 byte-correct main drafts gated 0/4, and I wrote up a nonexistent "linker defect" before the null-draft control exposed it (with NO draft substituted at all, `make build BINARY=main` still produced the same wrong hash). So main is gated like this instead: substitute every draft -> make extract BINARY=main -> make build BINARY=main -> compare SHA ONE clean build verifies the WHOLE BATCH, which is what makes this cheap: 34 functions banked in a single rebuild. On failure the batch is bisected so one bad draft cannot sink the rest. TWO MAIN-SPECIFIC HAZARDS THIS TOOL HANDLES: 1. IN-TU CROSS-DRAFT DECLARATION CONFLICTS. Batching N drafts into one .c means their `extern`s must agree WITH EACH OTHER, not merely with the file: wave J hit `D_800A4ED4` declared s16 by one draft and u16 by another, and `func_8001C9D0` as void / void* / s32 across three. C rejects the TU. We resolve greedily (keep in order, drop the incompatible) and REPORT the dropped set -- those drafts are usually correct and recoverable with a cast-at-use. Compatibility compares TYPE SIGNATURES ONLY: parameter NAMES are irrelevant, and a checker that compares them wrongly discards good work (R39 -- I made exactly that mistake first). 2. STALE .s AFTER A REVERT. Once a function becomes C, splat stops emitting its .s. If you then revert src/, `corpus.stubs()` raises because the tree and source disagree. Always `make extract BINARY=main` BEFORE resolving stubs. Usage: gate_main.py [--apply] [--no-bisect] slate.json: [{"fn": ..., "draft": ...}, ...] (binary is assumed main) default is a DRY RUN that reports what would be substituted and any conflicts. --apply performs the substitution + clean rebuild and leaves banked drafts in the tree. """ import argparse, collections, functools, json, re, subprocess, sys sys.path.insert(0, 'tools') import corpus # stdout is BUFFERED when redirected to a file -- a long run then looks hung with an # empty log (measured: 16 min of silence during a bisect). Always flush. print = functools.partial(print, flush=True) GOOD = '143dbb89f34491258bbc27810d0a12ec8b43a8dd' TYPES = {'void','char','short','int','long','unsigned','signed','float','double','const', 'volatile','s8','u8','s16','u16','s32','u32','f32','s64','u64','struct','union'} DECL = re.compile(r'^\s*extern\s+([^;]+?)\s*;\s*$', re.M) def sym_of(d): m = re.search(r'\b(D_[0-9A-Fa-f]{8}|func_[0-9A-Fa-f]{8}|[A-Za-z_]\w*)\s*(?:\[|\()', d) if m: return m.group(1) m = re.search(r'\b(D_[0-9A-Fa-f]{8}|func_[0-9A-Fa-f]{8})\b', d) return m.group(1) if m else None def typesig(d): """Type signature only. Parameter NAMES do not affect C compatibility (R39: comparing them dropped 2 good drafts before I fixed it) -- but the DECLARATOR SUFFIX absolutely does. `u8 D_x` and `u8 D_x[]` are INCOMPATIBLE; an earlier version of this function split on the symbol and kept only the prefix, so both reduced to ('u8', None) and a real conflict slipped through into the build (wave K, D_80078D98: one draft scalar, two array -> compile error AFTER the batch had reported BYTE-IDENTICAL). Too-coarse and too-strict are both defects.""" d = ' '.join(d.split()); sym = sym_of(d) or '' m = re.search(r'\((.*)\)\s*$', d) ret = d.split(sym)[0].strip() if sym and sym in d else d if not m: # data decl: keep the declarator suffix ('' vs '[]' vs '[N]' -> normalized to '[]') tail = d.split(sym, 1)[1].strip() if sym and sym in d else '' tail = '[]' if tail.startswith('[') else tail return (ret, tail) params = tuple(' '.join(t for t in re.findall(r'[A-Za-z_]\w*|\*', p) if t in TYPES or t == '*') for p in m.group(1).split(',')) return (ret, params) def run(cmd, **kw): return subprocess.run(cmd, shell=True, capture_output=True, text=True, **kw) def sha(): r = run("sha1sum build/us/SLUS_007.26") return r.stdout.split()[0] if r.returncode == 0 and r.stdout else None def resolve_conflicts(slate): kept, dropped, seen = [], [], {} for e in slate: ds = [(sym_of(d), typesig(d)) for d in DECL.findall(open(e['draft']).read())] ds = [(s, t) for s, t in ds if s] clash = [(s, seen[s], t) for s, t in ds if s in seen and seen[s] != t] if clash: dropped.append({'fn': e['fn'], 'symbol': clash[0][0], 'kept': str(clash[0][1]), 'this': str(clash[0][2])}) continue for s, t in ds: seen[s] = t kept.append(e) return kept, dropped TYPEDEF_BLOCK = re.compile( r'^[ \t]*typedef\s+(?:struct|union|enum)?[^;{]*\{[^{}]*\}\s*(\w+)\s*;[ \t]*\n', re.M) TYPEDEF_PLAIN = re.compile(r'^[ \t]*typedef\s+[\w\s\*]+?\s(\w+)\s*;[ \t]*\n', re.M) def strip_dup_typedefs(body, already): """Drop typedefs the destination TU (or an earlier body in this batch) already defines. Each draft is written to compile STANDALONE, so it carries its own `typedef struct {...} SVECTOR;`. Once one such function is banked, that typedef lives in the .c forever and every later draft defining its own collides -- a C89 duplicate-typedef error, not a byte miss. `harvest_verify` already does this; wave L lost a verified-correct draft because this tool did not. Returns (body, names_defined_now).""" defined = set() for pat in (TYPEDEF_BLOCK, TYPEDEF_PLAIN): out, pos = [], 0 for m in pat.finditer(body): name = m.group(1) if name in already: out.append(body[pos:m.start()]); pos = m.end() # drop the duplicate else: defined.add(name) out.append(body[pos:]) body = ''.join(out) return body, defined def substitute(entries): """Replace each INCLUDE_ASM stub line with its draft body. Returns count.""" stubs = {st.symbol: st for st in corpus.stubs('main').values()} byfile = collections.defaultdict(list) for e in entries: st = stubs.get(e['fn']) if st: byfile[st.path].append((e['fn'], st.asm_dir, e['draft'])) n = 0 for path, items in byfile.items(): t = open(path).read() # names the destination file already defines, plus anything a shared header provides seen = set(TYPEDEF_BLOCK.findall(t)) | set(TYPEDEF_PLAIN.findall(t)) for fn, asmdir, draft in items: body = "\n".join(l for l in open(draft).read().splitlines() if not l.strip().startswith('#include')) body, newly = strip_dup_typedefs(body, seen) seen |= newly old = f'INCLUDE_ASM("{asmdir}", {fn});' if old in t: t = t.replace(old, body); n += 1 open(path, 'w').write(t) return n def clean_build(): """The ONLY trustworthy main verification: extract (rewrites the .ld) then build. DELETE THE OUTPUT FIRST, AND CHECK THE RETURN CODE. sha() reads build/us/SLUS_007.26 off disk; if `make build` FAILS (e.g. a compile error) the PREVIOUS successful binary is still sitting there, so sha() returns the GOOD hash and this tool reports BYTE-IDENTICAL for a build that never ran -- a FALSE PASS. That is exactly how it once claimed "43 banked" on a batch whose TU did not compile; the clean-fleet R22 caught it afterwards. A verifier that can pass without building is worse than no verifier.""" run("rm -f build/us/SLUS_007.26") run("make extract BINARY=main") r = run("make build BINARY=main") if r.returncode != 0: return None, r # build failed -> no hash, and never a pass return sha(), r def try_batch(entries): run("git checkout -- src/") run("make extract BINARY=main") # regenerate .s for the reverted stubs (hazard 2) substitute(entries) got, r = clean_build() return got == GOOD, got, r def main(): ap = argparse.ArgumentParser() ap.add_argument('slate'); ap.add_argument('--apply', action='store_true') ap.add_argument('--no-bisect', action='store_true') a = ap.parse_args() slate = json.load(open(a.slate)) kept, dropped = resolve_conflicts(slate) print(f"slate {len(slate)} -> {len(kept)} compatible, {len(dropped)} dropped for in-TU decl conflict") for d in dropped: print(f" DROP {d['fn']}: {d['symbol']} kept={d['kept']} this={d['this']}") if dropped: print(" (dropped drafts are usually CORRECT -- recover with a cast-at-use: adopt the") print(" other declaration verbatim and adapt at the use site, e.g. (&D_x)[i].)") if not a.apply: print("\nDRY RUN. Re-run with --apply to substitute and clean-rebuild.") return ok, got, r = try_batch(kept) if ok: print(f"\nBANKED {len(kept)} main functions -- {got} BYTE-IDENTICAL") json.dump([e['fn'] for e in kept], open('.run/gate_main_banked.json', 'w')) return # A COMPILE error names its own culprit -- read it instead of bisecting. Bisection here costs # a FULL CLEAN REBUILD per step (~2-4 min), so it is worst-case hours; the compiler already # told us the symbol and line. (Measured the hard way: a 41-draft bisect ran 28+ min with no # output.) Only a byte MISMATCH with a clean compile genuinely needs bisection. err = (r.stderr or '') + (r.stdout or '') m = re.search(r'^(.*?):(\d+): previous declaration of `([^\']+)\'', err, re.M) if m: print(f"\nCOMPILE conflict on `{m.group(3)}' at {m.group(1)}:{m.group(2)} —" f" NOT bisecting; drop or reconcile the drafts declaring it and re-run.") offenders = [e['fn'] for e in kept if re.search(rf"\b{re.escape(m.group(3))}\b", open(e['draft']).read())] print(" drafts declaring it:", offenders) run("git checkout -- src/") return print(f"\nbatch FAILED (sha {got}); {'not bisecting' if a.no_bisect else 'bisecting'}") if a.no_bisect: run("git checkout -- src/"); return good = [] lo = kept while lo: half = max(1, len(lo)//2) head, lo = lo[:half], lo[half:] ok, got, _ = try_batch(good + head) if ok: good += head elif len(head) == 1: print(f" reject {head[0]['fn']}") else: lo = head + lo # split further ok, got, _ = try_batch(good) print(f"\nBANKED {len(good)} of {len(kept)} after bisection -- {got}" f"{' BYTE-IDENTICAL' if ok else ' *** STILL MISMATCHED ***'}") json.dump([e['fn'] for e in good], open('.run/gate_main_banked.json', 'w')) if __name__ == '__main__': main()