#!/usr/bin/env python3 """main_lane.py — the SECOND drafting lane, for the EXE (`main`), on its own gate cadence. WHY A SEPARATE LANE (P31 S59, Drew's call). `main` is excluded from every wave draw (`draw_wave` passes `--exclude-bins main`) for a good reason: its gate is a CLEAN WHOLE-EXE rebuild that bisects on failure, and putting that on the critical path of a loop whose other gates take minutes cost the campaign three measured stalls (39 min unfinished on 29 drafts, 25 on 8, 65+ on 8, while later waves queued behind it). So main sat outside the loop entirely — **1,713 open stubs, no lane, no cadence** — while the overlay lane ran at roughly a quarter of the API ceiling (mean 55 req/min against a 200-230 knee) because CARD SUPPLY, not throughput, is its constraint. Two populations, one idle half of a rate limit: main gets its own lane rather than a bigger wave. WHAT THIS LANE DOES, AND WHAT IT DELIBERATELY DOES NOT draw (main only) -> draft -> reloc pre-filter -> ONE gate_main batch -> commit -> repeat. * It NEVER gates through `sweep_parallel` or `gate_stage`. Both build INCREMENTALLY, and main's `make extract` re-runs `psyq_integrate`/`ld_interleave`, which rewrite the linker script — an incremental build after a source change then produces a FALSE DIFF. `sweep_parallel` used to accept main anyway: wave `ab` drew 105 main cards and banked 0 of 105 while its 115 non-main cards banked 94, and the whole wave read as a drafting failure (that is R43's origin). * It gates in BATCHES because one clean rebuild verifies the whole slate — that is what makes main affordable at all (34 functions banked in a single rebuild, measured). The reloc pre-filter is what keeps a batch from bisecting: a draft that names symbols the target never references cannot bank and must not enter the slate. * It COMMITS the moment a batch is green (R42). `gate_main` does not commit — it substitutes and verifies, and its output is unverified by construction until the SHA matches, which is exactly why the overlay gater REFUSES to adopt dirty main sources. * Parked drafts come first. `.run/main_queue/*.json` holds main drafts the overlay waves produced before main was excluded (170 of them today) — free work, already drafted, never gated. CONCURRENCY. `gate_main` takes `.run/auto/gate.main.lock` (blocking) so two main gates can never interleave into `src/800.c`. This lane adds no lock of its own; it just never runs two gates at once, and the overlay lanes are untouched by construction — different binaries, different gate. tools/main_lane.py --once # one cycle, for a supervised trial tools/main_lane.py # loop forever (the lane script's mode) """ import argparse import glob import re import json import os import subprocess import sys import time sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import ox_campaign as OX # noqa: E402 REPO = OX.REPO if hasattr(OX, "REPO") else os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PY = ".venv/bin/python" LEDGER = ".run/main_lane_ledger.jsonl" RED_SENTINEL = ".run/main_lane.BASELINE_RED" class BaselineRed(RuntimeError): """The committed tree does not build byte-identical with NO draft substituted. Raised instead of judging (or retrying) any draft: against a red baseline every gate verdict is FALSE — S59 measured it the hard way, when auto-commit 24bd25993 adopted a mid-flight gate_main substitution at 14:57 and this lane then drafted four 200-card rounds (m00-m03 cycles, ~737 drafts) and banked ZERO until 18:43, burning ~50 clean rebuilds and a tries strike on every parked draft it falsely rejected (R40/R43).""" def log(msg): print(f"[{time.strftime('%H:%M:%S')}] [main-lane] {msg}", flush=True) def next_tag(): """m00..m99 then n00..n99 — 3-character namespaces that can never collide with the overlay lane's 2-letter wave tags (its `used` set globs `.run/wave_??_cards.json`). The original single prefix EXHAUSTED on 2026-08-26 after the overnight campaign consumed all 100 m-tags, and the lane crash-looped "out of main-lane tags" every 20 s while ~103 parked drafts waited (the R51 finite-namespace class: a lifetime cap nobody expected to reach).""" used = {os.path.basename(q)[5:8] for pat in ("m", "n") for q in glob.glob(f".run/wave_{pat}??_cards.json")} for pfx in ("m", "n"): for i in range(100): t = f"{pfx}{i:02d}" if t not in used: return t return None MAX_RETRY = 2 def parked_slate(): """Main drafts waiting to be gated: the overlay waves' parked queue, plus this lane's own failures (a failed batch is evidence, not garbage — with bisecting, most of a failed slate is innocent). A draft that has failed MAX_RETRY times is left in the failed file and skipped, so a permanently-unbankable body cannot spin the lane forever.""" out = [] for p in sorted(glob.glob(".run/main_queue/*.json") + [".run/main_queue_failed.json"]): if not os.path.exists(p): continue try: rows = json.load(open(p)) except Exception: continue for r in rows: d = r.get("draft") or r.get("c") if d and os.path.exists(d) and int(r.get("tries", 0)) < MAX_RETRY: out.append({"fn": r.get("fn") or r.get("name"), "draft": d, "src": p, "tries": int(r.get("tries", 0))}) seen, uniq = set(), [] for r in out: # a function can be parked by two waves if r["fn"] and r["fn"] not in seen: seen.add(r["fn"]); uniq.append(r) return uniq def draw(tag, n, lo, hi): """Draw main-only cards. The quotas are OFF: tells and jtbl-carve are overlay lanes, and main's jtbl members are `main-manual` by the island probe's own classification.""" cards = f".run/wave_{tag}_cards.json" if os.path.exists(cards): log(f"{tag}: cards already drawn, reusing") return cards # --max-bins MUST be passed here (P31 S60). build_wave_atlas defaults to 12 gate groups, a cap # that exists because each group costs one whole-binary rebuild — and main's own --only-bins # docstring says the opposite applies to it: "main is gated ONCE per SLATE, so main has no # per-TU gate cost and --max-bins can be large." Nobody passed it, so this lane asked for 200 # cards and drew 32 (12 of main's TUs), every wave. Measured cost: main banked ~19 stubs/hour # against 1,291 remaining while the overlay lane ran 650-card waves beside it. bins = os.environ.get("MAIN_MAX_BINS", "400") r = OX.sh(f"{PY} tools/build_wave_atlas.py {cards} {n} --min-ins {lo} --max-ins {hi} " f"--max-bins {bins} " f"--only-bins main --one-per-gid --retry-unbanked --tells-quota 0 --jtbl-quota 0", timeout=3600, quiet=False) if not os.path.exists(cards): log(f"{tag}: draw produced no cards ({(r.stderr or '')[-200:]})") return None return cards def _stub_removals(): """Functions whose INCLUDE_ASM line disappeared from main's sources in the WORKING TREE. This is the ground truth for "what did this run bank", and it is deliberately not a tool's self-report: the first version of this function asked `corpus.stubs('main')` whether each name was still a stub, and `corpus.stubs` returns a dict keyed by ADDRESS (int). Comparing a function NAME against a set of ints is always True, so the lane reported 12 banked of 12 from a gate that had banked nothing and committed nothing. A check that is true about the wrong thing is worse than no check — it is the exact defect class R32/R40 exist for, and it took 17 seconds to produce a confident lie.""" r = OX.sh("git diff --unified=0 -- src/*.c | grep '^-INCLUDE_ASM' || true") out = [] for line in (r.stdout or "").splitlines(): i = line.rfind(",") if i > 0: out.append(line[i + 1:].strip().rstrip(");").strip()) return [x for x in out if x.startswith(("func_", "D_")) or x] def _main_sha_green(): """The arbiter (G3/P9): main builds byte-identical. Held under main's own gate lock so this can never race a gate_main.""" r = OX.sh("( flock -w 3600 9 || exit 1; make check BINARY=main 2>&1 | tail -3 ) " "9>.run/auto/gate.main.lock", timeout=7200) out = (r.stdout or "") + (r.stderr or "") return ("BYTE-IDENTICAL" in out), out.strip().splitlines()[-1] if out.strip() else "" def _gate_once(tag, slate): """Substitute this slate, clean-rebuild, and return (banked_names, gate_main's output).""" sp = f".run/main_slate_{tag}.json" json.dump([{"fn": s["fn"], "draft": s["draft"]} for s in slate], open(sp, "w"), indent=1) t0 = time.time() r = OX.sh(f"{PY} tools/gate_main.py {sp} --apply", timeout=14400, quiet=False) out = ((r.stdout or "") + (r.stderr or "")).strip() banked = _stub_removals() log(f"{tag}: {len(slate)} draft(s) -> {len(banked)} banked in {(time.time()-t0)/60:.1f} min") return banked, out def gate_batch(tag, slate, depth=0): """One clean whole-EXE rebuild for the slate, BISECTING on a compile conflict. gate_main bisects byte MISMATCHES but deliberately refuses to bisect a COMPILE conflict: it names the symbol and stops, because a decl clash is not a wrong body (§236). That is right for a tool whose caller might be a human, and wrong for an unattended lane — the first live batch hit `COMPILE conflict on func_80017778` where the symbol was in the TU, not in any draft, so there was nothing to drop and 40 innocent drafts died with it. Halving costs one rebuild per level and a main rebuild here measures ~18 s, so the lane bisects rather than discards. Nothing is credited until BOTH the stubs are gone from the tree AND main builds byte-identical. """ if not slate: return 0, [] log(f"{tag}: gating {len(slate)} draft(s)" + (f" (bisect depth {depth})" if depth else " — ONE clean whole-EXE rebuild")) banked, out = _gate_once(tag, slate) if 'BASELINE RED' in out: # gate_main ran its no-draft control and the TREE, not the slate, is broken. Do not # bisect, do not bump tries — the slate is unjudged and fully reusable (S59). raise BaselineRed("gate_main's no-draft control found the committed baseline RED " "mid-cycle — the slate was not judged") if banked: green, line = _main_sha_green() if not green: log(f"{tag}: REFUSING to credit {len(banked)} substitution(s) — main is NOT " f"byte-identical ({line}). Left for a human; gate_main reverts its own aborts.") return 0, [] log(f"{tag}: {len(banked)} banked, main byte-identical — {line}") # COMMIT HERE, not at the end of the cycle (R42). A bisect can run for many levels, and # deferring left byte-proven functions sitting uncommitted in src/ the whole time — exactly # the window in which a blind revert by any other tool destroys them. The tree is verified # byte-identical on this line; that is the moment it is safe and the moment it is durable. commit_banks(tag, banked) return len(banked), banked if "COMPILE conflict" in out and len(slate) > 1 and depth < 4: mid = len(slate) // 2 log(f"{tag}: compile conflict with nothing to drop — bisecting {len(slate)} into " f"{mid}+{len(slate)-mid}") n1, b1 = gate_batch(f"{tag}a", slate[:mid], depth + 1) n2, b2 = gate_batch(f"{tag}b", slate[mid:], depth + 1) return n1 + n2, b1 + b2 tail = "\n ".join(out.splitlines()[-4:]) log(f"{tag}: nothing banked. gate_main's last words:\n {tail}") return 0, [] def commit_banks(tag, banked): """R42: commit the moment a batch is green. Named paths only — other lanes bank into this tree at the same time and a blanket `git add -A` would adopt their in-flight work as ours.""" if not banked: return OX.sh("git add src/*.c config/symbols.us.txt 2>/dev/null || true") # -F a file, never -m with a shell-quoted string: json.dumps() escapes newlines and em-dashes, # so the first version of this committed a message reading "main lane m00 \\u2014 11 banked\\n\\n…" # as ONE literal line. mp = f".run/main_commit_{tag}.txt" with open(mp, "w") as f: f.write(f"feat(decomp): main lane {tag} — {len(banked)} banked\n\n" f"One clean whole-EXE rebuild verified the batch (gate_main), and main re-checked\n" f"BYTE-IDENTICAL against config/check.us.sha before anything was credited.\n\n" + "\n".join(f" {f_}" for f_ in sorted(banked)[:60]) + ("\n …" if len(banked) > 60 else "") + "\n") OX.sh(f"git commit -q -F {mp} || true") log(f"{tag}: committed {len(banked)} bank(s)") def cycle(a, tag): """One draw→draft→filter→gate→commit pass. Returns the number banked.""" # R43 (S59): assert the arbiter BEFORE spending tokens or rebuilds. One clean no-draft # rebuild (~40 s, under the gate lock) is the cheapest insurance this lane can buy: it # converts "every batch mysteriously gates 0 and the ledger reads as a drafting failure" # into one loud refusal naming the actual defect. r = OX.sh(f"{PY} tools/gate_main.py --assert-baseline", timeout=7200) if r.returncode != 0: tail = ((r.stdout or '') + (r.stderr or '')).strip().splitlines() raise BaselineRed(tail[-1] if tail else f"gate_main --assert-baseline rc={r.returncode}") if os.path.exists(RED_SENTINEL): os.unlink(RED_SENTINEL) log("baseline is GREEN again — resuming normal cadence") parked = parked_slate() if not a.no_parked else [] if parked: log(f"{tag}: {len(parked)} PARKED draft(s) from earlier overlay waves — gating those first " f"(already drafted, never gated)") batch = parked[:a.batch] n, banked = gate_batch(tag, batch) # gate_batch commits each verified sub-batch (R42) failed = [x for x in batch if x["fn"] not in set(banked)] if failed: fp = ".run/main_queue_failed.json" try: prev = json.load(open(fp)) except Exception: prev = [] seen = {x.get("fn") for x in prev} bumped = {x["fn"]: int(x.get("tries", 0)) + 1 for x in failed} for row in prev: # count a repeat failure if row.get("fn") in bumped: row["tries"] = bumped.pop(row["fn"]) prev += [{"fn": x["fn"], "draft": x["draft"], "tag": tag, "t": time.time(), "tries": bumped[x["fn"]]} for x in failed if x["fn"] in bumped and x["fn"] not in seen] json.dump(prev, open(fp, "w"), indent=1) log(f"{tag}: {len(failed)} draft(s) did not bank -> parked in {fp} with their tag " f"(recoverable; a failed draft is evidence, not garbage)") for r in {x["src"] for x in batch}: # consume the queue file once its rows are gated done = {x["fn"] for x in batch} try: rows = [x for x in json.load(open(r)) if (x.get("fn") or x.get("name")) not in done] json.dump(rows, open(r, "w"), indent=1) except Exception: pass return n cards = draw(tag, a.cards, a.min_ins, a.max_ins) if not cards: return 0 targets = OX.shard_targets(tag, cards, a.workers) if not targets: log(f"{tag}: no valid targets"); return 0 log(f"{tag}: {len(targets)} main card(s) -> drafting with {a.workers} worker cap") procs = OX.draft(tag, OX.parse_lanes(a.models, a.model, a.workers), a.maxtok, a.max_turns, wave_lane="main") drafts, trunc = OX.collect_drafts(tag, procs) # (paths, truncated-turn count) log(f"{tag}: {len(drafts)} draft(s) produced ({trunc} truncated)") keep, counts = OX.reloc_filter(tag, drafts, cards) log(f"{tag}: reloc_identity {counts} -> slate {len(keep)}") slate = [{"fn": k["fn"], "draft": k["draft"]} for k in keep][:a.batch] n, banked = gate_batch(tag, slate) # gate_batch commits each verified sub-batch (R42) with open(LEDGER, "a") as f: f.write(json.dumps({"t": time.time(), "tag": tag, "cards": len(targets), "drafts": len(drafts), "slate": len(slate), "banked": n, "reloc": counts}) + "\n") return n def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--cards", type=int, default=200) ap.add_argument("--workers", type=int, default=150, help="worker cap for THIS lane (default 150). The overlay lane runs ~313 agents " "at ~190 req/min against a measured 429 knee of 200-230; this lane is " "sized to use the headroom, not to compete for it.") ap.add_argument("--batch", type=int, default=40, help="drafts per gate_main invocation (default 40). One clean rebuild verifies " "the whole slate; a failure bisects, so a bigger batch is cheaper only " "while the reloc pre-filter keeps the failure rate low.") ap.add_argument("--min-ins", type=int, default=5) ap.add_argument("--max-ins", type=int, default=200) ap.add_argument("--models", default="stealth/ox-alpha:150") ap.add_argument("--model", default="stealth/ox-alpha") ap.add_argument("--maxtok", type=int, default=int(os.environ.get("MAXTOK", "8000"))) ap.add_argument("--max-turns", type=int, default=24) ap.add_argument("--no-parked", action="store_true", help="skip the .run/main_queue backlog") ap.add_argument("--once", action="store_true") a = ap.parse_args() os.chdir(REPO) while not os.path.exists(".run/ox_campaign.stop"): tag = next_tag() if not tag: log("out of main-lane tags"); return try: cycle(a, tag) except BaselineRed as e: log(f"{tag}: BASELINE RED — {e}") log(f"{tag}: REFUSING to draft or gate against a broken baseline (R43): every " f"verdict would be false and every rebuild wasted. Re-checking in 30 min; " f"fix the committed baseline (git log -- 'src/*.c') to resume.") with open(RED_SENTINEL, 'w') as f: f.write(time.strftime('%F %T') + '\n' + str(e) + '\n') if a.once: return time.sleep(1800) continue except Exception as e: # a lane must survive its own bad cycle log(f"{tag}: cycle failed: {type(e).__name__}: {e}") if a.once: return time.sleep(20) if __name__ == "__main__": main()