From 84706c9860aea6d682c25fb97a3faa3346f5eb3c Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Tue, 25 Aug 2026 16:18:43 -0600 Subject: [PATCH] feat(draw): draft EVERY instance, siblings included (ONE_PER_GID=0) + K attempts per card MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two ways to spend a free drafting window on a tail that is 92% walls. ONE_PER_GID=0 — the sibling collapse exists because a same-gid sibling banks by mechanical remap once its exemplar cracks, so drafting it pays for what the remap does free. That prices AGENT TOKENS as the scarce resource. On the free ox window they are not, and the collapse is what makes 3,271 open crackable functions look like 334 drawable skeletons — of which 308 are gen6+ walls whose exemplars have already refused six waves each. A sibling drafted directly can crack on its OWN terms instead of waiting on an exemplar that never will. Measured on a live draw rather than argued: uncollapsed 627 cards / 44,403 ins / 160 binaries / 215 gate groups = 2.9 drafts per rebuild collapsed 334 cards / 30,926 ins / 104 binaries / 127 gate groups = 2.6 drafts per rebuild The gate cost is per (binary, TU) group and chunked, so siblings landing in binaries the wave already touches are close to free at the gate — the card count nearly doubles and the gate gets MORE efficient per build, not less. ATTEMPTS=K — K independent shots at each card. The gate cost does not multiply: reloc_filter keys by fn and staging writes /.c, so a function still gets exactly one whole-binary build per wave; the attempts compete to BE that build, ranked by match_one, which is local and needs no build. Alternates stay on disk for a later recovery pass. Default 1 (no-op). A BUG I CAUGHT IN MY OWN SELECTOR before it shipped: it passed binof[fn] (a BINARY NAME) where match_one wants --asm-subdir (an asm DIRECTORY). Every attempt would have scored identically at infinity and the picker would have silently degraded to first-seen while appearing to rank — the same "true number about the wrong thing" class as the day's other defects. Fixed with a subof map, and a missing subdir now returns neutral instead of a fake score. --- tools/lanes/drafter.sh | 13 +++++++++- tools/ox_campaign.py | 56 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/tools/lanes/drafter.sh b/tools/lanes/drafter.sh index c81f24a380..e60f5b091e 100644 --- a/tools/lanes/drafter.sh +++ b/tools/lanes/drafter.sh @@ -110,7 +110,18 @@ export TAIL_DONE_FRAC=0.75 # took 98 MINUTES of gate to do it, while four drafted waves queued behind it and the free-ox # fleet sat at 14 agents / 10 req/min. At this conversion a 150-card wave banks what a # 430-card wave banks, in a third of the gate. Raise it again when the tail starts converting. -export MAX_BINS=50 +# ONE_PER_GID=0 + MAX_BINS 250 (P31 S60, Drew: "draft all remaining funcs, sib family be damned"). +# The sibling collapse was priced when agent tokens were scarce: a same-gid sibling banks by +# mechanical remap once its exemplar cracks, so drafting it pays for what the remap does free. On a +# free ox window that reasoning inverts — and the collapse is what makes 3,271 open crackable +# functions look like 334 drawable skeletons, 92% of which are gen6+ walls whose exemplars have +# already refused six waves. A sibling drafted directly can crack on its OWN terms. +# Measured on a live draw: 627 cards / 44,403 ins / 215 gate groups uncollapsed, vs 334 / 30,926 / +# 127 collapsed — and the gate got MORE efficient per build, 2.9 drafts per rebuild vs 2.6, because +# siblings land in binaries the wave already touches and the gate is per (binary, TU), chunked. +# MAX_BINS goes back up because the whole pool is now the target; revert both if the gate backs up. +export ONE_PER_GID=0 +export MAX_BINS=250 export MAX_429=10 while [ ! -e .run/ox_campaign.stop ]; do .venv/bin/python tools/ox_campaign.py --drafter \ diff --git a/tools/ox_campaign.py b/tools/ox_campaign.py index 6a8335323f..7c165f2d25 100644 --- a/tools/ox_campaign.py +++ b/tools/ox_campaign.py @@ -186,8 +186,18 @@ def draw_wave(tag, n, band, levers=None): # 224 gate groups. req/min is agents-in-flight x ~0.8, so the draw was setting the campaign's # throughput. Raise the cap to take essentially the whole pool; MAX_BINS overrides it. bins = os.environ.get("MAX_BINS", "160") + # ONE_PER_GID=0 — DRAFT EVERY INSTANCE, SIBLINGS INCLUDED (P31 S60, Drew). The collapse exists + # because a same-gid sibling banks by mechanical remap once its exemplar cracks, so drafting it + # is paying for what the remap does free. That reasoning prices AGENT TOKENS as scarce; on the + # free ox window they are not, and the collapse is what makes 3,652 open functions look like 334 + # drawable skeletons — 92% of which are gen6+ walls the remap will never be unlocked by. + # A sibling drafted directly can crack on its OWN terms and bank without waiting for an exemplar + # that has already refused six waves. The gate cost does not scale with functions: it is per + # (binary, TU) group, chunked, so siblings landing in binaries the wave already touches are + # close to free at the gate — which is exactly why this is worth trying rather than assuming. + gid = "" if os.environ.get("ONE_PER_GID") == "0" else " --one-per-gid" r = sh(f"{PY} tools/build_wave_atlas.py {cards} {n} --min-ins {lo} --max-ins {hi} " - f"--max-bins {bins} --one-per-gid --retry-unbanked --exclude-bins main{lv}", + f"--max-bins {bins}{gid} --retry-unbanked --exclude-bins main{lv}", timeout=3600, quiet=False) if not os.path.exists(cards): log(f" wave {tag}: DRAW FAILED — {(r.stderr or r.stdout)[-300:]}") @@ -205,6 +215,17 @@ def shard_targets(tag, cards_path, workers): "sub": c["sub"], "asm": f"{c['sub']}/{c['fn']}.s", "tu": c.get("tu"), "ghidra_c": f".run/ghidra_c/{c['fn']}.c"} for c in cards] targets = [t for t in targets if os.path.isfile(t["asm"])] # R32: assert, do not assume + # ATTEMPTS: K INDEPENDENT SHOTS PER CARD (P31 S60, Drew). Drafting is free on the ox window and + # the drawable pool is now 92% gen6+ walls — functions that refused five waves each. One more + # sample of the same wall converts at ~5%; K independent samples of it convert K times as often + # for the same zero token cost. The GATE cost does not multiply: reloc_filter keys by fn and + # staging writes /.c, so a function still consumes exactly one whole-binary build + # per wave — the extra attempts compete to BE that build (see pick_attempt in reloc_filter). + attempts = max(1, int(os.environ.get("ATTEMPTS", "1"))) + if attempts > 1: + targets = [dict(t, _attempt=k) for t in targets for k in range(attempts)] + log(f" ATTEMPTS={attempts}: {len(targets)} shard target(s) over " + f"{len(targets)//attempts} card(s)") for i in range(workers): json.dump(targets[i::workers], open(f".run/wave_{tag}_targets.{i}.json", "w"), indent=1) return targets @@ -425,6 +446,7 @@ def reloc_filter(tag, drafts, cards_path): cards = json.load(open(cards_path)) cards = cards if isinstance(cards, list) else cards.get("cards", []) binof = {c["fn"]: c["binary"] for c in cards} + subof = {c["fn"]: c.get("sub", "") for c in cards} # asm subdir — match_one's --asm-subdir batch = [] for d in drafts: fn = os.path.basename(d)[:-2] @@ -449,6 +471,38 @@ def reloc_filter(tag, drafts, cards_path): # 697 "gated" and banked 0. Only AGREE is a pass. (R43: refuse input the step cannot use.) keep = [b for b in batch if status.get(b["fn"]) == "AGREE"] + # PICK THE BEST ATTEMPT (P31 S60). With ATTEMPTS>1 the same function arrives several times from + # different agents. Staging would keep whichever landed last — an arbitrary choice of the one + # draft that gets this function's single whole-binary build. match_one is LOCAL (it compiles the + # one function and diffs it; no binary build, no lock), so it can rank the attempts for free and + # give the build to the best one. Ties and errors fall back to first-seen, and every alternate + # stays on disk in .run/wave_/shard*/ for a later recovery pass — nothing is discarded. + _by_fn = collections.defaultdict(list) + for b in keep: + _by_fn[(b["binary"], b["fn"])].append(b) + _dupes = {k: v for k, v in _by_fn.items() if len(v) > 1} + if _dupes: + def _score(b): + try: + sd = subof.get(b['fn']) + if not sd: + return 1 << 30 # no asm subdir on the card -> cannot rank, stay neutral + r = sh(f"{PY} tools/match_one.py {b['fn']} --c {b['draft']} " + f"--asm-subdir {sd} --json", timeout=300) + j = json.loads((r.stdout or "{}").strip().splitlines()[-1]) + return 0 if j.get("match") else int(j.get("diff", 1 << 30)) + except Exception: + return 1 << 30 + picked, dropped = [], 0 + for k, v in _by_fn.items(): + if len(v) == 1: + picked.append(v[0]); continue + best = min(v, key=_score) + picked.append(best); dropped += len(v) - 1 + keep = picked + log(f" attempts: {len(_dupes)} function(s) had multiple surviving drafts; " + f"match_one picked the best of each, {dropped} alternate(s) left on disk") + # CAPTURE THE PRE-FILTER REJECTS (P31 S59). Everything that reaches the GATE and fails gets a # backlog row with its closeness, class and best draft (gate_stage) — but a draft the reloc # pre-filter drops never reaches the gate, so it was recorded NOWHERE and simply sat on disk.