diff --git a/tools/build_wave_atlas.py b/tools/build_wave_atlas.py new file mode 100644 index 000000000..4e02f8899 --- /dev/null +++ b/tools/build_wave_atlas.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Build a campaign wave from the FRONTIER ATLAS, optimized for gate throughput. + +Why this exists (P31, 2026-08-14): the pre-baked adapt/weak card piles are the smallest, +best-seeded tail (12-42 ins). Drawing from them banks ~1,440 ins/wave against 635,744 open — +0.011pp of fleet per wave. The atlas knows where the mass actually is (cousin-multi 294k ins, +cold 183k, main-only 38k) and what lever each group needs. + +TWO selection principles, both measured: + 1. GATE COST SCALES WITH (binary, TU) GROUPS, NOT DRAFTS. Each group is a whole-binary rebuild. + Wave C was 35 drafts over 27 groups = 1.3 drafts/rebuild, ~50 min of gate for 32 banks. + So: CONCENTRATE the wave in few binaries. This is free throughput. + 2. MASS BEATS COUNT for the instruction-weighted metric — prefer bigger functions where a seed + exists, but keep them inside the model ladder's competence. + +Selection: open stubs (derived from corpus, R32/R33), not spent in a prior wave, from atlas +groups whose lever is agent-draftable; ranked by binary concentration then instruction mass. + +MUST NOT run while a gate is in flight (R35 — corpus.stubs() misreports substituted drafts). + +Usage: build_wave_atlas.py [N] [--max-bins K] [--min-ins M] [--levers a,b,c] +""" +import json, sys, collections, subprocess, argparse +sys.path.insert(0, 'tools') +import corpus + +ap = argparse.ArgumentParser() +ap.add_argument('out') +ap.add_argument('n', nargs='?', type=int, default=96) +ap.add_argument('--max-bins', type=int, default=12, help='concentrate the wave in this many binaries') +ap.add_argument('--min-ins', type=int, default=0) +ap.add_argument('--max-ins', type=int, default=120, help='above this the bulk ladder stops being honest') +ap.add_argument('--levers', default='head-crack,seeded-crack,redraft,len-vein,integration,family-sweep,tiny-direct', + help='agent-draftable levers; UNKNOWN/tell/jtbl/o0/cc1 need their own lanes') +ap.add_argument('--atlas', default='.run/atlas.json') +a = ap.parse_args() + +busy = subprocess.run(['pgrep', '-f', 'tools/(gate_stage|dedup_propagate|gate_lane)'], + capture_output=True, text=True) +if busy.returncode == 0 and busy.stdout.strip(): + sys.exit(f"REFUSING: gate in flight (pids {busy.stdout.split()}) — corpus.stubs() would misreport (R35).") + +PRIORS = ['.run/wave_p31%s_cards.json' % c for c in 'abcdefghijkl'] +taken = set() +for p in PRIORS: + try: + taken |= {c.get('fn') or c.get('name') for c in json.load(open(p))} + except (FileNotFoundError, json.JSONDecodeError): + pass + +levers = set(a.levers.split(',')) +atlas = json.load(open(a.atlas)) + +_open = {} +def is_open(binary, fn): + if binary not in _open: + # corpus.stubs() is addr -> Stub; the NAME lives on the record + _open[binary] = {st.symbol for st in corpus.stubs(binary).values()} + return fn in _open[binary] + +def model_for(nins): + if nins <= 50: return 'haiku' + if nins <= 120: return 'sonnet' + return 'opus' + +cands, skipped = [], collections.Counter() +for g in atlas['groups']: + if g['lever'] not in levers: + skipped['lever-not-in-lane'] += g['inst']; continue + ex = g.get('exemplar') or {} + seed = (g.get('seed') or {}).get('norm') or (g.get('seed') or {}).get('raw') or {} + for m in g.get('members', []): + fn, b, nins = m.get('name'), m.get('b'), m.get('nins') or 0 + if not fn or not b: skipped['no-name'] += 1; continue + if fn in taken: skipped['already-waved'] += 1; continue + if not (a.min_ins <= nins <= a.max_ins): skipped['out-of-band'] += 1; continue + if not is_open(b, fn): skipped['already-banked'] += 1; continue + sub = corpus.asm_path(b, fn) + if not sub: skipped['no-asm'] += 1; continue + cands.append({ + 'fn': fn, 'binary': b, 'lane': 'mass', 'model': model_for(nins), 'nins': nins, + 'addr': m.get('a'), 'sub': __import__('os').path.dirname(sub), + 'gid': g['gid'], 'lever': g['lever'], 'confidence': g.get('confidence'), + 'lever_alts': g.get('lever_alts', []), + 'exemplar': {'binary': ex.get('b'), 'fn': ex.get('name'), 'nins': ex.get('nins')}, + 'seed_sim': seed.get('sim'), + }) + +# principle 1: concentrate. rank binaries by how much draftable mass they hold, take the top K. +by_bin = collections.defaultdict(list) +for c in cands: + by_bin[c['binary']].append(c) +ranked_bins = sorted(by_bin, key=lambda b: -sum(c['nins'] for c in by_bin[b]))[:a.max_bins] + +wave = [] +for b in ranked_bins: # principle 2: within a binary, mass first + for c in sorted(by_bin[b], key=lambda c: -c['nins']): + if len(wave) >= a.n: break + wave.append(c) + if len(wave) >= a.n: break + +json.dump(wave, open(a.out, 'w'), indent=1) +tot = sum(c['nins'] for c in wave) +print(f"candidates {len(cands)} in {len(by_bin)} binaries (skipped {dict(skipped)})") +print(f"-> wave {len(wave)} drafts / {tot} ins across {len({c['binary'] for c in wave})} binaries " + f"= {len(wave)/max(len({c['binary'] for c in wave}),1):.1f} drafts per gate group") +if wave: + print("models:", dict(collections.Counter(c['model'] for c in wave))) + print("levers:", dict(collections.Counter(c['lever'] for c in wave))) + print("nins: %d-%d (avg %.0f)" % (min(c['nins'] for c in wave), max(c['nins'] for c in wave), tot/len(wave))) diff --git a/tools/gate_lane.py b/tools/gate_lane.py index 8b9171aab..66b6e564a 100644 --- a/tools/gate_lane.py +++ b/tools/gate_lane.py @@ -21,14 +21,22 @@ def dirty(): return subprocess.run("git status --porcelain -- src/ config/",shel def writers(): return subprocess.run("pgrep -af 'dedup_propagate|gate_stage|harvest_verify'", shell=True,capture_output=True,text=True).stdout if dirty(): sys.exit("TREE DIRTY at entry — resolve before gating:\n"+dirty()[:400]) + +# The stub's HOME .c, DERIVED from the corpus oracle (R33) rather than re-globbed. The old +# `glob("src//*.c")` was structurally blind to `main`, whose sources live at src/*.c — +# every main draft grouped under src=None (R36: a consumer silently ignoring a real binary). +# corpus.stubs() already carries the answer on each record: Stub.path. +_HOME={} +def _home(binary, fn): + if binary not in _HOME: + _HOME[binary]={st.symbol: st.path for st in corpus.stubs(binary).values()} + return _HOME[binary].get(fn) + groups=collections.defaultdict(list) for c in cards: p=c.get('draft') or f"{wavedir}/{c['name']}/{c['name']}.c" # explicit path wins (agents use per-binary dirs) if not os.path.isfile(p): continue - src=None - for f in sorted(glob.glob(f"src/{c['binary']}/*.c")): - t=open(f).read() - if "INCLUDE_ASM" in t and c['name'] in t: src=f; break + src=_home(c['binary'], c['name']) groups[(c['binary'],src)].append((c['name'],p)) n_g=sum(len(v) for v in groups.values()) missing=[c['name'] for c in cards if not os.path.isfile(c.get('draft') or f"{wavedir}/{c['name']}/{c['name']}.c")]