From 094dfebef4161c63db12cacd7b1882b3e2a64f31 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:49:43 -0600 Subject: [PATCH] =?UTF-8?q?fix(campaign):=20measure=20what=20the=20campaig?= =?UTF-8?q?n=20can=20see=20=E2=80=94=20banked-today=20from=20the=20INCLUDE?= =?UTF-8?q?=5FASM=20invariant,=20gate=5Fmin=20beside=20wall=5Fmin,=20a=20l?= =?UTF-8?q?ock-aware=20fleet=20R22=20window=20(P31=20S61=20T10+/S61-8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit campaign_status: 'today: N banked' summed '— N banked' commit subjects and missed every bank that rode in a chore/maint commit (S60: 2,185 reported vs 2,644 net stubs removed); it now derives the number from INCLUDE_ASM stub counts at last-commit-before-midnight / HEAD / working tree (R33), and alive() is anchored so pgrep no longer matches its own wrapper (every lane read ok with 0 processes). ox_campaign gater: the ledger's wall_min counts drafting + queue wait since the ready marker's t0; gate_min is the gate alone (the '30-67 min gates' picture was this conflation). maintenance lane: the fleet R22 sweep skipped whenever any gate was in flight, i.e. always (last real sweep 12:54 08-25); it now takes .run/auto/draw.lock and waits its turn, skipping only for gate_main. --- tools/campaign_status.py | 24 ++++++++++++++++++++---- tools/lanes/maintenance.sh | 14 +++++++++++--- tools/ox_campaign.py | 16 +++++++++++++++- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/tools/campaign_status.py b/tools/campaign_status.py index 2dce0d7d5..606f1e395 100644 --- a/tools/campaign_status.py +++ b/tools/campaign_status.py @@ -31,7 +31,9 @@ def sh(cmd): def alive(lane): - return bool(sh(f"pgrep -f 'bash .run/{lane}.sh' | head -1")) + """Anchored: `pgrep -f 'bash .run/x.sh'` matched THIS tool's own `bash -c "pgrep ..."` wrapper, so + every lane read `ok` with zero processes alive (S60 shutdown; verified S61 with a clean ps).""" + return bool(sh(f"pgrep -f '^bash \\.run/{lane}\\.sh' | head -1")) def agents_by_wave(): @@ -146,9 +148,23 @@ def main(): print(f" distill: {mined} wave(s) mined · ready batches: [{pend}]") # ---- totals ---------------------------------------------------------------------------------- - banked = sh("git log --since='00:00' --format='%s' | grep -oE '[—-] [0-9]{1,4} banked' " - "| awk '{s+=$2} END {print s+0}'") - print(f"\ntoday : {banked} banked · {sh('git log --since=00:00 --oneline | wc -l')} commits") + # BANKED TODAY IS DERIVED FROM THE INCLUDE_ASM INVARIANT, NOT FROM COMMIT SUBJECTS (S61, R33). + # The subject regex summed "— N banked" and missed every bank that rode in a chore/maint commit: + # the S60 A-prop pass put 357 banks in commit:2904 ("chore: ...") and the regex reported 2,185 + # for a day whose stub count fell by 2,644 net (frontier-analysis-s60 §1). Stubs are counted at + # the last commit before midnight, at HEAD, and in the working tree (uncommitted banks — R42 + # says commit them, the number says whether anyone did). Carves re-add stubs, so this is NET. + def _stubs(rev): + if rev == "tree": + return int(sh("grep -rc 'INCLUDE_ASM(' src/ --include='*.c' | awk -F: '{s+=$NF} END {print s+0}'") or 0) + return int(sh(f"git grep -c 'INCLUDE_ASM(' {rev} -- src/ | awk -F: '{{s+=$NF}} END {{print s+0}}'") or 0) + base = sh("git rev-list -1 --before='00:00' HEAD") + s_base, s_head, s_tree = _stubs(base), _stubs("HEAD"), _stubs("tree") + subj = sh("git log --since='00:00' --format='%s' | grep -oE '[—-] [0-9]{1,4} banked' " + "| awk '{s+=$2} END {print s+0}'") + print(f"\ntoday : {s_base - s_head} net stubs removed since {base[:9]} ({s_base} -> {s_head} committed" + f"{'' if s_tree == s_head else f', {s_head - s_tree:+d} more uncommitted in the tree'}) · " + f"{sh('git log --since=00:00 --oneline | wc -l')} commits · subject-regex sum {subj} (undercounts)") return 0 diff --git a/tools/lanes/maintenance.sh b/tools/lanes/maintenance.sh index 7cd5e4eb8..24f5aed29 100644 --- a/tools/lanes/maintenance.sh +++ b/tools/lanes/maintenance.sh @@ -172,9 +172,17 @@ PY # at BUILD, so every draft gated against it is rejected regardless of quality, and the wave reads # as a drafting failure. Five REDs in one day, each burning drafts until the ~3 h sweep noticed. # Gates now finish in ~35 min instead of 60, so the check is affordable at every pass. - if ! pgrep -f 'tools/sweep_parallel|tools/gate_stage|tools/gate_main' >/dev/null; then - say "fleet R22 sweep — this checks binaries no lane has touched" - make check-all JOBS=12 >.run/fleet_check.log 2>&1 || true + # LOCK-AWARE WINDOW (S61). The pgrep guard above this line's history ("skip while any gate is in + # flight") meant the sweep NEVER ran — the gater gates back-to-back, so one was always in flight + # (last real sweep 12:54 on 08-25, S60 open thread #4). The wave gater, the maintenance sweep and + # the resolver all hold .run/auto/draw.lock for exactly their gate window, so taking that lock IS + # the "no overlay gate in flight" condition, and blocking on it gives the sweep a turn instead of + # a skip. gate_main holds its own lock and is short (~1 min per batch): skip and retry next pass. + if pgrep -f 'tools/gate_main' >/dev/null; then + say "fleet R22 sweep skipped this pass: gate_main in flight (retry next pass)" + else + say "fleet R22 sweep — taking .run/auto/draw.lock (waits for the current gate), then checks every binary" + flock .run/auto/draw.lock make check-all JOBS=12 >.run/fleet_check.log 2>&1 || true grep -E "^\[FAIL\]" .run/check-all.txt 2>/dev/null | awk '{print $2}' > .run/fleet_red.txt || true NRED=$(grep -c . .run/fleet_red.txt 2>/dev/null || echo 0) if [ "$NRED" -gt 0 ]; then diff --git a/tools/ox_campaign.py b/tools/ox_campaign.py index 19fbd55e6..f22953e44 100644 --- a/tools/ox_campaign.py +++ b/tools/ox_campaign.py @@ -905,6 +905,7 @@ def run_gater(a): t0 = meta.get("t0", time.time()) drafts = sorted(glob.glob(f".run/wave_{tag}/shard*/*.c")) log(f"=== GATE {tag} · {len(drafts)} drafts ===") + gate_t0 = time.time() # the GATE's own clock; wall_min below also counts drafting + queue wait keep, counts = reloc_filter(tag, drafts, meta["cards"]) log(f" reloc_identity: {counts} -> gating {len(keep)}") lk = _drawlock() @@ -918,7 +919,11 @@ def run_gater(a): hl = (hr.stdout or "").strip().splitlines() log(f" harvest: {hl[-1] if hl else 'FAILED'}") rs = rate_slice(t0) + # wall_min = since the wave STARTED DRAFTING (t0 from the ready marker) — it was read as the gate's + # wall and produced the "30-67 min gates" picture (frontier-analysis-s60 §2.4; real gate walls were + # 12-31 min). gate_min is the gate alone: reloc pre-filter + sweep + commit + harvest. row = {"wave": tag, "t": t0, "wall_min": round((time.time() - t0) / 60, 1), + "gate_min": round((time.time() - gate_t0) / 60, 1), "workers": meta.get("workers"), "band": meta.get("band"), "lane": meta.get("lane"), "targets": meta.get("targets"), "drafts": len(drafts), "truncated_turns": meta.get("trunc"), "reloc": counts, "gated": len(keep), @@ -927,7 +932,8 @@ def run_gater(a): with open(LEDGER, "a") as fh: fh.write(json.dumps(row) + "\n") log(f" GATE {tag}: banked {n}/{len(keep)} gated of {len(drafts)} drafts · " - f"{rs['requests']} req · {rs['h429']} 429 · {row['wall_min']}min · {sha}") + f"{rs['requests']} req · {rs['h429']} 429 · {row['wall_min']}min since draft-start " + f"(gate {row['gate_min']}min) · {sha}") os.remove(ready[0]) log("gater: finished") @@ -1001,7 +1007,11 @@ def main(): n, banked = gate(tag, keep, a.gate_jobs) sha = commit(tag, n, banked) rs = rate_slice(t0) + # wall_min = since the wave STARTED DRAFTING (t0 from the ready marker) — it was read as the gate's + # wall and produced the "30-67 min gates" picture (frontier-analysis-s60 §2.4; real gate walls were + # 12-31 min). gate_min is the gate alone: reloc pre-filter + sweep + commit + harvest. row = {"wave": tag, "t": t0, "wall_min": round((time.time() - t0) / 60, 1), + "gate_min": round((time.time() - gate_t0) / 60, 1), "workers": 0, "targets": len(drafts), "drafts": len(drafts), "truncated_turns": None, "reloc": counts, "gated": len(keep), "banked": n, "commit": sha, "mode": "gate-only", "credit_left": credits_left(), **rs} @@ -1084,7 +1094,11 @@ def main(): n, banked = gate(tag, keep, a.gate_jobs) sha = commit(tag, n, banked) rs = rate_slice(t0) + # wall_min = since the wave STARTED DRAFTING (t0 from the ready marker) — it was read as the gate's + # wall and produced the "30-67 min gates" picture (frontier-analysis-s60 §2.4; real gate walls were + # 12-31 min). gate_min is the gate alone: reloc pre-filter + sweep + commit + harvest. row = {"wave": tag, "t": t0, "wall_min": round((time.time() - t0) / 60, 1), + "gate_min": round((time.time() - gate_t0) / 60, 1), "workers": workers, "targets": len(targets), "drafts": len(drafts), "truncated_turns": trunc, "reloc": counts, "gated": len(keep), "banked": n, "commit": sha, "credit_left": credits_left(),