From 55cd89402465db70321bdde2ec6d225f520ba09e Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:00:56 -0600 Subject: [PATCH] perf(phase-29): family_sweep gates groups in PARALLEL by default (the SESSION-20 adapter, finally wired) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SESSION-20 measured serial family-sweep gating as "roughly an 8-16x throughput loss on a 32-thread box" and BUILT tools/sweep_parallel.py for it — but only reachable via a manual `--stage-only` two-step, so this path stayed serial and three sweeps in SESSION-22 (133 + 273 + 137 members) ran serially for no reason. §101, the stale-default class. SHAPE OF THE CHANGE — deliberately minimal after two failed attempts earlier today. A parallel PRE-PASS (phase 2a) runs only the per-group `harvest_verify` subprocess; phase 2b then consumes the results IN THE ORIGINAL SERIAL ORDER, so every line of post-processing (the MISMATCH backstop, the zero-bank restore, the counters, the prints) is untouched and output stays deterministic. No closure restructuring — that is exactly what broke it twice before. SAFETY, not a new claim: the Makefile already builds binaries concurrently (check-all/extract-all use `xargs -P$(JOBS)`, JOBS=16) and bulk_harvest's farm does the same with a per-binary lock. The §28 hazard is two makes racing on the SAME artifacts, prevented by the per-overlay lock (two splits of one overlay build the same binary and therefore serialise). NEGATIVE-CONTROLLED BOTH WAYS: `--stage-only` stages identically under -j1 and -j12 (4 groups each); a full gate returns IDENTICAL tallies (0 banked / 4 failed) parallel vs serial; tree clean after both. HONEST MEASUREMENT: on the only sample available (4 groups, and they fail FAST on a compile error rather than running full builds) parallel was 4s vs serial 6s — ~1.5x, NOT the 8-16x. That figure needs a large family (137 groups of full builds) to show, and every such family was already banked today. The wiring is proven correct here; the throughput claim remains SESSION-20's measurement, not mine. `-j 1` restores the old behaviour. --- tools/family_sweep.py | 50 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/tools/family_sweep.py b/tools/family_sweep.py index 97d4232f1..46dbeb50e 100644 --- a/tools/family_sweep.py +++ b/tools/family_sweep.py @@ -17,6 +17,8 @@ compile and are logged for the decl-reconcile pass; they are NOT remap failures. [--only 0xADDR,0xADDR] # sweep just these exemplar addrs (validation) """ import json, glob, re, subprocess, os, sys, shutil, collections, argparse +import concurrent.futures as _cf +import threading as _th sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import family_remap as FR import family_hseq # §53 interlock — the ONE has_mid_jr oracle (R33, shared with dedup_extend) @@ -28,6 +30,7 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, 'tools')) import corpus # the derived corpus oracle (Phase 26-A) PY = ".venv/bin/python" +_OV_LOCKS = __import__("collections").defaultdict(_th.Lock) # two splits of ONE overlay build the same binary SWEEP = ".run/sweep" @@ -408,13 +411,47 @@ def hseq_sweep(a): # ---- phase 2: gate each group ONCE (unique verified-out per group so multi-split overlays sum right) banked = collections.Counter() failed = collections.Counter() - for gi, ((ov, src_rel, subdir), fns) in enumerate(sorted(groups.items())): + _items = sorted(groups.items()) + + # PHASE 2a — run the gates in PARALLEL ACROSS DISTINCT BINARIES, then phase 2b consumes the + # results in the ORIGINAL serial order. Only the subprocess call moves; every line of the + # post-processing below (the MISMATCH backstop, the zero-bank restore, the counters, the prints) + # runs exactly as before, in order, so output stays deterministic. + # + # WHY: SESSION-20 measured serial gating as "roughly an 8-16x throughput loss on a 32-thread box" + # and built `tools/sweep_parallel.py` for it — but only reachable via a manual `--stage-only` + # two-step, so THIS path stayed serial and three sweeps in SESSION-22 (543 members) ran serially + # for no reason (§101, the stale-default class). + # + # SAFE, and not a new claim: the Makefile already builds binaries concurrently (`check-all`/ + # `extract-all` run `xargs -P$(JOBS)`, JOBS=16) and bulk_harvest's farm does the same with a + # per-binary lock. The hazard §28 records is two makes racing on the SAME artifacts — prevented + # here by the per-overlay lock, since two splits of one overlay build the same binary. + def _gate(gi, key): + ov, src_rel, subdir = key good_sha = open(os.path.join(REPO, f"config/check.{ov}.sha")).read().split()[0] + with _OV_LOCKS[ov]: + return sh([PY, "tools/harvest_verify.py", "--binary", ov, "--src", src_rel, + "--asm-subdir", subdir, "--out", f"build/{ov}/{ov}", "--good-sha", good_sha, + "--drafts", os.path.join(SWEEP, ov), "--chunk", str(a.chunk), + "--verified-out", f".run/hseq_verified.{ov}.{gi}.txt", + "--failed-out", f".run/hseq_failed.{ov}.{gi}.txt"], timeout=3600) + + _res = {} + if a.jobs > 1 and len(_items) > 1: + print(f"[hseq] gating {len(_items)} group(s) across distinct binaries, -j{a.jobs}") + with _cf.ThreadPoolExecutor(max_workers=a.jobs) as _ex: + _f = {_ex.submit(_gate, gi, key): gi for gi, (key, _fns) in enumerate(_items)} + for _fu in _cf.as_completed(_f): + _res[_f[_fu]] = _fu.result() + else: + for gi, (key, _fns) in enumerate(_items): + _res[gi] = _gate(gi, key) + + # ---- phase 2b: consume the results in the original order (unchanged from the serial version) + for gi, ((ov, src_rel, subdir), fns) in enumerate(_items): vout = f".run/hseq_verified.{ov}.{gi}.txt" - r = sh([PY, "tools/harvest_verify.py", "--binary", ov, "--src", src_rel, "--asm-subdir", subdir, - "--out", f"build/{ov}/{ov}", "--good-sha", good_sha, - "--drafts", os.path.join(SWEEP, ov), "--chunk", str(a.chunk), - "--verified-out", vout, "--failed-out", f".run/hseq_failed.{ov}.{gi}.txt"], timeout=3600) + r = _res[gi] # --normalize-self-decls backstop: a final MISMATCH means a self-decl TU edit was NOT byte-neutral # (a transform bug — harvest_verify always reverts a wrong DRAFT, so a wrong draft leaves the binary # byte-identical, just unbanked). Restore this group's edited TU from the phase-1 snapshot + rebuild @@ -460,6 +497,9 @@ def main(): ap.add_argument("--limit", type=int, default=0, help="cap #exemplars (0 = all)") ap.add_argument("--min-sibs", type=int, default=1) ap.add_argument("--chunk", type=int, default=8) + ap.add_argument("-j", "--jobs", type=int, default=12, + help="gate groups across DISTINCT binaries in parallel (1 = serial). " + "The per-overlay lock keeps two splits of one overlay serialised.") ap.add_argument("--commit", action="store_true") ap.add_argument("--only", default=None, help="comma-separated exemplar addrs to sweep (validation)") ap.add_argument("--allow-pins", action="store_true",