diff --git a/.run/maintenance.sh b/.run/maintenance.sh index 77b30214d..7cd5e4eb8 100644 --- a/.run/maintenance.sh +++ b/.run/maintenance.sh @@ -163,23 +163,46 @@ PY say "nothing banked this pass" fi fi - # PERIODIC FLEET CHECK (P31 S59). Two binaries sat RED for hours — ov_SC07_010 from a commit whose - # tree state was never built, ov_SC07_002 from a stale 2-table jtbl pad spec — and NOTHING noticed, - # because every lane only ever checks the binary it is currently touching. A byte-gate is a - # correctness oracle with a null coverage model: it is silent about everything it did not build. - # So sweep the whole fleet on a slow cadence, report REDs loudly, and FIX NOTHING automatically — - # a wrong repair to a pad spec or a config is exactly how a silent byte shift gets committed. - # Every 4th pass (~3 h). Skipped while any gate is in flight: check-all rebuilds stale objects and - # must not race a gate's build for the same binary. - FC=$(cat .run/maint_fleet_count 2>/dev/null || echo 0); FC=$((FC+1)); echo "$FC" > .run/maint_fleet_count - if [ $((FC % 4)) -eq 0 ] && ! pgrep -f 'tools/sweep_parallel|tools/gate_stage|tools/gate_main' >/dev/null; then - say "fleet R22 sweep (every 4th pass) — this checks binaries no lane has touched" + # PERIODIC FLEET CHECK (P31 S59, cadence tightened S60). Two binaries sat RED for hours — every + # lane only ever checks the binary it is currently touching, and a byte gate is a correctness + # oracle with a null coverage model: it is silent about everything it did not build. So sweep the + # whole fleet and report REDs loudly. + # + # EVERY PASS, not every 4th (S60). Detection latency is the real cost of a RED: the binary fails + # at BUILD, so every draft gated against it is rejected regardless of quality, and the wave reads + # as a drafting failure. Five REDs in one day, each burning drafts until the ~3 h sweep noticed. + # Gates now finish in ~35 min instead of 60, so the check is affordable at every pass. + if ! pgrep -f 'tools/sweep_parallel|tools/gate_stage|tools/gate_main' >/dev/null; then + say "fleet R22 sweep — this checks binaries no lane has touched" make check-all JOBS=12 >.run/fleet_check.log 2>&1 || true grep -E "^\[FAIL\]" .run/check-all.txt 2>/dev/null | awk '{print $2}' > .run/fleet_red.txt || true NRED=$(grep -c . .run/fleet_red.txt 2>/dev/null || echo 0) if [ "$NRED" -gt 0 ]; then - say "*** $NRED BINARY/BINARIES ARE RED — see .run/fleet_red.txt (NOT auto-fixed, by design) ***" + say "*** $NRED BINARY/BINARIES ARE RED — see .run/fleet_red.txt ***" head -8 .run/fleet_red.txt | sed 's/^/ RED: /' + # THE ONE AUTO-REPAIR, AND ONLY BECAUSE IT PROVES ITSELF (S60). Everything else here still + # fixes NOTHING by design — a guessed pad spec or config edit is how a silent byte shift gets + # committed. jtbl_pads_fix is different in kind: it does not derive or guess, it ENUMERATES + # the 2^(N-1) candidate specs and accepts one only if it is the UNIQUE spec that rebuilds the + # binary byte-identical to config/check..sha, restoring the original otherwise. Three of + # the five REDs on 08-25 were this one class: JTBL_PADS stores a DERIVED property (how many + # jump tables an object emits) that every bank carrying a `switch` can change under it. + while read -r RB; do + [ -n "$RB" ] || continue + .venv/bin/python tools/jtbl_pads_fix.py "$RB" --apply 2>&1 | sed 's/^/ /' + done < .run/fleet_red.txt + if [ -n "$(git status --porcelain -- config/overlays.mk)" ]; then + make check-all JOBS=12 >.run/fleet_check.log 2>&1 || true + grep -E "^\[FAIL\]" .run/check-all.txt 2>/dev/null | awk '{print $2}' > .run/fleet_red2.txt || true + N2=$(grep -c . .run/fleet_red2.txt 2>/dev/null || echo 0) + if [ "$N2" -lt "$NRED" ]; then + git add config/overlays.mk + git commit -q -m "fix(jtbl): byte-proven pad-spec repair ($((NRED-N2)) binary/binaries) — maintenance lane" + say "pad-spec repair: $NRED RED -> $N2 RED, committed $(git rev-parse --short HEAD)" + else + say "pad-spec repair changed nothing measurable — left for a human" + fi + fi else say "fleet R22: all binaries byte-identical ($(grep -c '^\[ OK \]' .run/check-all.txt 2>/dev/null || echo 0) checked)" fi diff --git a/tools/jtbl_pads_fix.py b/tools/jtbl_pads_fix.py new file mode 100644 index 000000000..d18808f20 --- /dev/null +++ b/tools/jtbl_pads_fix.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""jtbl_pads_fix.py — repair a stale JTBL_PADS spec by SEARCH + BYTE PROOF, never by guessing. + +THE DEFECT CLASS THIS ENDS (P31 S60, three instances in one day). +`JTBL_PADS` is a per-object spec written by tools/jtbl_carve.py at CARVE time: one entry per +rodata `.align 3` the object emits, each 0 or 4, first always 0. It describes the object's jump +TABLE POPULATION — a property of the current source — and it is stored, not derived, so ANY change +to that population invalidates it and nothing notices: + + * ov_SC02_005, ov_SC07_006 — wave dd banked a function carrying a `switch`; the table count + changed under a spec that still described the old population. + * ov_SC04_018 — the identical symptom from the opposite cause: a maintenance pass REVERTED a + banked function and its table went with it. + +The build then dies with "consumed N rodata .align(s) but M pad spec(s) given", which is +jtbl_rodata_pads doing exactly the right thing (R43: refuse, never mishandle) — but the binary is +RED until a human looks, and a RED binary rejects every draft gated against it, so the cost is +paid in drafts that had nothing wrong with them. + +WHY NOT JUST DERIVE THE SPEC. The COUNT is derivable from the assembly stream; the VALUES are not. +A pad records where the ORIGINAL image has an inter-table `.align 3` pad, which depends on the +retail layout, not on anything in our source. Guessing wrong shifts every downstream data symbol — +a silent byte corruption, the worst possible outcome. So this tool does not derive: it ENUMERATES +the small candidate space (2^(N-1) specs: first entry 0, the rest in {0,4}) and lets the whole-binary +byte gate pick. A candidate is accepted only when it is the UNIQUE one that rebuilds the binary +byte-identical to config/check..sha. Zero matches or two matches => refuse and restore. + + tools/jtbl_pads_fix.py [--apply] [--max-tables 5] + +Exit 0 = repaired (or nothing to repair). Exit 1 = refused; a human should look. +""" +import argparse +import itertools +import os +import re +import subprocess +import sys + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +MK = os.path.join(REPO, "config", "overlays.mk") +PAD_ERR = re.compile(r"consumed (\d+) rodata \.align\(s\) but (\d+) pad spec\(s\) given") +OBJ_ERR = re.compile(r"\[(build/src/[^\]]+\.o)\]") + + +def sh(cmd, **kw): + return subprocess.run(cmd, cwd=REPO, shell=True, capture_output=True, text=True, **kw) + + +def good_sha(binary): + p = os.path.join(REPO, "config", f"check.{binary}.sha") + with open(p) as fh: + return fh.read().split()[0] + + +def armed_objects(binary): + """The objects that carry a JTBL_PADS spec for this binary (they alone run the pad filter).""" + out = [] + with open(MK) as fh: + for line in fh: + if line.startswith(f"build/src/{binary}/") and ": JTBL_PADS" in line: + out.append(line.split(":")[0].strip()) + return out + + +def build_sha(binary): + """Build under the SAME per-binary lock every gate takes; return the built sha1 or None. + + THE ARMED OBJECTS ARE DELETED FIRST (P31 S60, caught by this tool's own first test). JTBL_PADS + is a target-specific MAKE VARIABLE: changing it does not make the .o out of date, so an + incremental build silently keeps the object built with the PREVIOUS spec. The first run of this + tool reported "no pad-count drift" against a spec I had deliberately broken, and a candidate + search built on that would have scored every candidate identical — R22's incremental trap, + wearing a config costume. + """ + out = os.path.join(REPO, "build", binary, binary) + # DELETE THE OUTPUT TOO (P31 S60). A failed object build leaves the PREVIOUS binary sitting in + # build//, and `make build; sha1sum build//` then reports the OLD artifact's + # hash as if it were this build's — a false GREEN over a build that never linked. It read as a + # verified fix twice before the make exit code was checked. Same family as R49: an error inside + # something that looks like success. + for obj in armed_objects(binary) + [out]: + try: + os.remove(os.path.join(REPO, obj) if not os.path.isabs(obj) else obj) + except OSError: + pass + r = sh(f"flock .run/auto/gate.{binary}.lock make build BINARY={binary}") + if r.returncode != 0 or not os.path.exists(out): + return None, (r.stdout or "") + (r.stderr or "") + return sh(f"sha1sum build/{binary}/{binary}").stdout.split()[0], "" + + +def find_drift(binary): + """(object, emitted, declared) for the first object whose pad count drifted, else None.""" + _sha, log = build_sha(binary) + m = PAD_ERR.search(log) + if not m: + return None + obj = None + for line in log.splitlines(): + if "jtbl_rodata_pads" in line: + continue + o = OBJ_ERR.search(line) + if o: + obj = o.group(1) + return (obj, int(m.group(1)), int(m.group(2))) if obj else None + + +def pad_line_for(obj): + """(index, line, current pads) for the overlays.mk line that arms this object, else None.""" + with open(MK) as fh: + lines = fh.readlines() + for i, line in enumerate(lines): + if line.startswith(f"{obj}: JTBL_PADS"): + spec = line.split(":=", 1)[1].split("#")[0].strip() + return i, lines, [int(x) for x in spec.split(",") if x.strip()] + return None + + +def write_pads(idx, lines, pads, note): + lines[idx] = (f"{lines[idx].split(':')[0]}:{lines[idx].split(':')[1]}: JTBL_PADS := " + f"{','.join(str(p) for p in pads)} # {note}\n") + with open(MK, "w") as fh: + fh.writelines(lines) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("binary") + ap.add_argument("--apply", action="store_true", help="keep the proven spec (default: dry run)") + ap.add_argument("--max-tables", type=int, default=5, + help="refuse to search above this many tables (2^(N-1) candidates)") + a = ap.parse_args() + os.chdir(REPO) + + drift = find_drift(a.binary) + if not drift: + print(f"{a.binary}: no pad-count drift (nothing to repair)") + return 0 + obj, emitted, declared = drift + print(f"{a.binary}: {obj} emits {emitted} table(s), spec declares {declared}") + found = pad_line_for(obj) + if not found: + print(f"REFUSED: no JTBL_PADS line for {obj} in config/overlays.mk") + return 1 + idx, lines, original = found + if emitted > a.max_tables: + print(f"REFUSED: {emitted} tables is above --max-tables {a.max_tables}") + return 1 + + # THE CANDIDATE SPACE. pads[0] is always 0 (the first table starts the section, so it can carry + # no inter-table pad); every other entry is 0 or 4. 2^(N-1) candidates: 1, 2, 4, 8, 16. + cands = [[0] + list(rest) for rest in itertools.product((0, 4), repeat=max(0, emitted - 1))] + want = good_sha(a.binary) + winners = [] + for c in cands: + write_pads(idx, lines, c, "§8e pads — candidate under test (jtbl_pads_fix)") + got, _log = build_sha(a.binary) + mark = "BYTE-IDENTICAL" if got == want else (f"sha {got[:12]}" if got else "build failed") + print(f" {','.join(str(x) for x in c):<12} -> {mark}") + if got == want: + winners.append(c) + + if len(winners) != 1: + write_pads(idx, lines, original, "§8e pads (jtbl_carve.py) — restored, repair refused") + print(f"REFUSED: {len(winners)} candidate(s) rebuild byte-identical — a human should look. " + f"Original spec restored.") + return 1 + + win = winners[0] + if a.apply: + write_pads(idx, lines, win, f"§8e pads — {emitted} table(s), byte-proven by jtbl_pads_fix") + print(f"REPAIRED {obj}: {','.join(str(x) for x in original)} -> " + f"{','.join(str(x) for x in win)} (unique byte-identical spec)") + else: + write_pads(idx, lines, original, "§8e pads (jtbl_carve.py) — dry run, unchanged") + print(f"DRY RUN: {','.join(str(x) for x in win)} is the unique byte-identical spec. " + f"Re-run with --apply.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/lanes/maintenance.sh b/tools/lanes/maintenance.sh index 77b30214d..7cd5e4eb8 100644 --- a/tools/lanes/maintenance.sh +++ b/tools/lanes/maintenance.sh @@ -163,23 +163,46 @@ PY say "nothing banked this pass" fi fi - # PERIODIC FLEET CHECK (P31 S59). Two binaries sat RED for hours — ov_SC07_010 from a commit whose - # tree state was never built, ov_SC07_002 from a stale 2-table jtbl pad spec — and NOTHING noticed, - # because every lane only ever checks the binary it is currently touching. A byte-gate is a - # correctness oracle with a null coverage model: it is silent about everything it did not build. - # So sweep the whole fleet on a slow cadence, report REDs loudly, and FIX NOTHING automatically — - # a wrong repair to a pad spec or a config is exactly how a silent byte shift gets committed. - # Every 4th pass (~3 h). Skipped while any gate is in flight: check-all rebuilds stale objects and - # must not race a gate's build for the same binary. - FC=$(cat .run/maint_fleet_count 2>/dev/null || echo 0); FC=$((FC+1)); echo "$FC" > .run/maint_fleet_count - if [ $((FC % 4)) -eq 0 ] && ! pgrep -f 'tools/sweep_parallel|tools/gate_stage|tools/gate_main' >/dev/null; then - say "fleet R22 sweep (every 4th pass) — this checks binaries no lane has touched" + # PERIODIC FLEET CHECK (P31 S59, cadence tightened S60). Two binaries sat RED for hours — every + # lane only ever checks the binary it is currently touching, and a byte gate is a correctness + # oracle with a null coverage model: it is silent about everything it did not build. So sweep the + # whole fleet and report REDs loudly. + # + # EVERY PASS, not every 4th (S60). Detection latency is the real cost of a RED: the binary fails + # at BUILD, so every draft gated against it is rejected regardless of quality, and the wave reads + # as a drafting failure. Five REDs in one day, each burning drafts until the ~3 h sweep noticed. + # Gates now finish in ~35 min instead of 60, so the check is affordable at every pass. + if ! pgrep -f 'tools/sweep_parallel|tools/gate_stage|tools/gate_main' >/dev/null; then + say "fleet R22 sweep — this checks binaries no lane has touched" make check-all JOBS=12 >.run/fleet_check.log 2>&1 || true grep -E "^\[FAIL\]" .run/check-all.txt 2>/dev/null | awk '{print $2}' > .run/fleet_red.txt || true NRED=$(grep -c . .run/fleet_red.txt 2>/dev/null || echo 0) if [ "$NRED" -gt 0 ]; then - say "*** $NRED BINARY/BINARIES ARE RED — see .run/fleet_red.txt (NOT auto-fixed, by design) ***" + say "*** $NRED BINARY/BINARIES ARE RED — see .run/fleet_red.txt ***" head -8 .run/fleet_red.txt | sed 's/^/ RED: /' + # THE ONE AUTO-REPAIR, AND ONLY BECAUSE IT PROVES ITSELF (S60). Everything else here still + # fixes NOTHING by design — a guessed pad spec or config edit is how a silent byte shift gets + # committed. jtbl_pads_fix is different in kind: it does not derive or guess, it ENUMERATES + # the 2^(N-1) candidate specs and accepts one only if it is the UNIQUE spec that rebuilds the + # binary byte-identical to config/check..sha, restoring the original otherwise. Three of + # the five REDs on 08-25 were this one class: JTBL_PADS stores a DERIVED property (how many + # jump tables an object emits) that every bank carrying a `switch` can change under it. + while read -r RB; do + [ -n "$RB" ] || continue + .venv/bin/python tools/jtbl_pads_fix.py "$RB" --apply 2>&1 | sed 's/^/ /' + done < .run/fleet_red.txt + if [ -n "$(git status --porcelain -- config/overlays.mk)" ]; then + make check-all JOBS=12 >.run/fleet_check.log 2>&1 || true + grep -E "^\[FAIL\]" .run/check-all.txt 2>/dev/null | awk '{print $2}' > .run/fleet_red2.txt || true + N2=$(grep -c . .run/fleet_red2.txt 2>/dev/null || echo 0) + if [ "$N2" -lt "$NRED" ]; then + git add config/overlays.mk + git commit -q -m "fix(jtbl): byte-proven pad-spec repair ($((NRED-N2)) binary/binaries) — maintenance lane" + say "pad-spec repair: $NRED RED -> $N2 RED, committed $(git rev-parse --short HEAD)" + else + say "pad-spec repair changed nothing measurable — left for a human" + fi + fi else say "fleet R22: all binaries byte-identical ($(grep -c '^\[ OK \]' .run/check-all.txt 2>/dev/null || echo 0) checked)" fi