mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-09-26 05:32:17 -04:00
Drew T
366 lines
20 KiB
Python
366 lines
20 KiB
Python
#!/usr/bin/env python3
|
||
"""grinder.py — Phase 21 token-free permuter grinder (the CPU-bound worker).
|
||
|
||
Pulls the closest near-misses from the backlog, runs decomp-permuter on each (the permuter
|
||
closes regalloc/scheduling gaps — exactly the residual the worker's drafts leave), banks the
|
||
true byte-matches through the shared gate_stage (the sole arbiter, G3/P9), and re-logs any
|
||
improved-but-still-near draft. LLM-FREE — runs unattended for days under auto_supervisor.sh,
|
||
alongside the token-heavy worker waves (CPU budget vs token budget).
|
||
|
||
backlog near-miss -> decomp-permuter (output-0-* = true byte-match) -> gate_stage -> bank x reach
|
||
no winner -> re-log (closeness may improve) and move on
|
||
|
||
SAFE EXIT: touch .run/auto/STOP (tools/auto_stop.sh) — finishes the current permute+gate, exits 0.
|
||
HEARTBEAT: .run/auto/grinder_heartbeat.json — {ts, state, current, banked, fleet_pct}.
|
||
|
||
Usage: grinder.py [--permute-secs 120] [-j 14] [--batch 10] [--max-nins 220]
|
||
[--max-closeness 30] [--attempts 2] [--idle-secs 90] [--once]
|
||
"""
|
||
import argparse, glob, json, os, shutil, subprocess, sys, time
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
import p16_permute, gate_stage, backlog, autopsy, corpus
|
||
|
||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
|
||
|
||
def permute_ils(pd, secs, j, cycles, fn, log):
|
||
"""Iterated-local-search permute: `cycles` timeboxed runs, each WARM-RESTARTED from the best
|
||
byte-waypoint of the previous one. Returns the winning output-0 path, or None.
|
||
|
||
WHY (Phase-24 §G, and the P30 T4 reason this replaced the single cold call): a COLD
|
||
`run_permuter` plateaus at the base score — the big drops come from FRESH restarts seeded at the
|
||
best waypoint, not from letting one plateaued run continue (proven on func_80148094: 72 -> 36
|
||
over ~8 restarts). The grinder had `permuter_ils.py` sitting beside it since Phase 24 and never
|
||
used it, so every grind was a cold search: it burned the full time box re-descending ground the
|
||
previous cycle had already covered.
|
||
|
||
`cycles=1` reproduces exactly the old cold behaviour, so the change is opt-out.
|
||
The winner is still only a CANDIDATE — the whole-binary byte-gate remains the sole arbiter
|
||
(G3/P9); intermediate waypoints can be semantically divergent (the permuter rewrites stores for
|
||
byte-proximity), which is precisely why a waypoint is never banked, only re-seeded."""
|
||
import permuter_ils
|
||
prev = None
|
||
for cyc in range(1, max(1, cycles) + 1):
|
||
win = p16_permute.run_permuter(pd, secs, j)
|
||
if win:
|
||
return win
|
||
bw = permuter_ils.best_waypoint(pd)
|
||
if bw is None:
|
||
log(f" {fn}: ILS cycle {cyc}/{cycles} — no waypoint yet")
|
||
continue
|
||
score, d = bw
|
||
if prev is not None and score >= prev:
|
||
log(f" {fn}: ILS cycle {cyc}/{cycles} — best {score} (no gain; stopping early)")
|
||
break
|
||
log(f" {fn}: ILS cycle {cyc}/{cycles} — best {score}, warm-restarting")
|
||
shutil.copy(os.path.join(d, "source.c"), f"{pd}/base.c")
|
||
prev = score
|
||
return None
|
||
AUTODIR = ".run/auto"
|
||
STOP = f"{AUTODIR}/STOP"
|
||
HB = f"{AUTODIR}/grinder_heartbeat.json"
|
||
DRAFTS = f"{AUTODIR}/grinder_drafts"
|
||
BLACKLIST = f"{AUTODIR}/grinder_blacklist.json" # fns the permuter wins but the byte-gate rejects = plumbing-bound; never re-permute
|
||
# A BLACKLIST ENTRY IS A VERDICT FROM A SPECIFIC GATE, AND IT EXPIRES WHEN THAT GATE CHANGES.
|
||
# Purged 2026-07-15 (Phase-28 T2). The 22 entries were recorded when harvest_verify was single-TU: it
|
||
# spliced only into `src/<bin>/<bin>.c`, so a SPLIT-hosted fn's stub literal was never found, the draft
|
||
# was never compiled, and the permuter's byte-match was discarded UNBUILT — which this file then recorded
|
||
# permanently as "plumbing-bound, never re-permute". 16 of 22 were split-hosted (docs/tooling-audit.md
|
||
# :931). The proof it was manufactured, not observed: **8 of the 22 have since MATCHED anyway**
|
||
# (func_80131D68/80149374/8014FE60/80150528/8016BBE0/80171C64/80174684/8017F290), and the other 14 would
|
||
# have been skipped forever on a verdict from a gate that no longer exists. harvest_verify now derives
|
||
# each stub's home TU from the corpus oracle and splices into whichever TU holds it (:122, render()), so
|
||
# the mechanism that manufactured these is gone.
|
||
# RULE (R35): when the gate changes, purge this file — a persisted negative verdict is only as good as the
|
||
# instrument that produced it. Re-derive from the fixed gate; never inherit.
|
||
|
||
|
||
def stop_requested():
|
||
return os.path.exists(os.path.join(REPO, STOP))
|
||
|
||
|
||
def load_blacklist():
|
||
try:
|
||
return set(json.load(open(os.path.join(REPO, BLACKLIST))))
|
||
except Exception:
|
||
return set()
|
||
|
||
|
||
def save_blacklist(bl):
|
||
json.dump(sorted(bl), open(os.path.join(REPO, BLACKLIST), "w"), indent=1)
|
||
|
||
|
||
def draft_sig(r):
|
||
"""the input SIGNATURE the permuter's yield depends on: (best_draft mtime, closeness). A fn
|
||
whose signature is unchanged since we last permuted it can NEVER newly win (the permuter is
|
||
deterministic given base.c + target.o) — so re-permuting it just burns CPU. Used by the
|
||
input-changed idle gate (T5), replacing the old blind tried.clear() that re-tried every
|
||
floor-victim on every idle tick (the R14-identified churn — Phase 24 CURRENT_PHASE)."""
|
||
p = r.get("best_draft")
|
||
try:
|
||
m = round(os.path.getmtime(os.path.join(REPO, p)), 3) if p else 0.0
|
||
except OSError:
|
||
m = 0.0
|
||
return (m, r.get("closeness"))
|
||
|
||
|
||
def log(m):
|
||
print(f"[{time.strftime('%H:%M:%S')}] grinder: {m}", flush=True)
|
||
|
||
|
||
def asm_subdir_for(binary, fn):
|
||
"""the asm subdir holding binary's <fn>.s (main or any _a/_o0/_jr split); None if absent.
|
||
|
||
DERIVED from corpus.asm_path (R33) instead of a second glob implementation. The glob was not
|
||
split-BLIND (the `*` matches every subdir — the Phase-22 'split-file-blind lookup' carry item is
|
||
STALE, verified P30 T4), but it was a parallel oracle that silently took g[0] when several
|
||
matched, and a parallel oracle is the defect class corpus.py exists to retire."""
|
||
p = corpus.asm_path(binary, fn)
|
||
return os.path.dirname(os.path.relpath(p, REPO)) if p and os.path.exists(p) else None
|
||
|
||
|
||
def heartbeat(state, current=None, banked=0, fp=None):
|
||
os.makedirs(os.path.join(REPO, AUTODIR), exist_ok=True)
|
||
json.dump({"ts": time.strftime("%Y-%m-%d %H:%M:%S"), "state": state, "current": current,
|
||
"banked": banked, "fleet_pct": fp},
|
||
open(os.path.join(REPO, HB), "w"), indent=1)
|
||
|
||
|
||
ATTEMPTED = f"{AUTODIR}/grinder_attempted.json" # fn -> [n_attempts, draft_sig] ACROSS runs
|
||
|
||
|
||
def load_attempted():
|
||
"""Attempts PERSISTED across `--once` invocations.
|
||
|
||
`tried` is in-process only, so every fresh `--once` run re-permuted the previous run's losers
|
||
from scratch — the permuter is deterministic given (base.c, target.o), so that CPU can never
|
||
produce a new win. Measured 2026-07-21: a 20-target probe drew 19 targets the immediately
|
||
preceding run had already tried and failed. Keyed by draft_sig so an IMPROVED draft legitimately
|
||
re-opens the function (same rule as the in-process input-changed gate, T5)."""
|
||
try:
|
||
return json.load(open(os.path.join(REPO, ATTEMPTED)))
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def save_attempted(d):
|
||
json.dump(d, open(os.path.join(REPO, ATTEMPTED), "w"), indent=1, sort_keys=True)
|
||
|
||
|
||
def candidates(max_nins, max_close, tried, attempts, blacklist, verdicts=None, stats=None,
|
||
profile=None, attempted=None):
|
||
"""closest still-open near-misses with a saved best draft (permuter-amenable), least-tried first.
|
||
|
||
TARGETING (Phase-29 Task-13A). "closest" is a scalar and a bad proxy for "a search-closer can
|
||
reach it". Measured over the whole open backlog through the whole-binary compile path: of the
|
||
972 records this filter admits, **75 (7.7%) are permuter-shaped**; 547 are STRUCTURAL (a
|
||
different load width, an extra instruction, a flipped branch — no local mutation introduces
|
||
those) and 348 are drafts that are not the function at all (a 15-instruction body against a
|
||
132-instruction target, whose "closeness 117" is pure length artefact).
|
||
|
||
So the daemon has been spending ~92% of its CPU where the permuter provably cannot win. That —
|
||
not a missing transform — is why it banked 7 functions all-time, all in Phase 21, and 0 since
|
||
(the Phase-22 audit). Filtering by the MEASURED residual class is the whole fix, and it is free.
|
||
|
||
Degrades to the previous undirected behaviour when the corpus has not been collected, and says
|
||
which mode it is in rather than filtering silently (R32)."""
|
||
out = []
|
||
for r in backlog.load_best():
|
||
nm = r.get("name")
|
||
if r.get("status") != "near" or not r.get("best_draft") or not nm:
|
||
continue
|
||
if nm in blacklist: # permuter-won-but-gate-rejected (plumbing) — never re-permute
|
||
continue
|
||
if verdicts:
|
||
v = verdicts.get(nm)
|
||
if v is None: # not in the corpus (logged after the last collect)
|
||
if stats is not None:
|
||
stats["unclassified"] += 1 # kept: unknown is not a reason to skip
|
||
elif v.get("bucket") != "permuter":
|
||
if stats is not None:
|
||
stats[v.get("bucket") or "?"] += 1
|
||
continue
|
||
elif profile and v.get("profile") != profile:
|
||
if stats is not None:
|
||
stats["profile!=%s" % profile] += 1
|
||
continue
|
||
if tried.get(nm, 0) >= attempts:
|
||
continue
|
||
if attempted is not None:
|
||
prev = attempted.get(nm)
|
||
if prev and prev[0] >= attempts and prev[1] == list(draft_sig(r)):
|
||
if stats is not None:
|
||
stats["already-tried (unchanged draft)"] += 1
|
||
continue
|
||
c = r.get("closeness")
|
||
if c is None or c > max_close: # permuter closes small regalloc/sched gaps, not large rewrites
|
||
continue
|
||
if r.get("nins") is not None and r["nins"] > max_nins: # known-too-big; None (off-077 manifest) = allow
|
||
continue
|
||
if not os.path.exists(os.path.join(REPO, r["best_draft"])):
|
||
continue
|
||
out.append(r)
|
||
out.sort(key=lambda r: (tried.get(r["name"], 0), r.get("closeness") or 999))
|
||
return out
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--permute-secs", type=int, default=120,
|
||
help="per-CYCLE time box (see --cycles)")
|
||
ap.add_argument("--cycles", type=int, default=4,
|
||
help="ILS warm-restart cycles per function (P30 T4). Each cycle re-seeds base.c "
|
||
"from the previous cycle's best byte-waypoint; stops early on no gain. "
|
||
"--cycles 1 = the old single cold search.")
|
||
ap.add_argument("-j", type=int, default=14)
|
||
ap.add_argument("--batch", type=int, default=10)
|
||
ap.add_argument("--max-nins", type=int, default=220)
|
||
ap.add_argument("--max-closeness", type=int, default=30)
|
||
ap.add_argument("--attempts", type=int, default=2)
|
||
ap.add_argument("--idle-secs", type=int, default=90)
|
||
ap.add_argument("--once", action="store_true")
|
||
ap.add_argument("--profile", default=None,
|
||
help="restrict to ONE measured permuter profile (regalloc|schedule|cse|length) "
|
||
"— for probing a single residual class's conversion rate")
|
||
ap.add_argument("--no-targeting", action="store_true",
|
||
help="ignore the measured residual classes and search undirected (the "
|
||
"pre-Task-13 behaviour; for A/B-ing the targeting filter)")
|
||
a = ap.parse_args()
|
||
os.chdir(REPO)
|
||
os.makedirs(AUTODIR, exist_ok=True)
|
||
if stop_requested():
|
||
log("STOP present at startup; remove it to run."); return
|
||
tried, banked, fp = {}, 0, None
|
||
last_sig = {} # fn -> draft_sig at last permute (input-changed gate, T5)
|
||
blacklist = load_blacklist()
|
||
# the MEASURED residual class per function (tools/autopsy.py collect). Empty = not collected;
|
||
# the daemon then runs undirected exactly as before, and says so.
|
||
verdicts = {} if a.no_targeting else autopsy.verdicts()
|
||
attempted = load_attempted()
|
||
log(f"start (permute={a.permute_secs}s -j{a.j} batch={a.batch} max_close={a.max_closeness}; "
|
||
f"blacklist={len(blacklist)} plumbing-bound fns skipped)")
|
||
log("targeting: %s" % (
|
||
f"ON — {len(verdicts)} classified; only bucket=permuter is admitted"
|
||
if verdicts else
|
||
"OFF — no .run/autopsy/residuals.jsonl (run tools/autopsy.py collect); "
|
||
"searching UNDIRECTED, which historically wastes ~92% of the CPU"))
|
||
while True:
|
||
if stop_requested():
|
||
log("STOP — clean exit."); heartbeat("stopped", None, banked, fp); return
|
||
import collections as _c
|
||
skipped = _c.Counter()
|
||
cand = candidates(a.max_nins, a.max_closeness, tried, a.attempts, blacklist,
|
||
verdicts, skipped, a.profile, attempted)[:a.batch]
|
||
if skipped:
|
||
log("targeting skipped %d non-permuter candidate(s): %s"
|
||
% (sum(skipped.values()), dict(skipped)))
|
||
if not cand:
|
||
if a.once:
|
||
log("no candidates (once) — exit."); heartbeat("dry", None, banked, fp); return
|
||
heartbeat("idle", None, banked, fp)
|
||
log(f"no untried candidates; idle {a.idle_secs}s (worker may add more)")
|
||
for _ in range(a.idle_secs):
|
||
if stop_requested():
|
||
break
|
||
time.sleep(1)
|
||
if stop_requested():
|
||
continue
|
||
# INPUT-CHANGED GATING (T5, replaces the old blind `tried.clear()`): re-open ONLY the
|
||
# fns whose input signature changed since we last permuted them — i.e. the worker
|
||
# improved the draft or its closeness. The permuter is deterministic given base.c +
|
||
# target.o, so re-permuting an UNCHANGED floor-victim can never newly win; the old
|
||
# clear() re-tried every one on every idle tick, burning CPU for zero banks (the
|
||
# R14-identified idle-churn). With the T2 floor-free scorer, a genuinely closeable fn
|
||
# wins on its FIRST pass; anything still open after `attempts` is input-bound until the
|
||
# worker feeds it a better draft.
|
||
cur = {r.get("name"): draft_sig(r) for r in backlog.load_best() if r.get("name")}
|
||
reopened = [fn for fn in list(tried) if cur.get(fn) != last_sig.get(fn)]
|
||
for fn in reopened:
|
||
tried.pop(fn, None); last_sig.pop(fn, None)
|
||
if reopened:
|
||
log(f"input-changed: re-opened {len(reopened)} fn(s) "
|
||
f"({', '.join(reopened[:6])}{'…' if len(reopened) > 6 else ''})")
|
||
else:
|
||
log("no inputs changed since last pass — staying idle (no churn)")
|
||
continue
|
||
if os.path.exists(os.path.join(REPO, DRAFTS)):
|
||
shutil.rmtree(os.path.join(REPO, DRAFTS))
|
||
os.makedirs(os.path.join(REPO, DRAFTS), exist_ok=True)
|
||
won = [] # (fn, binary): gate grouped by binary (harvest_verify filters)
|
||
for r in cand:
|
||
if stop_requested():
|
||
break
|
||
fn = r["name"]; tried[fn] = tried.get(fn, 0) + 1
|
||
last_sig[fn] = draft_sig(r)
|
||
attempted[fn] = [attempted.get(fn, [0, None])[0] + 1, list(draft_sig(r))]
|
||
save_attempted(attempted) # record the input we're about to permute (T5 gate)
|
||
binary = r.get("binary") or "ov_SC01_077" # legacy records: the canonical site (a 077-stub fn still gates)
|
||
asm_sub = asm_subdir_for(binary, fn)
|
||
heartbeat("permuting", fn, banked, fp)
|
||
try:
|
||
if not asm_sub:
|
||
log(f"{fn}: no .s under {binary} — skip"); continue
|
||
draft = open(os.path.join(REPO, r["best_draft"])).read()
|
||
# §31-directed mutation (T5): the wave-diagnosed residual class biases the pass
|
||
# weights toward that class's levers (permuter_weights). A generic/absent class
|
||
# -> the plain gcc defaults (unchanged undirected search).
|
||
# the MEASURED class beats the logged label: 91% of records carry no label at
|
||
# all, so permuter_weights.classify() returned None and the search ran on gcc
|
||
# defaults. residual_class names the class from today's bytes.
|
||
_v = verdicts.get(fn) or {}
|
||
pd = p16_permute.setup(fn, draft, asm_sub,
|
||
klass=_v.get("profile") or r.get("klass"),
|
||
where=r.get("where_stuck") or "")
|
||
if not pd:
|
||
continue
|
||
win = permute_ils(pd, a.permute_secs, a.j, a.cycles, fn, log)
|
||
if win:
|
||
_wtext = p16_permute.winner_to_draft(open(win).read())
|
||
open(os.path.join(REPO, DRAFTS, fn + ".c"), "w").write(_wtext)
|
||
# Task-12: durably persist the winning C BEFORE gate_stage banks/propagates/commits, so a
|
||
# mid-flight interrupt (kill or a revert of the uncommitted bank) never loses a hard-won
|
||
# permuter crack (this session's lesson: 3 wins lost to exactly that). permuter-winners/ is
|
||
# the recovery source — winner_to_draft(...) form, re-gateable any time.
|
||
_wdir = os.path.join(REPO, ".run", "permuter-winners"); os.makedirs(_wdir, exist_ok=True)
|
||
open(os.path.join(_wdir, fn + ".c"), "w").write(_wtext)
|
||
won.append((fn, binary)); log(f"permuter WON {fn} @ {binary} (close was {r.get('closeness')})")
|
||
except Exception as e:
|
||
log(f"{fn}: {e}")
|
||
if won:
|
||
heartbeat("gating", None, banked, fp)
|
||
import collections
|
||
by_bin = collections.defaultdict(list)
|
||
for fn, binary in won:
|
||
by_bin[binary].append(fn)
|
||
verified = set()
|
||
for binary, fns in sorted(by_bin.items()): # one gate per source binary; propagate stamps × reach
|
||
# §55b LAW: bank with --no-propagate, commit the cheap verified banks, and run
|
||
# ONE targeted `dedup_propagate --addr <banked>` afterwards as its own batch.
|
||
# gate_stage's propagate=True path is `dedup_propagate --auto-from`, the fleet-wide
|
||
# slow route that timed out at 3600s and left 90/140 overlays broken (887 files) —
|
||
# and because it runs INSIDE the gate, a propagate failure takes the banks with it.
|
||
# The grinder is the unattended caller, so it must never fire that path.
|
||
s = gate_stage.run_gate(DRAFTS, binary=binary, source_tag="grinder", commit=True,
|
||
propagate=False)
|
||
banked += s.get("banked", 0); fp = s.get("fleet_pct", fp)
|
||
verified |= set(s.get("verified", []))
|
||
log(f"gate {binary}: banked {s.get('banked')} (UN-propagated by design, §55b); "
|
||
f"total {banked}; fleet {fp}%")
|
||
if s.get("verified"):
|
||
log(" -> propagate as its own batch: "
|
||
+ "tools/share_body.py --apply --bucket new --batches 1 (P35 T6: shares every new same-address class; "
|
||
+ "verified: " + " ".join(s["verified"][:4]) + ")")
|
||
# A permuter win the whole-binary gate STILL rejects is plumbing-bound (not regalloc/sched) —
|
||
# re-permuting can never bank it. Blacklist so the grinder stops churning it (the §20 trap).
|
||
rejected = [f for f, _b in won if f not in verified]
|
||
if rejected:
|
||
blacklist.update(rejected); save_blacklist(blacklist)
|
||
log(f"blacklisted {len(rejected)} permuter-won/gate-rejected (plumbing): {', '.join(rejected)}")
|
||
heartbeat("running", None, banked, fp)
|
||
if a.once:
|
||
log(f"once done — banked {banked}."); heartbeat("done", None, banked, fp); return
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|