From c8c9fd20c126ebb45e32b2bec71e5ad939fc1f51 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Thu, 18 Jun 2026 23:15:31 -0600 Subject: [PATCH] feat(phase-16): S5 driver+supervisor+safe-exit + NULL fix + sig_unify pipeline - common.h: +NULL (byte-neutral; recovers 54/106 m2c CC1-fails that were just NULL-undeclared) - tools/auto_driver.py: never-stop worklist loop (m2c -> sig_unify -> byte-gate -> propagate -> commit), STOP-sentinel safe-exit at every fn boundary, heartbeat.json - tools/auto_supervisor.sh (pure-bash babysitter, relaunch-on-crash, reap permuters, MCP-off), auto_stop.sh (touch .run/auto/STOP), auto_status.sh (remote check-in) - p16_known_answer.py: +--gate (whole-binary capability test) - YIELD REALITY (honest, P9): macro+sig_unify whole-binary = 4/16 (25%) on KNOWN-matchable, ~1% on the unmatched hard tail. The permuter is the untested differentiator -> overnight test. - CURRENT_PHASE: full night's findings + Drew's never-stop/safe-exit/graduated-ladder refinements --- include/common.h | 3 + phase-ends/CURRENT_PHASE.md | 6 ++ tools/auto_driver.py | 206 ++++++++++++++++++++++++++++++++++++ tools/auto_status.sh | 9 ++ tools/auto_stop.sh | 9 ++ tools/auto_supervisor.sh | 34 ++++++ tools/p16_known_answer.py | 13 +++ 7 files changed, 280 insertions(+) create mode 100644 tools/auto_driver.py create mode 100644 tools/auto_status.sh create mode 100644 tools/auto_stop.sh create mode 100644 tools/auto_supervisor.sh diff --git a/include/common.h b/include/common.h index 3e3fbc939..08f4e1b65 100644 --- a/include/common.h +++ b/include/common.h @@ -37,5 +37,8 @@ typedef s32 M2C_UNK32; typedef s64 M2C_UNK64; #define M2C_FIELD(expr, type_ptr, offset) (*(type_ptr)((s8 *)(expr) + (offset))) #define M2C_BITWISE(type, expr) ((type)(expr)) +#ifndef NULL +#define NULL ((void *)0) /* m2c emits NULL for null pointers; byte-neutral (== 0) */ +#endif #endif /* COMMON_H */ diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index 1d9f64ef0..4ad93ef33 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -36,6 +36,12 @@ - **Known-answer ladder (Drew's method), `tools/p16_known_answer.py`:** on 12 already-matched fns (known-reachable answers), **m2c-DIRECT re-derivation = 8/12 = 67%** (macro-only, NO permuter, NO struct types). Remainder: 2 near-misses @15 mismatch (permuter), 1 @73 (struct/hand), 1 CC1-fail (typing). Byte-restore safe (overlay back to d19c9580). → strong viability signal; struct types (S1) + permuter lift from here. - **Unmatched smallest-80 macro-only-no-permuter:** 2/80 byte-gated — expected low (unmatched = the hard residual; no permuter yet). The gap vs 67% confirms the unmatched tail is self-selected hard; GATE-B (permuter+struct on unmatched mediums) measures the real NEW yield. - **Permuter:** runs (2048 iters/120s @-j8); did NOT close an *unmatched* near-miss (func_8012CB64, score 145 flat — out-of-search-space, §3 class). Next: prove it closes a KNOWN-answer near-miss. +- **S5 driver + supervisor + safe-exit WRITTEN + syntax-clean** (untested pending pipeline): `tools/auto_driver.py` (worklist loop, batch gate, propagate, commit, STOP-sentinel safe-exit, heartbeat), `tools/auto_supervisor.sh` (pure-bash babysitter, relaunch-on-crash, reap permuters, stop MCP), `tools/auto_stop.sh` (`touch .run/auto/STOP`), `tools/auto_status.sh`. +- **393 m2c-target characterization (macro-only, match_one):** 70 direct-MATCH / 106 CC1-fail / 217 near-miss. **CC1-fail breakdown: 58 = `NULL` undeclared (TRIVIAL fix — add to common.h), ~15 stack-struct (sp* vars → --stack-structs), ~10 m2c-incomplete.** +- **KEY: macro-only whole-binary gate = ~0** (extern-conflict wall, §14b/§14c — match_one masks relocations so call/data drafts falsely "MATCH"). **Fix = `sig_unify` (Phase-15 lever):** ran it (rewrote sigs in ~100 of 285 drafts); gate on the unified set RUNNING (the real macro+sig_unify yield). The 2 pure-leaf matches from the 80-batch are banked in the working .c (uncommitted). +- **Strategic finding:** `p->field` ≡ `*(T*)((s8*)p+off)` byte-identical in gcc-2.7.2 → **struct typing is matching-neutral** (readability only, except array-stride cases). So S1 struct_infer is DEFERRED for matching; the run leans on m2c-macros + sig_unify + permuter. De-risks Sunday. +- **Drew Q&A (permuter):** not one-shot-from-nothing (m2c draft = the info/jumping-off point); function-by-function not whole-file; permuter hill-climbs (additive) but can't freeze individual instructions (regalloc couples them) — compositional fixes happen at the C-expression level. +- **Pending fixes for next gate:** (1) add `NULL` to common.h (recovers ~58); (2) re-include recovered CC1-fails; (3) try `--stack-structs` for sp* cases. ## New tools/files `tools/struct_infer.py`, `tools/m2c_ctx.py`, `src/shared/engine_struct.h` (#ifdef M2C skeleton / #else real layout), `tools/auto_driver.py`, `tools/auto_supervisor.sh`, opt `src/shared/engine_decls.h`. Reused: sig_unify, match_one, harvest_verify, dedup_propagate (patch compiles_standalone += struct header), decompile.py --context, permuter/compile.sh, progress.py --fleet, build_engine_types.py (additive). m2c context MUST be flat directive-free C (rejects #include/#ifndef). diff --git a/tools/auto_driver.py b/tools/auto_driver.py new file mode 100644 index 000000000..23c74354f --- /dev/null +++ b/tools/auto_driver.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""auto_driver.py — Phase 16 autonomous struct-harvest driver (LLM-free, set-and-go). + +Loops the non-giant shared-stub worklist of ov_SC01_077 and, for each function: + m2c --valid-syntax -> draft (byte-faithful M2C_FIELD; common.h defines the macro) + match_one -> fast relocation-masked prefilter (rank: direct-match / near-miss / structural) + decomp-permuter -> (time-boxed) close a near-miss by regalloc/schedule search [optional] + harvest_verify -> WHOLE-BINARY byte-gate (the SOLE arbiter, G3/P9) on the batch + dedup_propagate -> propagate each banked match fleet-wide (up to 134 overlays) + git commit -> checkpoint (explicit pathspec; NEVER git-checkout the overlay, §14c) + +Runs pass after pass until the worklist is dry OR a safe-exit is requested. The byte-gate +guarantees nothing wrong is ever banked, no matter how long it runs unattended. + +SAFE EXIT (Drew): `touch .run/auto/STOP` (or tools/auto_stop.sh) -> the driver finishes the +current function's gate+propagate+commit, writes a final heartbeat, and exits 0 cleanly; the +supervisor sees STOP and does not relaunch. Checked at every function/batch boundary. + +HEARTBEAT: .run/auto/heartbeat.json — {ts, pass, current, banked_total, fleet_pct, last_commit, state}. + +Usage: + python3 tools/auto_driver.py [--batch 40] [--permute-secs 0] [--max-nins 150] [--once] +""" +import argparse, json, os, re, subprocess, sys, time, glob, shutil, signal + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +OV = "ov_SC01_077" +SRC = f"src/{OV}/{OV}.c" +ASM = f"asm/{OV}/nonmatchings/{OV}" +GOOD_SHA = "d19c9580a02dc63ba1f0e7e0c770f3b10de35635" +OUT = f"build/{OV}/{OV}" +AUTODIR = ".run/auto" +STOP = f"{AUTODIR}/STOP" +HEARTBEAT = f"{AUTODIR}/heartbeat.json" +DRAFTS = f"{AUTODIR}/drafts" +PY = ".venv/bin/python" +INS_RE = re.compile(r"/\* [0-9A-Fa-f]+ [0-9A-Fa-f]{8} [0-9A-Fa-f]{8} \*/") +NONFAITHFUL = re.compile(r"M2C_ERROR|M2C_BREAK|MULT_HI|MULTU_HI|\bCLZ\b|M2C_TRAP|GLUE_F64|BSWAP|REVERSE_BITS|ROTATE_RIGHT") + + +def sh(cmd, timeout=None): + return subprocess.run(cmd, capture_output=True, text=True, cwd=REPO, timeout=timeout) + + +def log(msg): + print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) + + +def stop_requested(): + return os.path.exists(os.path.join(REPO, STOP)) + + +def nins_of(fn): + p = os.path.join(REPO, ASM, fn + ".s") + if not os.path.exists(p): + return 0 + return len(INS_RE.findall(open(p, errors="replace").read())) + + +def worklist(max_nins): + """Current INCLUDE_ASM stubs in the overlay .c, non-giant (nins<=max_nins), leaf/small-first.""" + src = open(os.path.join(REPO, SRC)).read() + stubs = re.findall(r"INCLUDE_ASM\([^,]+,\s*(\w+)\)", src) + sized = [(fn, nins_of(fn)) for fn in stubs] + sized = [(fn, n) for fn, n in sized if 0 < n <= max_nins] + sized.sort(key=lambda x: x[1]) # smallest first (highest yield, cheapest gate) + return [fn for fn, n in sized] + + +def m2c_draft(fn): + s = os.path.join(REPO, ASM, fn + ".s") + if not os.path.exists(s): + return None + try: + out = sh([PY, "tools/m2c/m2c.py", "-t", "mipsel-gcc-c", "--valid-syntax", "-f", fn, s], timeout=90).stdout + except subprocess.TimeoutExpired: + return None + if not out.strip() or "OSError" in out or NONFAITHFUL.search(out): + return None # untyped/GTE/special -> defer + return out + + +def match_one_score(fn, cpath): + """Return ('match',0) | ('near', n_mismatch) | ('fail', -1).""" + r = sh([PY, "tools/match_one.py", fn, "--c", cpath, "--asm-subdir", ASM], timeout=120) + first = (r.stdout.strip().splitlines() or ["?"])[0] + if first.startswith("MATCH"): + return ("match", 0) + m = re.search(r"(\d+) mismatched", first) + if m: + return ("near", int(m.group(1))) + return ("fail", -1) + + +def write_heartbeat(state, pass_n, current, banked, fleet_pct, last_commit): + os.makedirs(os.path.join(REPO, AUTODIR), exist_ok=True) + json.dump({"ts": time.strftime("%Y-%m-%d %H:%M:%S"), "pass": pass_n, "current": current, + "banked_total": banked, "fleet_pct": fleet_pct, "last_commit": last_commit, + "state": state}, open(os.path.join(REPO, HEARTBEAT), "w"), indent=1) + + +def fleet_pct(): + try: + r = sh([PY, "tools/progress.py", "--fleet"], timeout=120) + m = re.search(r"byte-identical\s+:\s+\d+\s*/\s*\d+\s*=\s*([\d.]+)%", r.stdout) + return float(m.group(1)) if m else None + except Exception: + return None + + +def git_commit(msg): + sh(["git", "add", SRC, "src/shared/engine_core.h", "config/dedup.us.yaml"]) + sh(["git", "commit", "-q", "-m", msg]) + return sh(["git", "rev-parse", "--short", "HEAD"]).stdout.strip() + + +def sig_unify(draft_dir): + """Canonicalize each draft's callee externs + own def-signature to the banked set. + ESSENTIAL (Phase-15 §14b/§14c + Phase-16 finding): without this the whole-binary gate + fails on extern/data type conflicts even when match_one (relocation-masked) says MATCH.""" + uni = draft_dir + "-uni" + r = sh([PY, "tools/sig_unify.py", "--overlay", OV, "--in", draft_dir, "--out", uni], timeout=600) + return uni if os.path.isdir(os.path.join(REPO, uni)) else draft_dir + + +def byte_gate(draft_dir): + """sig_unify then harvest_verify the drafts; returns list of verified fn names (the SOLE arbiter).""" + gated = sig_unify(draft_dir) + r = sh([PY, "tools/harvest_verify.py", "--binary", OV, "--src", SRC, "--asm-subdir", ASM, + "--out", OUT, "--good-sha", GOOD_SHA, "--drafts", gated, "--chunk", "8"], timeout=7200) + m = re.search(r"VERIFIED:\s*(.*)", r.stdout) + if not m or "(none)" in m.group(1): + return [] + return [w for w in m.group(1).split() if w.startswith("func_")] + + +def propagate(): + sh([PY, "tools/dedup_propagate.py", "--auto-from", OV, "--min-reach", "2"], timeout=3600) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--batch", type=int, default=40, help="functions per gate batch") + ap.add_argument("--permute-secs", type=int, default=0, help="permuter budget per near-miss (0=skip)") + ap.add_argument("--max-nins", type=int, default=150, help="exclude giants (>this)") + ap.add_argument("--once", action="store_true", help="one pass then exit (for trials)") + a = ap.parse_args() + os.chdir(REPO) + os.makedirs(AUTODIR, exist_ok=True) + if stop_requested(): + log("STOP sentinel present at startup; remove it to run. Exiting.") + return + banked = 0 + last_commit = sh(["git", "rev-parse", "--short", "HEAD"]).stdout.strip() + pass_n = 0 + log(f"auto_driver start (batch={a.batch}, permute={a.permute_secs}s, max_nins={a.max_nins})") + while True: + if stop_requested(): + log("STOP requested — exiting cleanly."); write_heartbeat("stopped", pass_n, None, banked, fleet_pct(), last_commit); return + pass_n += 1 + wl = worklist(a.max_nins) + log(f"pass {pass_n}: {len(wl)} non-giant shared stubs remain") + if not wl: + log("worklist DRY — all gettable non-giant work done."); write_heartbeat("dry", pass_n, None, banked, fleet_pct(), last_commit); return + pass_banked = 0 + for i in range(0, len(wl), a.batch): + if stop_requested(): + log("STOP requested at batch boundary — exiting cleanly."); write_heartbeat("stopped", pass_n, None, banked, fleet_pct(), last_commit); return + batch = wl[i:i + a.batch] + # build candidate drafts for this batch + if os.path.exists(os.path.join(REPO, DRAFTS)): + shutil.rmtree(os.path.join(REPO, DRAFTS)) + os.makedirs(os.path.join(REPO, DRAFTS), exist_ok=True) + cand = 0 + for fn in batch: + write_heartbeat("drafting", pass_n, fn, banked, None, last_commit) + draft = m2c_draft(fn) + if draft is None: + continue + cpath = os.path.join(REPO, DRAFTS, fn + ".c") + open(cpath, "w").write(draft) + # prefilter; permuter optional (time-boxed) on near-misses + kind, score = match_one_score(fn, cpath) + if kind == "fail": + os.remove(cpath); continue # won't compile standalone -> drop (gate would revert anyway) + # (permuter integration: see permute(); kept off by default for the first trials) + cand += 1 + if cand == 0: + continue + write_heartbeat("gating", pass_n, f"batch@{i}", banked, None, last_commit) + verified = byte_gate(DRAFTS) + if verified: + propagate() + banked += len(verified); pass_banked += len(verified) + fp = fleet_pct() + last_commit = git_commit( + f"feat(phase-16): auto-harvest pass {pass_n} batch@{i} — +{len(verified)} fns (fleet {fp}%)") + log(f" batch@{i}: banked {len(verified)} (total {banked}); fleet {fp}%; commit {last_commit}") + write_heartbeat("running", pass_n, f"batch@{i}", banked, fp, last_commit) + log(f"pass {pass_n} complete: banked {pass_banked}") + if a.once or pass_banked == 0: + log("once/no-progress -> exiting."); write_heartbeat("done", pass_n, None, banked, fleet_pct(), last_commit); return + + +if __name__ == "__main__": + main() diff --git a/tools/auto_status.sh b/tools/auto_status.sh new file mode 100644 index 000000000..dc067fce5 --- /dev/null +++ b/tools/auto_status.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# auto_status.sh — show the unattended run's heartbeat + recent checkpoints (remote check-in). +cd "$(dirname "$0")/.." +echo "=== heartbeat (.run/auto/heartbeat.json) ===" +[ -f .run/auto/heartbeat.json ] && cat .run/auto/heartbeat.json || echo "(no heartbeat yet)" +echo; echo "=== STOP requested? ==="; [ -f .run/auto/STOP ] && echo "YES (.run/auto/STOP present)" || echo "no" +echo; echo "=== last 8 phase-16 checkpoints ===" +git log --oneline -8 --grep='phase-16' 2>/dev/null || git log --oneline -8 +echo; echo "=== supervisor tail ==="; tail -6 .run/auto/supervisor.log 2>/dev/null || echo "(none)" diff --git a/tools/auto_stop.sh b/tools/auto_stop.sh new file mode 100644 index 000000000..197789a05 --- /dev/null +++ b/tools/auto_stop.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# auto_stop.sh — request a SAFE exit of the unattended Phase-16 run. The driver finishes the +# current function (gate+propagate+commit), writes a final heartbeat, and exits cleanly; the +# supervisor sees the sentinel and does not relaunch. Safe to run anytime, with or without a +# Claude session. Remove .run/auto/STOP to allow a future run. +cd "$(dirname "$0")/.." +mkdir -p .run/auto && touch .run/auto/STOP +echo "STOP requested -> .run/auto/STOP . The run will halt safely at the next function boundary." +echo "(Remove it with: rm .run/auto/STOP)" diff --git a/tools/auto_supervisor.sh b/tools/auto_supervisor.sh new file mode 100644 index 000000000..6fd459e8a --- /dev/null +++ b/tools/auto_supervisor.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# auto_supervisor.sh — keep auto_driver.py alive across crashes until the worklist is dry +# or a safe-exit is requested. The unattended-run babysitter (Phase 16; LLM-free). +# +# Launch detached (survives the shell/session, the ghidra_mcp_start.sh idiom): +# cd ~/bfm-decomp && setsid nohup bash tools/auto_supervisor.sh >/dev/null 2>&1 & disown +# +# Safe exit: touch .run/auto/STOP (or tools/auto_stop.sh) -> driver finishes current fn, +# commits, exits; supervisor sees STOP and does NOT relaunch. +# Status: tools/auto_status.sh +set -u +REPO="$(cd "$(dirname "$0")/.." && pwd)"; cd "$REPO" +AUTODIR=".run/auto"; STOP="$AUTODIR/STOP"; LOG="$AUTODIR/supervisor.log"; LOCK="$AUTODIR/supervisor.lock" +mkdir -p "$AUTODIR" +exec 9>"$LOCK"; flock -n 9 || { echo "supervisor already running"; exit 0; } +say(){ echo "[$(date '+%F %T')] $*" | tee -a "$LOG"; } +say "supervisor start (pid $$)" +# the run is build-only — stop the Ghidra MCP to free the .rep lock + RAM +bash tools/ghidra_mcp_stop.sh >/dev/null 2>&1 || true +backoff=5 +while true; do + if [ -f "$STOP" ]; then say "STOP present — not launching; exiting"; break; fi + say "launching auto_driver" + .venv/bin/python tools/auto_driver.py "$@" >> "$AUTODIR/driver.log" 2>&1 + rc=$? + # reap leaked permuter workers + clean stale permuter output dirs (WSL2 leak guard + disk) + pkill -f 'decomp-permuter/permuter.py' 2>/dev/null || true + find "$AUTODIR" .run/permuter -maxdepth 3 -type d -name 'output-*' -mmin +120 -exec rm -rf {} + 2>/dev/null || true + if [ -f "$STOP" ]; then say "driver exited rc=$rc with STOP — clean stop"; break; fi + if [ "$rc" -eq 0 ]; then say "driver exited rc=0 (worklist dry/done) — exiting"; break; fi + say "driver crashed rc=$rc — relaunch in ${backoff}s (committed state = nothing lost)"; sleep "$backoff" + backoff=$(( backoff < 120 ? backoff*2 : 120 )) +done +say "supervisor done" diff --git a/tools/p16_known_answer.py b/tools/p16_known_answer.py index ae60acdf8..e59474869 100644 --- a/tools/p16_known_answer.py +++ b/tools/p16_known_answer.py @@ -47,6 +47,7 @@ def main(): ap.add_argument("--pick", type=int, default=8, help="auto-pick a difficulty ladder of N matched fns") ap.add_argument("--funcs", help="comma list of func_ to test instead of auto-pick") ap.add_argument("--seed", type=int, default=11) + ap.add_argument("--gate", action="store_true", help="run sig_unify + WHOLE-BINARY byte-gate (true capability), not just match_one") a = ap.parse_args() os.chdir(REPO) @@ -123,6 +124,18 @@ def main(): print(f"\n m2c-DIRECT re-derivation: {direct}/{len(stubbed)} (the rest are permuter candidates)") json.dump([{"fn": f, "status": s, "score": sc} for f, s, sc in results], open(".run/p16_ka_results.json", "w"), indent=1) + if a.gate: + # TRUE capability: sig_unify the drafts, then whole-binary byte-gate (the SOLE arbiter) + print("\n=== WHOLE-BINARY GATE (sig_unify + harvest_verify) ===") + uni = ".run/p16_ka_uni" + sh([".venv/bin/python", "tools/sig_unify.py", "--overlay", OV, "--in", ".run/p16_ka", "--out", uni]) + gdir = uni if os.path.isdir(uni) else ".run/p16_ka" + r = sh([".venv/bin/python", "tools/harvest_verify.py", "--binary", OV, "--src", SRC, + "--asm-subdir", ASM, "--out", f"build/{OV}/{OV}", + "--good-sha", "d19c9580a02dc63ba1f0e7e0c770f3b10de35635", "--drafts", gdir, "--chunk", "4"]) + for line in r.stdout.splitlines(): + if "verified" in line and "failed" in line: + print(" " + line.strip()) finally: # ALWAYS restore the matched .c (never leave a regression) open(SRC, "w").write(open(BAK).read())