diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index 4ad93ef33c..9b748519e2 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -43,6 +43,14 @@ - **Drew Q&A (permuter):** not one-shot-from-nothing (m2c draft = the info/jumping-off point); function-by-function not whole-file; permuter hill-climbs (additive) but can't freeze individual instructions (regalloc couples them) — compositional fixes happen at the C-expression level. - **Pending fixes for next gate:** (1) add `NULL` to common.h (recovers ~58); (2) re-include recovered CC1-fails; (3) try `--stack-structs` for sp* cases. +### Yield reality + overnight test (2026-06-18 late Thu) — the go/no-go crux +- **NULL fix DONE** (byte-neutral, committed commit:0124): recovers **54/106** CC1-fails. Compiling drafts now 341/393. +- **Known-answer WHOLE-BINARY capability (the honest number):** m2c+sig_unify = **4/16 = 25%** on KNOWN-matchable fns (match_one's 9/16 over-counts — masks relocations). Unmatched hard tail ≈ **1%** (the 964 are the residual Phase-15 couldn't crack — hard by selection). +- **THE KEY UNKNOWN = the permuter close-rate on near-misses.** One 120s test didn't close (out-of-search-space). Built `tools/p16_permute.py` (M2C_FIELD→cast expander + permuter setup/run/parse). **Overnight test RUNNING:** macro+sig_unify+NULL full gate (banks direct matches) → then 40 near-misses × 7min permuter each (~4.5h) → `.run/permute_overnight.log`. **Friday AM: read the close-rate = the 5-day-run go/no-go.** +- **Honest framing (P9):** the "85-90% ceiling" was theoretical. Practical reality: modest-but-real yield. Even 10% of 964 × 134 propagation ≈ +3-4% fleet — worth the CHEAP compute run IF the permuter adds meaningful lift. The weekend tests (small tonight → overnight → analyze Fri → trial Sat → go Sun) resolve it — exactly the de-risk Drew wanted. +- **S5 driver/supervisor/safe-exit BUILT + committed** (untested end-to-end — test Fri after the yield read). S1 struct_infer RESOLVED as not-needed-for-matching (struct typing byte-neutral vs macros). +- Commits this session: commit:0122 (S0+common.h) → commit:0123 (known-answer 67%) → commit:0124 (S5+NULL+sig_unify). Working .c has ~5 banked leaf matches (uncommitted; the running gate will add more). + ## New tools/files `tools/struct_infer.py`, `tools/m2c_ctx.py`, `src/shared/engine_struct.h` (#ifdef M2C skeleton / #else real layout), `tools/auto_driver.py`, `tools/auto_supervisor.sh`, opt `src/shared/engine_decls.h`. Reused: sig_unify, match_one, harvest_verify, dedup_propagate (patch compiles_standalone += struct header), decompile.py --context, permuter/compile.sh, progress.py --fleet, build_engine_types.py (additive). m2c context MUST be flat directive-free C (rejects #include/#ifndef). diff --git a/src/ov_SC01_077/ov_SC01_077.c b/src/ov_SC01_077/ov_SC01_077.c index b9b353a824..6601d25820 100644 --- a/src/ov_SC01_077/ov_SC01_077.c +++ b/src/ov_SC01_077/ov_SC01_077.c @@ -247,7 +247,12 @@ INCLUDE_ASM("asm/ov_SC01_077/nonmatchings/ov_SC01_077", func_8012ACE0); DEFINE_func_8012AD44() /* dedup: shared engine-core @0x8012AD44 (src/shared) */ -INCLUDE_ASM("asm/ov_SC01_077/nonmatchings/ov_SC01_077", func_8012AD50); +s32 func_8012AD50(void * arg0) +{ + M2C_FIELD(arg0, s16 *, 0x34) = 0; + M2C_FIELD(arg0, u16 *, 2) = (u16) (M2C_FIELD(arg0, u16 *, 2) + 1); +} + void func_8012AD64(s32 *a0, s16 a1) { *(s16*)((s32)a0 + 0x34) = a1; @@ -1674,13 +1679,19 @@ DEFINE_func_80147054() /* dedup: shared engine-core @0x80147054 (src/shared) */ DEFINE_func_80147060() /* dedup: shared engine-core @0x80147060 (src/shared) */ -INCLUDE_ASM("asm/ov_SC01_077/nonmatchings/ov_SC01_077", func_8014706C); +void func_8014706C(void *arg0) { + M2C_FIELD(arg0, s8 *, 0x4D) = 2; +} + DEFINE_func_80147078() /* dedup: shared engine-core @0x80147078 (src/shared) */ DEFINE_func_80147084() /* dedup: shared engine-core @0x80147084 (src/shared) */ -INCLUDE_ASM("asm/ov_SC01_077/nonmatchings/ov_SC01_077", func_8014708C); +void func_8014708C(void *arg0) { + M2C_FIELD(arg0, s8 *, 0x221) = 1; +} + DEFINE_func_80147098() /* dedup: shared engine-core @0x80147098 (src/shared) */ @@ -1688,7 +1699,11 @@ DEFINE_func_801470A0() /* dedup: shared engine-core @0x801470A0 (src/shared) */ DEFINE_func_801470AC() /* dedup: shared engine-core @0x801470AC (src/shared) */ -INCLUDE_ASM("asm/ov_SC01_077/nonmatchings/ov_SC01_077", func_801470B4); +void func_801470B4(s32 arg0) +{ + M2C_FIELD(arg0, s8 *, 0x222) = 1; +} + DEFINE_func_801470C0() /* dedup: shared engine-core @0x801470C0 (src/shared) */ @@ -1804,7 +1819,16 @@ DEFINE_func_80148534() /* dedup: shared engine-core @0x80148534 (src/shared) */ DEFINE_func_8014856C() /* dedup: shared engine-core @0x8014856C (src/shared) */ -INCLUDE_ASM("asm/ov_SC01_077/nonmatchings/ov_SC01_077", func_801485B8); +void func_801485B8(s32 arg0, s32 arg1, s32 arg2) +{ + M2C_FIELD(arg1, s16 *, 0) = (s16) (s8) M2C_FIELD(arg0, u8 *, 0); + M2C_FIELD(arg1, s16 *, 2) = (s16) (s8) M2C_FIELD(arg0, u8 *, 1); + M2C_FIELD(arg1, s16 *, 4) = (s16) (s8) M2C_FIELD(arg0, u8 *, 2); + M2C_FIELD(arg2, s16 *, 0) = (s16) (s8) M2C_FIELD(arg0, u8 *, 3); + M2C_FIELD(arg2, s16 *, 2) = (s16) (s8) M2C_FIELD(arg0, u8 *, 4); + M2C_FIELD(arg2, s16 *, 4) = (s16) (s8) M2C_FIELD(arg0, u8 *, 5); +} + /* func_80148634: ori v0,0x8080; sh 0 @0xAA; sh 0 @0xAC; sh 0x8080 @0xAE * 0x8080 loaded via ori (positive 16-bit imm) => store an unsigned 16-bit value. */ @@ -2411,7 +2435,24 @@ INCLUDE_ASM("asm/ov_SC01_077/nonmatchings/ov_SC01_077", func_8014E6A0); DEFINE_func_8014E6F8() /* dedup: shared engine-core @0x8014E6F8 (src/shared) */ -INCLUDE_ASM("asm/ov_SC01_077/nonmatchings/ov_SC01_077", func_8014E790); +s32 func_80135888(s32, s32, M2C_UNK, M2C_UNK); /* extern */ + +s32 func_8014E790(s32 arg0, s16 * arg1, s16 * arg2) +{ + s32 temp_a1; + void *temp_s0; + + temp_s0 = M2C_FIELD(arg0, void **, 0x180); + if ((M2C_FIELD(temp_s0, u16 *, 0) == 0) || !(M2C_FIELD(temp_s0, u16 *, 0x5C) & 0x20) || (temp_a1 = M2C_FIELD(temp_s0, s32 *, 0x58), (temp_a1 == 0)) || (func_80135888(M2C_FIELD(temp_s0, s32 *, 0x20), temp_a1, arg1, arg2) == 0)) { + M2C_FIELD(arg0, void **, 0x180) = NULL; + return 0; + } + M2C_FIELD(arg0, u16 *, 6) = (u16) M2C_FIELD(temp_s0, u16 *, 6); + M2C_FIELD(arg0, u16 *, 0xA) = (u16) M2C_FIELD(temp_s0, u16 *, 0xA); + M2C_FIELD(arg0, u16 *, 0xE) = (u16) M2C_FIELD(temp_s0, u16 *, 0xE); + return 1; +} + INCLUDE_ASM("asm/ov_SC01_077/nonmatchings/ov_SC01_077", func_8014E83C); diff --git a/tools/p16_improve.py b/tools/p16_improve.py new file mode 100644 index 0000000000..1962f6470f --- /dev/null +++ b/tools/p16_improve.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""p16_improve.py — known-answer-guided harness improvement loop (Phase 16, Drew's method). + +Take ALREADY-matched functions (known answers) in a difficulty band, revert them to stubs, +run the real pipeline (m2c --valid-syntax -> sig_unify -> match_one / optional whole-binary +gate), and CATEGORIZE every failure into an actionable CLASS. Since the answer is known-reachable, +every fail is a harness gap we can diagnose + fix generally; the fix then transfers to the +unmatched functions that hit the same class. + +Reports: re-derivation rate (the harness CEILING for this band) + a ranked failure-class +histogram (what to fix next). Restores the overlay .c afterward (never git-checkout, §14c). + + python3 tools/p16_improve.py --band easy --n 40 # easy band, match_one categorize + python3 tools/p16_improve.py --band med --n 30 --gate # medium, whole-binary ceiling too + bands: tiny(1-4) easy(5-12) med(13-40) big(41+) (stmt-count proxy from the macro body) +""" +import argparse, os, re, subprocess, glob, random + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +OV = "ov_SC01_077"; SRC = f"src/{OV}/{OV}.c"; ECORE = "src/shared/engine_core.h" +ASM = f"asm/{OV}/nonmatchings/{OV}"; BAK = ".run/p16_improve_bak.c"; PY = ".venv/bin/python" +GOOD = "d19c9580a02dc63ba1f0e7e0c770f3b10de35635" +BANDS = {"tiny": (1, 4), "easy": (5, 12), "med": (13, 40), "big": (41, 10**9)} + + +def sh(cmd, **kw): return subprocess.run(cmd, capture_output=True, text=True, cwd=REPO, **kw) + + +def macro_bodies(): + txt = open(os.path.join(REPO, ECORE)).read() + out = {} + for m in re.finditer(r"#define DEFINE_func_([0-9A-Fa-f]+)\(\)((?:.*\\\n)*.*\n)", txt): + out[m.group(1)] = len(re.findall(r"[;}]", m.group(2))) + return out + + +def classify_fail(fn, cpath): + """Run match_one; return ('match',0) | ('near',n) | ('class', label) for an actionable bucket.""" + r = sh([PY, "tools/match_one.py", fn, "--c", cpath, "--asm-subdir", ASM]) + out = r.stdout + first = (out.strip().splitlines() or ["?"])[0] + if first.startswith("MATCH"): + return ("match", 0) + m = re.search(r"(\d+) mismatched", first) + if m: + return ("near", int(m.group(1))) + # compile failure -> sub-classify by the error text (the actionable class) + if "CPP FAIL" in out: + return ("class", "cpp-fail") + if re.search(r"`NULL' undeclared", out): + return ("class", "compile:NULL") + if re.search(r"`sp[0-9A-Fa-f]+' undeclared|_m2c_stack", out): + return ("class", "compile:stack-var") + if re.search(r"`(subroutine_arg|saved_reg)\w*' undeclared", out): + return ("class", "compile:m2c-incomplete-arg") + m2 = re.search(r"`([A-Za-z_]\w*)' undeclared", out) + if m2: + return ("class", f"compile:undeclared-{'D_' if m2.group(1).startswith('D_') else 'other'}") + if "conflicting types" in out: + return ("class", "compile:conflicting-types") + if "redefinition" in out: + return ("class", "compile:redefinition") + if "parse error" in out: + return ("class", "compile:parse-error") + return ("class", "compile:other") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--band", choices=list(BANDS), default="easy") + ap.add_argument("--n", type=int, default=40) + ap.add_argument("--seed", type=int, default=3) + ap.add_argument("--gate", action="store_true", help="also run sig_unify + whole-binary gate for the true ceiling") + a = ap.parse_args() + os.chdir(REPO) + lo, hi = BANDS[a.band] + bodies = macro_bodies() + srctxt = open(SRC).read() + cand = [addr for addr, n in bodies.items() + if lo <= n <= hi and re.search(rf"DEFINE_func_{addr}\(\)", srctxt)] + random.seed(a.seed); random.shuffle(cand) + funcs = [f"func_{addr}" for addr in cand[:a.n]] + print(f"band={a.band}({lo}-{hi} stmts) testing {len(funcs)} known-answer fns") + + open(BAK, "w").write(srctxt) + s = srctxt + for fn in funcs: + s = re.sub(rf"^[ \t]*DEFINE_func_{fn[5:]}\(\).*$", f'INCLUDE_ASM("{ASM}", {fn});', s, flags=re.M) + open(SRC, "w").write(s) + try: + if sh(["make", "extract", f"BINARY={OV}"]).returncode: + print("EXTRACT FAIL"); return + os.makedirs(".run/p16_imp", exist_ok=True) + for f in glob.glob(".run/p16_imp/*.c"): + os.remove(f) + cats, near_hist, matched = {}, [], 0 + for fn in funcs: + spath = f"{ASM}/{fn}.s" + if not os.path.exists(spath): + cats["no-asm"] = cats.get("no-asm", 0) + 1; continue + out = sh([PY, "tools/m2c/m2c.py", "-t", "mipsel-gcc-c", "--valid-syntax", "-f", fn, spath]).stdout + if not out.strip() or "OSError" in out: + cats["m2c-empty"] = cats.get("m2c-empty", 0) + 1; continue + if re.search(r"M2C_ERROR|M2C_BREAK|MULT_HI|\bCLZ\b|M2C_TRAP|GLUE_F64|BSWAP", out): + cats["nonfaithful(GTE/special)"] = cats.get("nonfaithful(GTE/special)", 0) + 1; continue + cp = f".run/p16_imp/{fn}.c"; open(cp, "w").write(out) + kind, val = classify_fail(fn, cp) + if kind == "match": + matched += 1 + elif kind == "near": + near_hist.append(val); cats[f"near-miss"] = cats.get("near-miss", 0) + 1 + else: + cats[val] = cats.get(val, 0) + 1 + n = len(funcs) + print(f"\n=== BAND {a.band}: match_one re-derivation = {matched}/{n} = {100*matched//max(n,1)}% (the harness ceiling, match_one) ===") + if near_hist: + ez = sum(1 for x in near_hist if x <= 8) + print(f" near-misses: {len(near_hist)} (permuter candidates; {ez} are <=8 mismatch = easy permute)") + print(" FAILURE CLASSES (ranked — fix the top ones):") + for cls, c in sorted(cats.items(), key=lambda x: -x[1]): + print(f" {c:4} {cls}") + if a.gate: + print("\n=== whole-binary ceiling (sig_unify + gate) ===") + sh([PY, "tools/sig_unify.py", "--overlay", OV, "--in", ".run/p16_imp", "--out", ".run/p16_imp-uni"]) + gd = ".run/p16_imp-uni" if os.path.isdir(os.path.join(REPO, ".run/p16_imp-uni")) else ".run/p16_imp" + r = sh([PY, "tools/harvest_verify.py", "--binary", OV, "--src", SRC, "--asm-subdir", ASM, + "--out", f"build/{OV}/{OV}", "--good-sha", GOOD, "--drafts", gd, "--chunk", "4"]) + for line in r.stdout.splitlines(): + if "verified" in line and "failed" in line: + print(" " + line.strip()) + finally: + open(SRC, "w").write(open(BAK).read()) + print(f"\nrestored {SRC}.") + + +if __name__ == "__main__": + main() diff --git a/tools/p16_permute.py b/tools/p16_permute.py new file mode 100644 index 0000000000..5dce4fcd3c --- /dev/null +++ b/tools/p16_permute.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""p16_permute.py — measure decomp-permuter close-rate on m2c near-miss drafts (Phase 16). + +For each function: take its (sig-unified) m2c draft, expand M2C_FIELD macros to plain casts so +pycparser can parse it, build base.c + target.o, run decomp-permuter (time-boxed, -j), and report +whether it reached score 0 (a byte match). This measures the permuter's yield — the differentiator +that decides whether the unattended 5-day run is worthwhile. + +A score-0 hit is written to .run/permuter//output-*/source.c; we copy it to --winners for gating. + + python3 tools/p16_permute.py --funcs func_A,func_B --secs 300 --j 8 + python3 tools/p16_permute.py --from-drafts .run/drafts-full-uni --near-max 8 --limit 12 --secs 300 +""" +import argparse, os, re, subprocess, sys, glob, shutil, time + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +OV = "ov_SC01_077" +ASM = f"asm/{OV}/nonmatchings/{OV}" +PY = ".venv/bin/python" +TYPEDEFS = ("typedef unsigned char u8; typedef unsigned short u16; typedef unsigned int u32;\n" + "typedef signed char s8; typedef short s16; typedef int s32;\n" + "typedef unsigned long long u64; typedef long long s64; typedef double f64;\n" + "typedef s32 M2C_UNK; typedef s8 M2C_UNK8; typedef s16 M2C_UNK16; typedef s32 M2C_UNK32; typedef s64 M2C_UNK64;\n" + "#define NULL ((void*)0)\n") + + +def sh(cmd, **kw): + return subprocess.run(cmd, capture_output=True, text=True, cwd=REPO, **kw) + + +def split_args(s): + """split a top-level comma-separated arg list (balanced parens).""" + out, depth, cur = [], 0, "" + for ch in s: + if ch == "(": depth += 1; cur += ch + elif ch == ")": depth -= 1; cur += ch + elif ch == "," and depth == 0: out.append(cur); cur = "" + else: cur += ch + if cur.strip(): + out.append(cur) + return out + + +def expand_m2c_field(c): + """M2C_FIELD(EXPR, TYPE, OFF) -> (*(TYPE)((s8*)(EXPR)+(OFF))), innermost-first, repeatedly.""" + while True: + i = c.find("M2C_FIELD(") + if i < 0: + return c + # find the matching close paren for this call + j = i + len("M2C_FIELD(") + depth = 1 + while j < len(c) and depth: + if c[j] == "(": depth += 1 + elif c[j] == ")": depth -= 1 + j += 1 + inner = c[i + len("M2C_FIELD("):j - 1] + args = split_args(inner) + if len(args) != 3: + # malformed; bail to avoid an infinite loop + return c + expr, typ, off = (a.strip() for a in args) + # if expr still has M2C_FIELD, expand it first (recurse on the substring) + if "M2C_FIELD(" in expr: + expr = expand_m2c_field(expr) + repl = f"(*({typ})((s8*)({expr})+({off})))" + c = c[:i] + repl + c[j:] + + +def strip_externs_and_includes(c): + """drop #include / #define lines and m2c's leading extern/typedef decls — keep the function def.""" + lines = [] + for ln in c.splitlines(): + s = ln.strip() + if s.startswith("#"): + continue + if re.match(r"^(extern|typedef)\b", s): + continue + lines.append(ln) + return "\n".join(lines) + + +def make_base_c(draft_c): + body = expand_m2c_field(draft_c) + body = strip_externs_and_includes(body) + return TYPEDEFS + body + "\n" + + +def setup(fn, draft_c): + pd = os.path.join(REPO, ".run/permuter", fn) + if os.path.exists(pd): + shutil.rmtree(pd) + os.makedirs(pd) + open(f"{pd}/base.c", "w").write(make_base_c(draft_c)) + s = os.path.join(REPO, ASM, fn + ".s") + tgt = f"{pd}/target.s" + with open(tgt, "w") as f: + f.write('.set noat\n.set noreorder\n.include "macro.inc"\n.section .text\n\n') + f.write(open(s, errors="replace").read()) + r = sh(["mipsel-linux-gnu-as", "-Iinclude", "-march=r3000", "-mtune=r3000", + "-no-pad-sections", "-O1", "-G0", tgt, "-o", f"{pd}/target.o"]) + if r.returncode: + return None + open(f"{pd}/settings.toml", "w").write(f'func_name = "{fn}"\ncompiler_type = "gcc"\n') + open(f"{pd}/compile.sh", "w").write(f'#!/bin/bash\nexec {REPO}/tools/permuter/compile.sh "$@"\n') + os.chmod(f"{pd}/compile.sh", 0o755) + return pd + + +def run_permuter(pd, secs, j): + env = dict(os.environ, PATH=f"{REPO}/tools/permuter/bin:" + os.environ["PATH"]) + try: + subprocess.run([PY, "tools/decomp-permuter/permuter.py", pd, "-j", str(j), "--stop-on-zero"], + cwd=REPO, env=env, capture_output=True, text=True, timeout=secs) + except subprocess.TimeoutExpired: + pass + # kill stragglers + subprocess.run(["pkill", "-f", "decomp-permuter/permuter.py"], capture_output=True) + win = glob.glob(f"{pd}/output-*/source.c") + return win[0] if win else None + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--funcs") + ap.add_argument("--from-drafts", default=".run/drafts-full-uni") + ap.add_argument("--near-max", type=int, default=8) + ap.add_argument("--limit", type=int, default=12) + ap.add_argument("--secs", type=int, default=300) + ap.add_argument("--j", type=int, default=8) + ap.add_argument("--winners", default=".run/permuter-winners") + a = ap.parse_args() + os.chdir(REPO) + os.makedirs(a.winners, exist_ok=True) + + if a.funcs: + funcs = a.funcs.split(",") + else: + # pick near-miss drafts (match_one mismatch in [1, near-max]) from the drafts dir + funcs = [] + for cf in sorted(glob.glob(a.from_drafts + "/*.c")): + fn = os.path.basename(cf)[:-2] + r = sh([PY, "tools/match_one.py", fn, "--c", cf, "--asm-subdir", ASM]) + first = (r.stdout.strip().splitlines() or ["?"])[0] + m = re.search(r"(\d+) mismatched", first) + if m and 1 <= int(m.group(1)) <= a.near_max: + funcs.append(fn) + if len(funcs) >= a.limit: + break + print(f"permuter batch: {len(funcs)} near-miss fns, {a.secs}s each @ -j{a.j}") + won = 0 + for k, fn in enumerate(funcs, 1): + cf = os.path.join(a.from_drafts, fn + ".c") + if not os.path.exists(cf): + print(f" [{k}/{len(funcs)}] {fn}: no draft"); continue + pd = setup(fn, open(cf).read()) + if not pd: + print(f" [{k}/{len(funcs)}] {fn}: setup failed (target.o)"); continue + t0 = time.time() + win = run_permuter(pd, a.secs, a.j) + dt = int(time.time() - t0) + if win: + shutil.copy(win, os.path.join(a.winners, fn + ".c")); won += 1 + print(f" [{k}/{len(funcs)}] {fn}: *** MATCH in {dt}s ***", flush=True) + else: + print(f" [{k}/{len(funcs)}] {fn}: no match ({dt}s)", flush=True) + print(f"\nPERMUTER YIELD: {won}/{len(funcs)} closed -> .run/permuter-winners (gate these whole-binary)") + + +if __name__ == "__main__": + main()