#!/usr/bin/env python3 """idiom_loop.py — Phase-21 NEVERENDING idiom-learning meta-loop (the wall->next-idiom->fuel engine). The harvest model: each session drains the EASY reach-134 wave fuel until ~80% is consumed (a WALL), then learns the next gcc-quirk IDIOM that unlocks the highest-leverage residual, and hands off to a fresh session (clean context) to crack + harvest it. Repeat forever (Phase 21 ends only when Drew says). --fuel-status : how much FRESH cached reach-134 wave fuel remains (the wall detector). "WALL" when fresh ≤ --wall (default 12) -> run --assess. --assess : cluster the backlog near-misses by gcc-quirk class, rank by reach-134 leverage, and name (a) the next IDIOM to LEARN (top close>0 codegen class) and (b) the cheap DETERMINISTIC recovery lever (close=0 plumbing/decl). --gen-fuel : write the fuel manifest (.run/auto/idiom_fuel.json) for : each fn's addr/nins/reach/closeness/where_stuck/best_draft/asm/ghidra_c — the research+harvest fuel for the next idiom-learning session. A fresh session then: research the gcc pass (R17, tools/reference/gcc-papermario) -> crack the lowest- closeness exemplar -> distill the idiom (cookbook) -> re-wave the class with it ×134 -> at ~80% of THAT fuel, --assess again. Knowledge compounds toward 100% decomp. """ import argparse, collections, glob, json, os, re ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) BACKLOG = os.path.join(ROOT, ".run/backlog.jsonl") GHIDRA_C = ".run/ghidra_c" # asm subdirs to probe per region (first existing wins) ASM_DIRS = ["asm/ov_SC01_077/nonmatchings/ov_SC01_077_a", "asm/ov_SC01_077/nonmatchings/ov_SC01_077", "asm/ov_SC01_077/nonmatchings/ov_SC01_077_o0"] def _open_stubs(): """func names STILL INCLUDE_ASM stubs in ov_SC01_077*.c (unmatched). A name absent => banked/gone (mirrors backlog._matched_now; the drop-now-matched P9 honesty).""" stub_re = re.compile(r"INCLUDE_ASM\([^,]+,\s*(\w+)\)") stubs = set() for f in glob.glob(os.path.join(ROOT, "src/ov_SC01_077/ov_SC01_077*.c")): try: stubs.update(stub_re.findall(open(f).read())) except OSError: pass return stubs def load_backlog(): """last record per fn (the freshest near-miss state); only still-unbanked, STILL-STUBBED near/fail. Drops fns banked SINCE their last ledger record (stale status 'near' but no longer a stub) — the same drop-now-matched honesty backlog.render applies (P9); without it every assess count over-inflates with already-matched fns (cont.7: func_8016EDEC/EE40 etc.).""" recs = {} if os.path.exists(BACKLOG): for line in open(BACKLOG): line = line.strip() if not line: continue try: r = json.loads(line) except ValueError: continue if r.get("name"): recs[r["name"]] = r open_stubs = _open_stubs() return [r for r in recs.values() if r.get("status") in ("near", "failed") and r.get("name") in open_stubs] def klass_of(r): k = r.get("klass") or "unknown" return k.strip().split()[0] if k.strip() else "unknown" def asm_for(name): for d in ASM_DIRS: p = os.path.join(ROOT, d, name + ".s") if os.path.exists(p): return os.path.join(d, name + ".s") return None def cluster(rows): by = collections.defaultdict(lambda: {"n": 0, "r134": 0, "r1": 0, "close": []}) for r in rows: d = by[klass_of(r)] d["n"] += 1 rch = r.get("reach") or 0 if rch and rch >= 134: d["r134"] += 1 elif rch == 1: d["r1"] += 1 c = r.get("closeness") if isinstance(c, int): d["close"].append(c) return by def med(xs): xs = sorted(xs) return xs[len(xs) // 2] if xs else None def cmd_assess(rows): by = cluster(rows) print(f"{'class':<16}{'count':>6}{'r134':>6}{'r1':>5}{'med_close':>10}") for k, d in sorted(by.items(), key=lambda kv: -kv[1]["r134"]): print(f"{k:<16}{d['n']:>6}{d['r134']:>6}{d['r1']:>5}{str(med(d['close'])):>10}") # the next IDIOM to LEARN = the top class whose residual is real codegen (median closeness > 0), # excluding the deterministic/plumbing buckets (those are a recovery-tool job, not a gcc idiom). RECOVERY = {"plumbing", "loose-typing"} codegen = [(k, d) for k, d in by.items() if k not in RECOVERY and k not in ("unknown", "GIANT") and (med(d["close"]) or 0) > 0] codegen.sort(key=lambda kv: -kv[1]["r134"]) # the cheap DETERMINISTIC lever = close=0 reach-134 fns (byte-correct, declaration-blocked). # HONESTY (cont.7, byte-proven): a close=0 fn that ALREADY went through a recovery gate (source # 'recover-*') and is STILL unbanked is the §16/§20 DEF-SIDE loose-typing WALL, not cheap fuel — # re-running recovery on it banks 0 (cont.6 main 0/40 + cont.7 _a 0/20). match_one reports close=0 # (masked MATCH) but the whole-binary gate rejects the def-sig conflict. Only NEVER-recovery-gated # close=0 fns are genuine ~0-token fuel; split the count so the loop stops re-burning the wall. c0 = [r for r in rows if (r.get("reach") or 0) >= 134 and r.get("closeness") == 0] det_walled = sum(1 for r in c0 if str(r.get("source", "")).startswith("recover")) det_fresh = len(c0) - det_walled print() if codegen: k, d = codegen[0] print(f"NEXT IDIOM TO LEARN -> '{k}' ({d['r134']} reach-134 fns, median {med(d['close'])} ins off)") print(f" gen its fuel: .venv/bin/python tools/idiom_loop.py --gen-fuel {k}") print(f"DETERMINISTIC RECOVERY LEVER -> {det_fresh} FRESH close=0 reach-134 fns (never recovery-gated) " f"= the only genuine ~0-token ×134 recovery fuel; gate these first.") print(f" (+{det_walled} close=0 are recovery-FAILED = the def-side loose-typing WALL (cont.6/7): do NOT " f"re-run recovery — they need a per-fn type-split or hand-finish, NOT a free bank.)") def cmd_gen_fuel(rows, klass, out): fuel = [] for r in rows: if klass_of(r) != klass: continue name = r["name"] addr = r.get("addr") fuel.append({ "name": name, "addr": (f"0x{addr:08x}" if isinstance(addr, int) else addr), "nins": r.get("nins"), "reach": r.get("reach"), "closeness": r.get("closeness"), "where_stuck": r.get("where_stuck"), "best_draft": r.get("best_draft"), "asm": asm_for(name), "ghidra_c": (f"{GHIDRA_C}/{name}.c" if os.path.exists(os.path.join(ROOT, GHIDRA_C, name + ".c")) else None), }) # closest-first: the lowest-closeness exemplar is the one to crack the idiom on fuel.sort(key=lambda f: (f["closeness"] if isinstance(f["closeness"], int) else 9999)) os.makedirs(os.path.dirname(os.path.join(ROOT, out)), exist_ok=True) open(os.path.join(ROOT, out), "w").write(json.dumps(fuel, indent=1)) cached = sum(1 for f in fuel if f["ghidra_c"] and f["asm"]) r134 = sum(1 for f in fuel if (f["reach"] or 0) >= 134) print(f"wrote {len(fuel)} '{klass}' fns -> {out} ({r134} reach-134, {cached} cached/draftable)") print("exemplars to crack the idiom on (closest first):") for f in fuel[:5]: print(f" {f['name']} close={f['closeness']} reach={f['reach']} nins={f['nins']}: " f"{(f['where_stuck'] or '')[:90]}") def cmd_fuel_status(wall): """remaining FRESH cached reach-134 fns wave_targets would serve (not yet in the backlog).""" seen = set() if os.path.exists(BACKLOG): for line in open(BACKLOG): try: seen.add(json.loads(line).get("name")) except ValueError: pass # approximate fresh fuel = cached ghidra_c fns not yet attempted, with an _a/main asm. cdir = os.path.join(ROOT, GHIDRA_C) cached = {f[:-2] for f in os.listdir(cdir)} if os.path.isdir(cdir) else set() fresh = [n for n in cached if n not in seen and asm_for(n)] state = "WALL — run --assess" if len(fresh) <= wall else "fuel OK — keep waving" print(f"fresh cached unbanked fuel ≈ {len(fresh)} (wall ≤ {wall}) -> {state}") def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--assess", action="store_true") ap.add_argument("--gen-fuel", metavar="CLASS") ap.add_argument("--fuel-status", action="store_true") ap.add_argument("--out", default=".run/auto/idiom_fuel.json") ap.add_argument("--wall", type=int, default=12) a = ap.parse_args() rows = load_backlog() if a.fuel_status: cmd_fuel_status(a.wall) elif a.gen_fuel: cmd_gen_fuel(rows, a.gen_fuel, a.out) elif a.assess: cmd_assess(rows) else: ap.print_help() if __name__ == "__main__": main()