BFM-decomp/tools/autopsy.py

#!/usr/bin/env python3
"""autopsy.py — the permuter-failure autopsy: mine the search-closer's FAILURES (Phase-29 Task-13).

docs/hindsight-study.md §7. Our flywheel (R16) has always evolved the offline tooling REACTIVELY
FROM SUCCESSES: a human or an LLM cracks a function, we distil the idiom into the cookbook and the
permuter weights. The permuter's FAILURES — every near-miss it plateaued on — are a larger and
completely unused signal. This tool turns that log into a corpus, classifies it deterministically,
and routes each cluster to one of:

  missing-transform  the byte-match IS reachable by local mutation but the permuter's mutation set
                     never tried the needed move -> extend permuter_weights. A PERMANENT offline
                     win: that whole class now closes with no LLM. The highest-value bucket.
  mis-classified     the recorded class is wrong, so the search was directed at the wrong levers
                     -> fix the routing (free).
  seed-structural    the C STRUCTURE is wrong; local mutation cannot introduce it -> one LLM seed
                     rewrite, then hand back to the permuter. The LLM spent once, minimally.
  genuine-wall       compiler-internal at this tier -> file WITH A REASON and an expiry (R35: the
                     verdict dies when the gate or the model tier changes). Never a silent give-up.

WHY A COLLECT STEP EXISTS AT ALL. Task-12 plumbed structured-residual telemetry into the backlog,
but telemetry only fills records written AFTER it landed: 1 of 6,169 records carries a `residual`
and 0 carry `passes_tried`. Meanwhile 1,752 open near-misses have their best draft ON DISK and
their target .s ON DISK, so the residual is not missing — it is one second of CPU away, and
recomputing it is strictly better than trusting a stored one (uncapped, current toolchain, current
tree). `collect` recomputes the whole corpus through the EXISTING validated compile path
(tools/match_one.py, R33 — never a second copy of the pipeline).

THE TWO INPUTS THAT MUST BE DERIVED, NOT GUESSED — both are silent-artefact generators:
  * the asm subdir  — an overlay has up to 14 TUs; the wrong one scores a draft against a
                      DIFFERENT function's asm (Phase 26-A).
  * the -O0 flag    — an -O0 TU's target bytes are -O0; scoring its draft at -O2 mismatches
                      everywhere no matter how correct the C is (§53/§54, Task-1's phantom "~3%").
Both come from corpus.py: the stub's self-describing INCLUDE_ASM line, and the Makefile's own -O0
rules. A record whose stub cannot be found is REPORTED, never silently dropped (R32) — a skipped
target produces silence, not an error, which is how 91.6% of remaining gain once went invisible.

  tools/autopsy.py collect [--limit N] [--sample N] [-j 12]   -> .run/autopsy/residuals.jsonl
  tools/autopsy.py report  [--top N]                          -> docs/autopsy.md + stdout
"""
import argparse
import collections
import json
import os
import random
import subprocess
import sys
from concurrent.futures import ThreadPoolExecutor

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import backlog
import corpus
import residual_class

REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
PY = ".venv/bin/python"
OUT = ".run/autopsy/residuals.jsonl"
MD = "docs/autopsy.md"
DEFAULT_BINARY = "ov_SC01_077"          # the legacy-record convention (backlog.load_best)


# --------------------------------------------------------------------------------------------
# collect
# --------------------------------------------------------------------------------------------
def _targets():
    """Open backlog near-misses with a draft on disk, plus the reason any record is excluded.

    Returns (targets, excluded) where excluded is a Counter — every input is accounted for, so a
    shrinking corpus can never be mistaken for a shrinking frontier (R32)."""
    todo, skip = [], collections.Counter()
    for r in backlog.load_best():
        name, draft = r.get("name"), r.get("best_draft")
        if not name:
            skip["no name"] += 1
            continue
        if not draft:
            skip["no saved draft"] += 1
            continue
        if not os.path.exists(os.path.join(REPO, draft)):
            skip["draft path gone"] += 1
            continue
        todo.append(r)
    return todo, skip


def _resolve(rec):
    """(asm_dir, o0, src_path) for a record, DERIVED — or (None, None, reason)."""
    binary = rec.get("binary") or DEFAULT_BINARY
    try:
        st = corpus.stubs(binary)
    except corpus.CorpusError as e:
        return None, None, "corpus error (%s): %s" % (binary, str(e).split("\n")[0][:80])
    hit = next((s for s in st.values() if s.symbol == rec["name"]), None)
    if hit is None:
        return None, None, "not a live stub in %s" % binary
    return hit.asm_dir, corpus.is_o0(hit.path), hit.path


def _one(rec):
    """Recompile one draft through match_one --json and return an enriched autopsy row."""
    name = rec["name"]
    binary = rec.get("binary") or DEFAULT_BINARY
    asm, o0, why = _resolve(rec)
    row = {"name": name, "addr": rec.get("addr"), "binary": binary,
           "reach": rec.get("reach"), "nins_logged": rec.get("nins"),
           "closeness_logged": rec.get("closeness"), "klass_logged": rec.get("klass"),
           "where_logged": rec.get("where_stuck"), "source": rec.get("source"),
           "draft": rec.get("best_draft")}
    if asm is None:
        row.update({"status": "unresolved", "note": why})
        return row
    row.update({"asm_dir": asm, "o0": bool(o0), "src": why})
    cmd = [PY, "tools/match_one.py", name, "--c", rec["best_draft"], "--asm-subdir", asm, "--json"]
    if o0:
        cmd.append("--o0")
    try:
        p = subprocess.run(cmd, capture_output=True, text=True, cwd=REPO, timeout=180)
    except subprocess.TimeoutExpired:
        row.update({"status": "timeout"})
        return row
    lines = [ln for ln in p.stdout.strip().splitlines() if ln.strip()]
    try:
        j = json.loads(lines[-1])
    except Exception:
        # a compile failure (CPP/CC1/MASPSX/AS FAIL) is itself a CLASS — the draft does not build
        # in isolation, which is plumbing, not codegen. Keep the reason; never drop the row.
        row.update({"status": "nobuild",
                    "note": (lines[0] if lines else (p.stderr.strip().splitlines() or ["?"])[0])[:120]})
        return row
    v = j.get("verdict") or {}
    row.update({"status": j.get("status"), "closeness": j.get("closeness"), "nins": j.get("nins"),
                "klass": v.get("klass"), "bucket": v.get("bucket"), "profile": v.get("profile"),
                "sig": v.get("sig"), "detail": v.get("detail"),
                "disagree": v.get("closeness_disagree"), "residual": j.get("residual")})
    return row


def collect(limit=None, sample=None, jobs=12):
    todo, skip = _targets()
    if sample:
        random.seed(1729)                       # fixed seed: a re-run samples the SAME functions
        todo = random.sample(todo, min(sample, len(todo)))
    if limit:
        todo = todo[:limit]
    print("autopsy collect: %d target(s)%s; excluded %s"
          % (len(todo), " (sampled)" if sample else "", dict(skip) or "{}"), flush=True)
    rows, done = [], 0
    with ThreadPoolExecutor(max_workers=jobs) as ex:
        for row in ex.map(_one, todo):
            rows.append(row)
            done += 1
            if done % 100 == 0:
                print("  ... %d/%d" % (done, len(todo)), flush=True)
    os.makedirs(os.path.join(REPO, os.path.dirname(OUT)), exist_ok=True)
    with open(os.path.join(REPO, OUT), "w") as f:
        for r in rows:
            f.write(json.dumps(r) + "\n")
    # COVERAGE + the second-oracle cross-check, both loud (R32/R34)
    st = collections.Counter(r.get("status") for r in rows)
    dis = [r for r in rows if r.get("disagree")]
    err = [r for r in rows if r.get("klass") == "ERROR"]
    print("\nwrote %s  (%d rows)" % (OUT, len(rows)))
    print("status:", dict(st))
    if len(rows) != len(todo):
        raise SystemExit("COVERAGE DEFECT: %d targets in, %d rows out" % (len(todo), len(rows)))
    if dis:
        raise SystemExit("ORACLE DISAGREEMENT on %d row(s): residual_class closeness != "
                         "structured_diff closeness. Fix before trusting any verdict.\n  %s"
                         % (len(dis), ", ".join(r["name"] for r in dis[:8])))
    if err:
        raise SystemExit("CLASSIFIER ERROR on %d row(s): %s"
                         % (len(err), err[0].get("detail", {}).get("error")))
    print("cross-check: closeness agrees with masked_diff.structured_diff on all %d built rows; "
          "0 classifier errors." % sum(1 for r in rows if r.get("klass")))
    return rows


# --------------------------------------------------------------------------------------------
# report
# --------------------------------------------------------------------------------------------
def _rows():
    p = os.path.join(REPO, OUT)
    if not os.path.exists(p):
        raise SystemExit("no %s — run: tools/autopsy.py collect" % OUT)
    return [json.loads(l) for l in open(p) if l.strip()]


def verdicts():
    """{fn_name: row} for the collected corpus, or {} if it has not been collected.

    The consumer-facing accessor (tools/grinder.py). Returns {} rather than raising so a missing
    corpus degrades to the previous undirected behaviour instead of breaking the daemon — but the
    caller must SAY which mode it is in (R32: a filter that silently does nothing is the defect).

    THE ROUTE IS RE-DERIVED HERE, NEVER READ FROM THE FILE (R33, Phase 29 T54). `klass` is the
    measurement and is trusted; `profile`/`bucket` are a policy lookup over it, and a stored lookup
    output makes a weeks-old file authoritative for a decision `residual_class._ROUTE` owns — a
    route correction would be inert until someone re-ran the whole collect, and the stale row would
    silently out-vote the live table. `route_for` reproduces the stored routes exactly under an
    unchanged table (verified 1610/1610), so this is a no-op except where the table has since been
    corrected — which is precisely when it must not be a no-op."""
    p = os.path.join(REPO, OUT)
    if not os.path.exists(p):
        return {}
    out = {}
    for line in open(p):
        if line.strip():
            r = json.loads(line)
            if r.get("name") and r.get("klass"):
                r["profile"], r["bucket"] = residual_class.route_for(r["klass"], r.get("detail"))
                out[r["name"]] = r
    return out


def report(top=25):
    rows = _rows()
    built = [r for r in rows if r.get("klass")]
    by_bucket = collections.Counter(r["bucket"] for r in built)
    by_klass = collections.Counter(r["klass"] for r in built)
    by_status = collections.Counter(r.get("status") for r in rows)

    # weight by REACH: a class that blocks a reach-138 core is worth ~138 members
    reach_w = collections.Counter()
    for r in built:
        reach_w[r["klass"]] += (r.get("reach") or 1)

    # the SECOND ORACLE (R34): where the deterministic verdict disagrees with the human/LLM label
    # a wave agent typed. Neither is authoritative — the DISAGREEMENT is the finding, and it is
    # precisely the autopsy's `mis-classified` bucket.
    agree, disagree = 0, []
    for r in built:
        kl = (r.get("klass_logged") or "").strip()
        if not kl or kl.upper() in ("WAVE", "GIANT", "STUB", "OTHER", "PINS", "STRUCT"):
            continue                                    # generic tags carry no class claim
        want = residual_class._ROUTE.get(r["klass"], (None, None))[0]
        import permuter_weights
        got = permuter_weights.classify(kl, r.get("where_logged") or "")
        if want and got and want == got:
            agree += 1
        elif want and got:
            disagree.append((r["name"], kl, got, r["klass"], want))

    clusters = collections.Counter(r["sig"] for r in built)

    L = []
    L.append("# Autopsy — the permuter's failures, classified\n")
    L.append("> Generated by `tools/autopsy.py report` from `.run/autopsy/residuals.jsonl` "
             "(built by `collect`, which recompiles every open backlog draft through "
             "`tools/match_one.py` and classifies the FULL residual with `tools/residual_class.py`).\n")
    L.append("> A verdict here is a **candidate**, never a bank: the whole-binary byte-gate "
             "remains the sole arbiter (G3/P9).\n")
    L.append("**Corpus:** %d record(s) · %d classified · status %s\n"
             % (len(rows), len(built), dict(by_status)))

    L.append("\n## Buckets — what tool each failure actually wants\n")
    L.append("| bucket | fns | reach-weighted | meaning |")
    L.append("|--------|----:|---------------:|---------|")
    MEAN = {"permuter": "local mutation CAN reach it — permuter fuel (a plateau here is a "
                        "**missing-transform** candidate)",
            "structural": "local mutation CANNOT introduce it — wants a C-level idiom, not CPU",
            "integration": "byte-correct standalone; blocked on plumbing (§58/§59 reconcile ladder)",
            "redraft": "the stored draft is NOT this function — un-attempted work misfiled as a "
                       "near-miss; route to a fresh crack, not to a wall",
            "unknown": "the residue the LLM tier is for"}
    for b, n in by_bucket.most_common():
        rw = sum(r.get("reach") or 1 for r in built if r["bucket"] == b)
        L.append("| %s | %d | %d | %s |" % (b, n, rw, MEAN.get(b, "")))

    L.append("\n## Classes\n")
    L.append("| class | fns | reach-weighted | bucket | permuter profile |")
    L.append("|-------|----:|---------------:|--------|------------------|")
    for k, n in by_klass.most_common():
        prof, bucket = residual_class._ROUTE.get(k, (None, "?"))
        L.append("| %s | %d | %d | %s | %s |" % (k, n, reach_w[k], bucket, prof or "—"))

    L.append("\n## Top clusters (the batch-autopsy work items)\n")
    L.append("One representative per signature diagnoses the whole cluster — that is what makes "
             "the LLM pass a *batch* autopsy rather than N per-function reads.\n")
    L.append("| # | signature | fns | reach-weighted | example |")
    L.append("|--:|-----------|----:|---------------:|---------|")
    for i, (sg, n) in enumerate(clusters.most_common(top), 1):
        members = [r for r in built if r["sig"] == sg]
        rw = sum(r.get("reach") or 1 for r in members)
        ex = min(members, key=lambda r: (r.get("closeness") or 10**9))
        L.append("| %d | `%s` | %d | %d | %s (close=%s, %s ins) |"
                 % (i, sg, n, rw, ex["name"], ex.get("closeness"), ex.get("nins")))

    L.append("\n## Second oracle — deterministic verdict vs the recorded label\n")
    L.append("The recorded `klass` is what a wave agent typed months ago; the verdict is derived "
             "from today's bytes. Neither is authoritative — **the disagreement is the finding**, "
             "and it is exactly the `mis-classified` bucket (a wrongly-labelled residual sends the "
             "directed search at the wrong levers, so it plateaus and then reads as a wall).\n")
    L.append("- labelled records comparable: **%d** · agree **%d** · disagree **%d**\n"
             % (agree + len(disagree), agree, len(disagree)))
    if disagree:
        L.append("| fn | recorded label | label routes to | measured class | measured routes to |")
        L.append("|----|----------------|-----------------|----------------|--------------------|")
        for nm, kl, got, mk, want in disagree[:top]:
            L.append("| %s | `%s` | %s | %s | %s |" % (nm, kl, got, mk, want))

    open(os.path.join(REPO, MD), "w").write("\n".join(L) + "\n")

    print("\n== buckets ==")
    for b, n in by_bucket.most_common():
        print("  %-12s %5d fns  reach-weighted %7d   %s"
              % (b, n, sum(r.get("reach") or 1 for r in built if r["bucket"] == b), MEAN.get(b, "")))
    print("\n== classes ==")
    for k, n in by_klass.most_common():
        print("  %-18s %5d   reach-weighted %7d   -> %s / %s"
              % (k, n, reach_w[k], residual_class._ROUTE.get(k, (None, "?"))[1],
                 residual_class._ROUTE.get(k, (None, None))[0] or "—"))
    print("\n== top clusters ==")
    for sg, n in clusters.most_common(top):
        print("  %-34s %4d" % (sg, n))
    print("\n== second oracle ==")
    print("  labelled+comparable %d: agree %d, disagree %d" % (agree + len(disagree), agree, len(disagree)))
    for nm, kl, got, mk, want in disagree[:12]:
        print("    %-18s recorded %-16s -> %-9s | measured %-17s -> %s" % (nm, kl, got, mk, want))
    print("\nwrote %s" % MD)


def main():
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    sub = ap.add_subparsers(dest="cmd", required=True)
    c = sub.add_parser("collect")
    c.add_argument("--limit", type=int)
    c.add_argument("--sample", type=int, help="random SAMPLE (fixed seed) — probe before scaling")
    c.add_argument("-j", "--jobs", type=int, default=12)
    r = sub.add_parser("report")
    r.add_argument("--top", type=int, default=25)
    a = ap.parse_args()
    if a.cmd == "collect":
        collect(a.limit, a.sample, a.jobs)
    else:
        report(a.top)


if __name__ == "__main__":
    main()