BFM-decomp/tools/frontier_classify.py

#!/usr/bin/env python3
"""frontier_classify.py — classify EVERY remaining open stub by its TRUE BLOCKER, deterministically.

WHY THIS EXISTS (P31 S75). "69 functions left" is a count of stubs, not a measure of difficulty, and
this session proved the difference is enormous: five functions carried as a five-session "codegen
wall" were one jump-table extent bug, and `ov_SC03_108:func_8016AE5C` -- logged as
"match_one MATCH but the whole-binary gate rejected -- CAUSE NOT DETERMINED" -- banked with a
byte-perfect body the moment the carve was fixed. A frontier list that does not name the BLOCKER
routes drafting agents at problems that are not drafting problems.

Everything here is computed from artifacts already on disk (R33 / offline-tooling-first): the .s,
the sig registries, the backlog, the corpus. Zero tokens, no agents, no builds.

    tools/frontier_classify.py [--json out.json]
"""
import argparse, collections, glob, importlib.util, json, os, re, subprocess, sys

REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(REPO, 'tools'))

def _load(m, rel):
    s = importlib.util.spec_from_file_location(m, os.path.join(REPO, rel))
    x = importlib.util.module_from_spec(s); sys.modules[m] = x; s.loader.exec_module(x); return x

LINKED = set('apicard1 apicard2 apicard3 apicard4 libc2_1 libc2_2 libcd1 libcd2 libetc libgpu '
             'libgs1 libgs2 libgs3 libgs4 libgs5 libgs6 libgte1 libgte10 libgte11 libgte12 libgte13 '
             'libgte14 libgte15 libgte16 libgte17 libgte18 libgte19 libgte2 libgte20 libgte21 '
             'libgte22 libgte3 libgte4 libgte5 libgte6 libgte7 libgte8 libgte9 libmcrd1 libmcrd2 '
             'snd1 snd2 snd3 snd4 snd5 snd6 snd7 snd8 snd9'.split())
INS = re.compile(r'\s*/\* [0-9A-F]+ [0-9A-F]{8} ')



def fleet_binaries():
    """The REAL fleet, derived from the Makefile + the two .mk lists (R33 — never a hardcoded list)."""
    out = {'main', 'resident'}
    for f, var in (('config/overlays.mk', 'OVERLAY_BINARIES'), ('config/modules.mk', 'MODULE_BINARIES')):
        p = os.path.join(REPO, f)
        if os.path.exists(p):
            m = re.search(rf'^{var}\s*:=\s*(.*)$', open(p).read(), re.M)
            if m:
                out |= set(m.group(1).split())
    return out


def open_stubs():
    """[(fn, binary, path, asm_path, nins)] for every live non-LINKED INCLUDE_ASM stub."""
    out = []
    for root, _d, files in os.walk(os.path.join(REPO, 'src')):
        for f in files:
            if not f.endswith('.c') or f[:-2] in LINKED:
                continue
            p = os.path.join(root, f)
            rel = os.path.relpath(root, os.path.join(REPO, 'src'))
            binary = 'main' if rel == '.' else rel
            for line in open(p, errors='ignore'):
                m = re.match(r'\s*INCLUDE_ASM\("([^"]+)",\s*(\w+)', line)
                if not m:
                    continue
                a = os.path.join(REPO, m.group(1), m.group(2) + '.s')
                n = sum(1 for l in open(a, errors='ignore') if INS.match(l)) if os.path.exists(a) else 0
                out.append((m.group(2), binary, os.path.relpath(p, REPO), a, n))
    return out


def asm_facts(asm_path):
    """Structural tells read straight from the function's own .s."""
    if not os.path.exists(asm_path):
        return {}
    txt = open(asm_path, errors='ignore').read()
    return {
        'jtbl':  bool(re.search(r'\bjtbl_[0-9A-Fa-f]{8}\b', txt)),   # owns a switch jump table
        'jr':    bool(re.search(r'\bjr\s+\$(?!ra)', txt)),           # indirect jump (dispatch)
        'jalr':  bool(re.search(r'\bjalr\b', txt)),                  # indirect call
        'mult':  bool(re.search(r'\b(mult|multu|div|divu)\b', txt)),
        'float': bool(re.search(r'\b(cop2|mtc2|mfc2|ctc2|cfc2)\b', txt)),
        'calls': len(re.findall(r'\bjal\s+\w', txt)),
    }


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--json', default=None)
    a = ap.parse_args()

    corpus = _load('corpus', 'tools/corpus.py')
    stubs = open_stubs()

    # ---- evidence layers, all pre-existing artifacts -------------------------------------------
    # BEST closeness, not LAST — and keep the draft that ACHIEVED it (P31 S75).
    #
    # `.run/backlog.jsonl` is APPEND-ONLY: a function accumulates one row per attempt, across every
    # lane and session. Taking the last row means taking whichever lane wrote most recently, which is
    # evidence about THAT LANE'S SEED, not about the function. Two redraft agents caught this within
    # an hour of each other:
    #
    #   func_8017DB98 — brief said "closeness 115, materially wrong, re-derive". journal_notes showed
    #                   THREE of six recorded attempts were already MATCH; attempt #6 records that the
    #                   pack's warm-start had been the OLDER, WORSE shard. A warm-start regression
    #                   looks exactly like an unsolved function from inside the wave.
    #   func_800CB00C — "the 'materially wrong 168' and a byte-exact closeness-0 body were two
    #                   different files for the same function."
    #
    # So a function with a banked-quality draft in its history was being classified F-FAR (redraft)
    # and would have been handed to an agent to re-derive work that was already done. MIN is the
    # honest reduction; `close_last` is kept alongside so a large gap between them is itself visible
    # as a warm-start regression signal.
    close, close_last, verdict, draft, attempts = {}, {}, {}, {}, collections.Counter()
    bl = os.path.join(REPO, '.run/backlog.jsonl')
    if os.path.exists(bl):
        for ln in open(bl):
            try: r = json.loads(ln)
            except Exception: continue
            n = r.get('name')
            if not n: continue
            attempts[n] += 1
            c = r.get('closeness')
            if c is not None:
                close_last[n] = c
                if n not in close or c < close[n]:
                    close[n] = c
                    if r.get('best_draft'):
                        draft[n] = r['best_draft']      # the draft that ACHIEVED the best score
            if r.get('where_stuck'): verdict[n] = r['where_stuck']
            if r.get('best_draft') and n not in draft: draft[n] = r['best_draft']
    for p in glob.glob(os.path.join(REPO, '.run/harvest_failed.*.classified.txt')):
        for ln in open(p):
            q = ln.rstrip('\n').split('\t')
            if len(q) >= 2 and q[0] not in verdict:
                verdict[q[0]] = q[1]

    # SECOND EVIDENCE SOURCE: the agent journals (R34 — a disagreeing oracle, not a better assertion).
    #
    # `.run/backlog.jsonl` is not the only record of what has been tried, and it is not the best one.
    # `tools/journal_notes.py` mines the AGENT JOURNALS, which carry outcomes the backlog never
    # received. Measured on func_8017DB98 (P31 S75): backlog best == last == 115, so best-vs-last
    # could not help — while the journal holds "Attempt 2 (MATCH · closeness 0) … MATCH 122/122 …
    # BANK BLOCKER is TU plumbing, not the body (§376/§378)" WITH the draft path and the exact line
    # to change. Classified F-FAR ("redraft") on the backlog alone; it is really C-PLUMBING, and an
    # agent would have re-derived a body that was already byte-exact.
    #
    # Cheap: one subprocess per function that has no closeness-0 already, and only for the functions
    # we are about to classify.
    def journal_best(fn):
        """(closeness, draft_path) from the agent journals, or (None, None)."""
        try:
            r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/journal_notes.py'),
                                '--fn', fn], capture_output=True, text=True, timeout=60, cwd=REPO)
        except (OSError, subprocess.SubprocessError):
            return None, None
        # SPLIT on the attempt marker; do NOT consume past it (P31 S75, third instance of this
        # defect class in one session). The first cut used
        # `re.finditer(r'\*\*Attempt \d+\*\* \(([^)]*)\)(.{0,400})', ..., re.S)` — the 400-char
        # body window SWALLOWS THE NEXT ATTEMPT'S HEADER, so every record that follows another was
        # invisible. On func_8017DB98 that hid attempts 2 AND 6, both `MATCH · closeness 0`, and
        # returned 2 (attempt 1's NEAR) as the best — the exact records the oracle exists to find.
        # A regex that consumes an unbounded body cannot enumerate the items after the first.
        chunks = re.split(r'(?=\*\*Attempt \d+\*\*)', r.stdout)
        best, path = None, None
        for ch in chunks:
            hm = re.match(r'\*\*Attempt \d+\*\* \(([^)]*)\)', ch)
            if not hm:
                continue
            head = hm.group(1)
            cm = (re.search(r'closeness\s*[=:]?\s*(\d+)', head)
                  or re.search(r'closeness\s*[=:]?\s*(\d+)', ch))
            # a header that says MATCH with no number IS closeness 0
            c = int(cm.group(1)) if cm else (0 if re.match(r'\s*MATCH\b', head) else None)
            if c is None:
                continue
            if best is None or c < best:
                best = c
                pm = re.search(r'(\.run/[\w./-]+\.c)', ch)
                path = pm.group(1) if pm else None
        return best, path

    # h_exact reach: is this function byte-identical to code in another binary? (a twin is a REMAP,
    # never a redraft -- §168) and is any twin already MATCHED (i.e. is there a proven exemplar)?
    #
    # THE SIG DIRECTORY IS NOT THE FLEET (P31 S75 — caught by the known-true check, R14). Alongside
    # the 213 real binaries `.run/` also holds `SLUS_007.26` (a STALE duplicate of main under the ROM
    # filename), `resident_image`, and two CROSS-BUILD binaries — `sep8_SLUS_007.26` and
    # `aug31_USA_DEMO.EXE`. Counting those as peers made every main stub look like it had a
    # byte-identical twin ALREADY BANKED somewhere: 38 functions / 7,516 instructions of pure
    # artifact, most of them "proven" in main ITSELF under its other name, the rest in a PROTOTYPE
    # that R13 forbids as verified evidence. Derive the fleet from the Makefile and refuse the rest.
    fleet = fleet_binaries()
    seen_sigs = {os.path.basename(q)[4:-6] for q in glob.glob(os.path.join(REPO, '.run/sig.*.jsonl'))}
    dropped = sorted(seen_sigs - fleet)
    if dropped:
        print(f"[classify] IGNORING {len(dropped)} non-fleet sig registr(ies) as twin evidence: "
              f"{', '.join(dropped)}", file=sys.stderr)
    if not fleet:
        sys.exit("frontier_classify: could not derive BINARIES from the Makefile — refusing to "
                 "classify against an unknown fleet (R32)")
    sig, banked_addrs = {}, set()
    for p in glob.glob(os.path.join(REPO, '.run/sig.*.jsonl')):
        b = os.path.basename(p)[4:-6]
        if b not in fleet:
            continue
        for ln in open(p):
            try: r = json.loads(ln)
            except Exception: continue
            sig.setdefault(int(r['addr'], 16), []).append((b, r.get('h_exact')))
    open_pairs = {(f, b) for f, b, _p, _a, _n in stubs}
    for addr, rows in sig.items():
        for b, _h in rows:
            if (f'func_{addr:08X}', b) not in open_pairs:
                banked_addrs.add((addr, b))

    # Any stored draft on disk, anywhere under .run — ONE pruned walk, not a `.run/S7*` glob.
    #
    # The glob was `.run/S7*/**/*.c`, which is NARROWER THAN THE CLAIM IT SUPPORTED (P31 S75, R32 —
    # the session's recurring defect, committed by me in the very tool written to classify it). Real
    # drafts live in `.run/S69m2/`, `.run/S68m1/`, `.run/s67m1/` (lowercase), `.run/wave_ds2/`,
    # `.run/gate_lane/`, `.run/backlog_drafts/` — none of which match `S7*`. Result: all 32 drafted
    # main functions were classified H-VIRGIN ("never drafted -- genuine new drafting work"), which
    # would have sent agents to redraft 6,328 instructions that already have drafts sitting on disk.
    # The worktrees are pruned because they are full repo copies (7.4 GB) of the SAME sources.
    disk_drafts = collections.defaultdict(list)
    for dp, dn, fns in os.walk(os.path.join(REPO, '.run')):
        dn[:] = [d for d in dn if d != '.git' and not d.startswith('wt_')
                 and d not in ('pgate', 'obj40', 'audit')]
        for f in fns:
            if f.endswith('.c'):
                disk_drafts[f[:-2]].append(os.path.join(dp, f))

    rows = []
    for fn, binary, path, asm, nins in stubs:
        try: addr = int(fn[5:], 16)
        except ValueError: addr = None
        f = asm_facts(asm)
        h = None; twins = []
        if addr in sig:
            me = [x for x in sig[addr] if x[0] == binary]
            h = me[0][1] if me else None
            if h:
                twins = [b for b, hh in sig[addr] if hh == h and b != binary]
        proven_twin = [b for b in twins if (addr, b) in banked_addrs]
        c = close.get(fn)
        jc, jpath = (None, None)
        if c is None or c > 0:                      # only ask when the backlog has no proven body
            jc, jpath = journal_best(fn)
            if jc is not None and (c is None or jc < c):
                c = jc
                if jpath:
                    draft[fn] = jpath
        v = (verdict.get(fn) or '')
        o0 = False
        try: o0 = bool(corpus.is_o0(binary, addr)) if addr else False
        except Exception: pass

        # ---- the blocker ladder: most specific evidence first ----------------------------------
        if proven_twin:
            k = 'A-TWIN-REMAP'          # a byte-identical copy is ALREADY BANKED elsewhere
        elif f.get('jtbl'):
            k = 'B-CARVE'               # owns a switch table -> the S75 carve class
        elif c == 0:
            k = 'C-PLUMBING'            # body proven, TU refuses it
        elif c is not None and c <= 25:
            k = 'D-NEAR'                # permuter fuel, not drafting
        elif o0:
            k = 'E-O0'                  # -O0 island
        elif c is not None and c <= 150:
            k = 'F-FAR'                 # a draft exists but is materially wrong
        elif fn in disk_drafts or fn in draft:
            k = 'G-DRAFTED-UNKNOWN'     # tried, no usable verdict recorded
        else:
            k = 'H-VIRGIN'              # never drafted: genuine new work
        regressed = (close_last.get(fn) is not None and c is not None
                     and close_last[fn] > c + 10)      # a later attempt scored materially worse
        rows.append(dict(fn=fn, binary=binary, nins=nins, klass=k, closeness=c,
                         closeness_last=close_last.get(fn), attempts=attempts.get(fn, 0),
                         warmstart_regression=regressed, journal_closeness=jc,
                         jtbl=f.get('jtbl'), jr=f.get('jr'), jalr=f.get('jalr'), o0=o0,
                         calls=f.get('calls'), twins=len(twins), proven_twin=proven_twin[:3],
                         drafts=len(disk_drafts.get(fn, [])), verdict=v[:90], path=path))

    rows.sort(key=lambda r: (r['klass'], -r['nins']))
    by = collections.defaultdict(list)
    for r in rows: by[r['klass']].append(r)
    NAME = {
        'A-TWIN-REMAP':     'a byte-identical copy is ALREADY BANKED elsewhere -> mechanical remap',
        'B-CARVE':          'owns a switch jump table -> the S75 jtbl_carve class',
        'C-PLUMBING':       'closeness 0: body proven, the TU refuses the signature',
        'D-NEAR':           'closeness <=25 -> permuter fuel, NOT drafting',
        'E-O0':             '-O0 island',
        'F-FAR':            'a draft exists but is materially wrong -> redraft',
        'G-DRAFTED-UNKNOWN':'drafted before, no usable verdict on record',
        'H-VIRGIN':         'never drafted -- genuine new drafting work',
    }
    tot_n = sum(r['nins'] for r in rows)
    print(f"OPEN STUBS: {len(rows)}  ({len({r['fn'] for r in rows})} distinct)  {tot_n:,} instructions\n")
    print(f"{'class':20s} {'fns':>4s} {'ins':>7s}  what it actually needs")
    print('-' * 100)
    for k in sorted(by):
        g = by[k]
        print(f"{k:20s} {len(g):4d} {sum(x['nins'] for x in g):7,d}  {NAME[k]}")
    print('-' * 100)
    print(f"{'TOTAL':20s} {len(rows):4d} {tot_n:7,d}")
    for k in sorted(by):
        print(f"\n== {k} — {NAME[k]}")
        for r in by[k]:
            extra = []
            if r['closeness'] is not None:
                extra.append(f"close={r['closeness']}(best of {r['attempts']})")
            if r.get('journal_closeness') is not None:
                extra.append(f"journal={r['journal_closeness']}")
            if r.get('warmstart_regression'):
                extra.append(f"!!WARMSTART-REGRESSION last={r['closeness_last']}")
            if r['jtbl']: extra.append('jtbl')
            if r['jr']: extra.append('jr')
            if r['o0']: extra.append('-O0')
            if r['twins']: extra.append(f"twins={r['twins']}")
            if r['proven_twin']: extra.append(f"proven@{','.join(r['proven_twin'])}")
            if r['drafts']: extra.append(f"drafts={r['drafts']}")
            print(f"   {r['nins']:5d}  {r['fn']:20s} {r['binary']:14s} {' '.join(extra)}")
            if r['verdict']: print(f"          verdict: {r['verdict']}")
    if a.json:
        json.dump(rows, open(a.json, 'w'), indent=1)
        print(f"\n-> {a.json}")


if __name__ == '__main__':
    main()