BFM-decomp/tools/difficulty.py

#!/usr/bin/env python3
"""Difficulty-ranked inventory of UNMATCHED functions — the harvest queue for matching.

Ghidra-free: parses each unmatched stub's asm/nonmatchings/800/<name>.s for size, control
flow, jump-table presence, and call count, then ranks easiest-first. Jump-table functions
score high (they need the deferred rodata-island workflow, Task 2'). Writes the actionable
easy queue to docs/difficulty.md and the full CSV to .run/difficulty.csv.

Usage: tools/difficulty.py [TOP] [--binary <alias>]   (TOP default 120; binary default main)
"""
import re, sys, pathlib

ROOT = pathlib.Path(__file__).resolve().parent.parent

# Per-binary config (Phase 9). main = the retail EXE (current paths = no-op default).
# The overlay src/asm subtree LAYOUT is a Phase-10 decision (main = the originals).
BINARIES = {
    "main": dict(src="src", asm="asm/nonmatchings", md="docs/difficulty.md", csv=".run/difficulty.csv"),
    "resident": dict(src="src/resident", asm="asm/resident/nonmatchings",
                     md="docs/difficulty.resident.md", csv=".run/difficulty.resident.csv"),
    "ov_SC01_077": dict(src="src/ov_SC01_077", asm="asm/ov_SC01_077/nonmatchings",
                        md="docs/difficulty.ov_SC01_077.md", csv=".run/difficulty.ov_SC01_077.csv"),
    # <<< overlays: tools/new_overlay.sh inserts ov_* entries above this line (Phase 13) >>>
}
SRCS = ASM_ROOT = MD = CSV = None    # set by main() from --binary

def find_s(name):
    """Locate <name>.s in any asm/nonmatchings/<seg>/ subdir."""
    for p in sorted(ASM_ROOT.glob(f"*/{name}.s")):
        return p
    return None

INSTR = re.compile(r'^\s*/\*\s*[0-9A-Fa-f]+\s+([0-9A-Fa-f]+)\s+[0-9A-Fa-f]+\s*\*/\s+([a-z][a-z0-9.]*)')
BRANCH = re.compile(r'^b(eq|ne|gez|gtz|lez|ltz|nez|eqz|c1t|c1f|gezal|ltzal)?z?$')

def is_data_blob(txt):
    return 'glabel' not in txt and 'jlabel' not in txt and 'dlabel' in txt

def unmatched_stubs():
    """INCLUDE_ASM names that are real functions (exclude data-blobs) and not inside NON_MATCHING."""
    out = []
    for src in SRCS:
        lines = src.read_text().split('\n'); n = len(lines); i = 0
        while i < n:
            s = lines[i].strip()
            if s.startswith('#ifdef NON_MATCHING'):
                while i < n and not lines[i].strip().startswith('#endif'):
                    i += 1
                i += 1; continue
            m = re.match(r'INCLUDE_ASM\("[^"]+",\s*(\w+)\)', s)
            if m:
                out.append(m.group(1))
            i += 1
    return out

def analyze(name):
    p = find_s(name)
    if p is None: return None
    txt = p.read_text()
    if is_data_blob(txt): return None                     # not a function
    nins = branches = ncalls = 0; last_vaddr = None; jtbl = False
    if 'jtbl_' in txt: jtbl = True
    for ln in txt.splitlines():
        m = INSTR.match(ln)
        if not m: continue
        nins += 1; last_vaddr = int(m.group(1), 16); mn = m.group(2)
        if mn == 'jal': ncalls += 1
        elif mn == 'j': branches += 1
        elif mn == 'jr' and '$ra' not in ln: jtbl = True   # indirect jump = switch/jumptable
        elif BRANCH.match(mn): branches += 1
    score = nins + 3*branches + 25*(1 if jtbl else 0) + 2*ncalls
    return dict(name=name, nins=nins, branches=branches, ncalls=ncalls,
                jtbl=jtbl, leaf=(ncalls == 0), score=score)

def main():
    import argparse
    global SRCS, ASM_ROOT, MD, CSV
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("top", nargs="?", type=int, default=120)
    ap.add_argument("--binary", default="main", choices=list(BINARIES))
    a = ap.parse_args()
    cfg = BINARIES[a.binary]
    SRCS = sorted((ROOT / cfg["src"]).glob("*.c"))
    ASM_ROOT = ROOT / cfg["asm"]
    MD = ROOT / cfg["md"]
    CSV = ROOT / cfg["csv"]
    top = a.top
    rows = [r for r in (analyze(n) for n in unmatched_stubs()) if r]
    rows.sort(key=lambda r: (r['score'], r['name']))

    CSV.parent.mkdir(exist_ok=True)
    CSV.write_text("name,score,nins,branches,ncalls,jtbl,leaf\n" +
                   "".join(f"{r['name']},{r['score']},{r['nins']},{r['branches']},"
                           f"{r['ncalls']},{int(r['jtbl'])},{int(r['leaf'])}\n" for r in rows))

    trivial = sum(1 for r in rows if r['nins'] <= 5)
    leaves  = sum(1 for r in rows if r['leaf'] and not r['jtbl'])
    jtbls   = sum(1 for r in rows if r['jtbl'])
    md = ["# Unmatched difficulty inventory  (generated by tools/difficulty.py — harvest queue)",
          "",
          f"unmatched functions : {len(rows)}",
          f"trivial (<=5 ins)   : {trivial}",
          f"non-jtbl leaves     : {leaves}    (best harvest targets)",
          f"jump-table funcs    : {jtbls}    (deferred — need the rodata-island workflow, Task 2')",
          "",
          f"## Easiest {min(top, len(rows))} unmatched (score asc) — the work queue",
          "| score | name | nins | br | calls | jtbl | leaf |",
          "|---|---|---|---|---|---|---|"]
    for r in rows[:top]:
        md.append(f"| {r['score']} | {r['name']} | {r['nins']} | {r['branches']} | "
                  f"{r['ncalls']} | {'Y' if r['jtbl'] else '-'} | {'Y' if r['leaf'] else '-'} |")
    MD.write_text("\n".join(md) + "\n")
    print("\n".join(md[:9]))
    print(f"... full table -> {MD.relative_to(ROOT)} (top {top}), CSV -> {CSV.relative_to(ROOT)}")

if __name__ == "__main__":
    main()