BFM-decomp/tools/split_indicator.py

#!/usr/bin/env python3
"""split_indicator.py -- which code subsegs MUST be split before their switch functions can bank?

THE CONSTRAINT. A compiled object contributes exactly ONE contiguous `.rodata` run. A binary whose
gcc switch jump tables sit in an island of several SEPARATED spans can therefore carve only one span
per code object -- so every switch function whose table lives in a second span is **unbankable**, at
any effort: cc1 emits its table while the raw copy is still emitted from the data segment, the image
grows, and every symbol above the insertion point shifts.

WHY THIS IS A TOOL AND NOT A PARAGRAPH (P31 S72). `main` sat in exactly that state from Phase 7 to
Phase 31. The evidence was written down by hand in `config/splat.us.exe.yaml` on 2026-06-15 -- the
island's contents, the divider, the library tables -- and nothing ever compared it against the
subseg list. The cost of noticing late is not the delay, it is that a TU split costs a yaml edit at
0% matched and a declaration refactor at 94% (accelerators #20): `src/800.c` went from 13 externs /
0 typedefs to 2,378 / 175 in between. Eleven functions were meanwhile recorded as "PROVEN
gate-rejects" and ten of them banked byte-identical the moment the carve existed.

So this is the check that should run on day one of a decomp, and on every binary, forever:

    tools/split_indicator.py                 # every binary
    tools/split_indicator.py --binary main   # one
    tools/split_indicator.py --self-test     # negative control (see below)

WHAT IT REPORTS
  * **NEEDS SPLIT** -- a code subseg references raw tables in >= 2 non-adjacent spans. Names the
    spans, the owning functions, and the address boundary to split at. Those functions cannot bank
    until it is done. Exit 1.
  * **OK** -- every subseg's raw tables are confined to one contiguous span.
  * **REFUSED** -- a table it cannot attribute to any subseg. Reported loudly and counted as a
    failure, never folded into OK (R43/R32: a checker that silently skips what it cannot parse
    reports a true number about a narrower world than the reader believes).

WHAT IT DELIBERATELY DOES NOT DO. It does not propose splits on TU archaeology. A contiguous run of
tables IS one translation unit's rodata, so the spans reveal (some of) the original TU boundaries --
but a TU containing no `switch` emits no table and is invisible here, so the spans are a LOWER BOUND
on the original structure, never a reconstruction of it. Split where the BUILD forces a boundary and
nowhere else: the minimum that satisfies the constraint is the right answer, and extra splits cost
declaration duplication across the new TUs for nothing.
"""
import argparse, collections, functools, glob, os, re, subprocess, sys

print = functools.partial(print, flush=True)
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
JTBL = re.compile(r'jtbl_([0-9A-Fa-f]{8})')


def splat_yaml(binary):
    p = os.path.join(REPO, 'config',
                     'splat.us.exe.yaml' if binary == 'main' else 'splat.%s.yaml' % binary)
    return p if os.path.exists(p) else None


def file0_vram(text):
    """The vram matching byte 0 of the target file: `vram - start` for a segment with a header
    (main's PS-X EXE), and plain `vram` for a flat overlay blob. Same derivation as
    jtbl_rodata_pads._file0_vram -- yaml offsets and addresses must agree in ONE expression."""
    m = re.search(r'-\s*name:\s*\w+\s*\n\s*type:\s*code\s*\n\s*start:\s*(0x[0-9A-Fa-f]+)'
                  r'\s*\n\s*vram:\s*(0x[0-9A-Fa-f]+)', text)
    if m:
        return int(m.group(2), 16) - int(m.group(1), 16)
    m = re.search(r'^\s*vram:\s*(0x[0-9A-Fa-f]+)', text, re.M)
    return int(m.group(1), 16) if m else None


def pieces(text, base):
    """-> (code_subsegs, rodata_carves) as {name: [(lo_vram, hi_vram)]}, from the yaml piece list."""
    rows = [(int(a, 16), k, n) for a, k, n in
            re.findall(r'^\s*- \[0x([0-9A-Fa-f]+), (\S+), (\S+?)\]', text, re.M)]
    rows.sort()
    code, rod = collections.defaultdict(list), collections.defaultdict(list)
    for i, (off, kind, name) in enumerate(rows):
        hi = base + rows[i + 1][0] if i + 1 < len(rows) else None
        (code if kind == 'c' else rod if kind == '.rodata' else {}).setdefault(name, []) \
            .append((base + off, hi)) if kind in ('c', '.rodata') else None
    return code, rod


def raw_tables(binary):
    """Jump tables STILL IN RAW DATA (a migrated table lives in a .s and cannot conflict).
    -> sorted [(lo, hi)] vram spans, one per table."""
    d = os.path.join(REPO, 'asm', 'data') if binary == 'main' \
        else os.path.join(REPO, 'asm', binary, 'data')
    out = []
    for f in sorted(glob.glob(os.path.join(d, '*.s'))):
        cur, lo, hi = None, None, None
        for ln in open(f, errors='replace'):
            m = re.match(r'\s*dlabel\s+(jtbl_[0-9A-Fa-f]{8})', ln)
            if m:
                cur, lo, hi = m.group(1), None, None
                continue
            if cur is None:
                continue
            m = re.match(r'\s*/\*\s*[0-9A-Fa-f]+\s+([0-9A-Fa-f]{8})', ln)
            if m:
                a = int(m.group(1), 16)
                lo = a if lo is None else lo
                hi = a + 4
            elif ln.strip().startswith('enddlabel') and lo is not None:
                out.append((lo, hi)); cur = None
    return sorted(out)


def spans(tables):
    """Merge tables that abut into contiguous spans -> [(lo, hi, count)]."""
    out = []
    for lo, hi in tables:
        if out and out[-1][1] == lo:
            out[-1][1], out[-1][2] = hi, out[-1][2] + 1
        else:
            out.append([lo, hi, 1])
    return [tuple(s) for s in out]


def owners(binary):
    """-> {table_addr: {(subseg, fn)}} from every stub .s that references a jtbl symbol."""
    root = os.path.join(REPO, 'asm', 'nonmatchings') if binary == 'main' \
        else os.path.join(REPO, 'asm', binary, 'nonmatchings')
    out = collections.defaultdict(set)
    if not os.path.isdir(root):
        return out
    r = subprocess.run(['grep', '-rEo', '--include=*.s', 'jtbl_[0-9A-Fa-f]{8}', root],
                       capture_output=True, text=True)
    for line in r.stdout.splitlines():
        path, _, sym = line.rpartition(':')
        if not path:
            continue
        out[int(sym[5:], 16)].add((os.path.basename(os.path.dirname(path)),
                                   os.path.basename(path)[:-2]))
    return out


def check(binary, remap=None, tabs_override=None, own_override=None):
    """-> (status, lines).

    `tabs_override` / `own_override` inject a synthetic island so the DECISION LOGIC can be tested
    against a case whose answer is known, independently of the current tree. The first version of
    the self-test tried to simulate 'before the split' by merging subseg NAMES, and reported OK:
    undoing the split would also mean un-carving spans B and C, and a carved table is no longer
    raw, so there was nothing left to find. A control that cannot fail is not a control."""
    y = splat_yaml(binary)
    if not y:
        return 'REFUSED', ['no splat config']
    txt = open(y).read()
    base = file0_vram(txt)
    if base is None:
        return 'REFUSED', ['no vram in the splat config']
    code, _rod = pieces(txt, base)
    # LINKED LIBRARY SUBSEGS CANNOT CONFLICT, AND FLAGGING THEM IS A FALSE POSITIVE. Their code
    # comes from a PsyQ .a, not from our C, so cc1 never emits a table for them and their raw
    # tables are inert forever. Measured: without this filter main reports NEEDS SPLIT on `libgs6`
    # (5 spans) and `libmcrd1` (2) while the game subsegs are clean — the detector's own self-test
    # caught it before it could be believed.
    try:
        sys.path.insert(0, os.path.join(REPO, 'tools'))
        import progress
        progress.set_binary(binary)
        linked = set(progress.linked_subsegs())
    except Exception:
        linked = set()
    tabs = raw_tables(binary) if tabs_override is None else tabs_override
    if not tabs:
        return 'OK', ['no raw jump tables']
    sp = spans(tabs)
    own = owners(binary) if own_override is None else own_override
    by_sub, unattributed = collections.defaultdict(set), []
    for i, (lo, hi, _n) in enumerate(sp):
        hit = False
        for addr, who in own.items():
            if lo <= addr < hi:
                for sub, fn in who:
                    if sub in linked:
                        hit = True
                        continue
                    by_sub[remap(sub) if remap else sub].add((i, fn))
                hit = True
        if not hit:
            unattributed.append((lo, hi))
    lines, bad = [], False
    for sub, entries in sorted(by_sub.items()):
        idxs = sorted({i for i, _ in entries})
        if len(idxs) > 1:
            bad = True
            lines.append(f"  NEEDS SPLIT — subseg `{sub}` owns raw tables in {len(idxs)} "
                         f"non-adjacent spans; only ONE can carve:")
            for i in idxs:
                lo, hi, n = sp[i]
                fns = sorted(fn for j, fn in entries if j == i)
                a = [f for f in fns if re.fullmatch(r'func_[0-9A-Fa-f]{8}', f)]
                rng = (f"owners 0x{min(int(f[5:],16) for f in a):08x}.."
                       f"0x{max(int(f[5:],16) for f in a):08x}") if a else "owners: see below"
                lines.append(f"      span 0x{lo:08x}-0x{hi:08x} ({n} tables) — {rng}")
                lines.append(f"          {', '.join(fns[:6])}"
                             + (f" … +{len(fns)-6}" if len(fns) > 6 else ""))
            lines.append(f"      -> split `{sub}` between those owner ranges (cookbook §431); "
                         f"until then those functions CANNOT bank.")
    if unattributed:
        # A raw table that NO stub references is owned by linked library code. The reasoning is a
        # derived invariant, not a guess (R33): the binary builds byte-identical, therefore every
        # MATCHED function's table is already carved into its object; so a table still sitting raw
        # and referenced by no stub cannot belong to code we compile. Reported, never hidden — but
        # not counted as a failure, because there is nothing to split.
        lines.append(f"  note: {len(unattributed)} raw span(s) referenced by no stub — "
                     f"linked-library owned (the build is byte-identical, so every matched "
                     f"function's table is already carved): "
                     + ", ".join(f"0x{lo:08x}-0x{hi:08x}" for lo, hi in unattributed[:4]))
    return ('NEEDS SPLIT' if bad else 'OK'), (lines or [f"{len(sp)} span(s), each confined "
                                                        f"to one subseg"])


def self_test():
    """Known-true cases, both directions.

    1. main as it stands must be OK — the S72 split fixed it.
    2. main's island BEFORE that split (spans A+B+C all raw, all owned by subseg `800`) must be
       flagged, with the boundary named. Fed synthetically, because the real tree no longer
       contains that state."""
    ok = True
    st, _ = check('main')
    print(f"  main, as it is now                     -> {st:12s} {'OK' if st == 'OK' else 'FAIL'}")
    ok &= (st == 'OK')

    # The pre-S72 island: three separated game spans, every owner in subseg `800`.
    tabs = [(0x80072A38, 0x80072C70), (0x80072E44, 0x80073140), (0x800732A0, 0x8007344C)]
    own = {0x80072A7C: {('800', 'func_8001A114')}, 0x80072B88: {('800', 'func_80024448')},
           0x80072E44: {('800', 'func_8002B0B4')}, 0x8007302C: {('800', 'func_8002EED8')},
           0x800732A0: {('800', 'func_80035270')}, 0x80073420: {('800', 'func_80039C70')}}
    st, lines = check('main', tabs_override=tabs, own_override=own)
    fired = st == 'NEEDS SPLIT'
    print(f"  main's island BEFORE the S72 split     -> {st:12s} {'OK' if fired else 'FAIL'}")
    if fired:
        print("\n".join("    " + l for l in lines[:5]))
    ok &= fired

    # And a control that must NOT fire: one span, many owners, one subseg.
    st, _ = check('main', tabs_override=[(0x80072A38, 0x80072C70)],
                  own_override={0x80072A7C: {('800', 'func_8001A114')},
                                0x80072B88: {('800', 'func_80024448')}})
    print(f"  one span, one subseg (must NOT fire)   -> {st:12s} {'OK' if st == 'OK' else 'FAIL'}")
    ok &= (st == 'OK')
    print('SELF-TEST', 'PASS' if ok else 'FAIL')
    return 0 if ok else 1


if __name__ == '__main__':
    os.chdir(REPO)
    ap = argparse.ArgumentParser()
    ap.add_argument('--binary')
    ap.add_argument('--self-test', action='store_true')
    ap.add_argument('--quiet', action='store_true', help='only print non-OK binaries')
    a = ap.parse_args()
    if a.self_test:
        sys.exit(self_test())
    bins = [a.binary] if a.binary else \
        (open('.run/S70_bins_sorted.txt').read().split()
         if os.path.exists('.run/S70_bins_sorted.txt')
         else ['main'] + sorted(os.path.basename(p)[len('splat.'):-len('.yaml')]
                                for p in glob.glob('config/splat.*.yaml')
                                if 'us.exe' not in p and 'template' not in p))
    bad = 0
    for b in bins:
        st, lines = check(b)
        if st != 'OK':
            bad += 1
        if st != 'OK' or not a.quiet:
            print(f"{b}: {st}")
            if st != 'OK':
                print("\n".join(lines))
    print(f"\nsplit_indicator: {len(bins)-bad} OK, {bad} needing attention, of {len(bins)} binaries")
    sys.exit(1 if bad else 0)