mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-09-26 13:33:34 -04:00
Drew T
332 lines
17 KiB
Python
332 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""split_indicator.py -- which code subsegs MUST be split before their switch functions can bank?
|
|
|
|
THE CONSTRAINT. A compiled object contributes exactly ONE contiguous `.rodata` run. A binary whose
|
|
gcc switch jump tables sit in an island of several SEPARATED spans can therefore carve only one span
|
|
per code object -- so every switch function whose table lives in a second span is **unbankable**, at
|
|
any effort: cc1 emits its table while the raw copy is still emitted from the data segment, the image
|
|
grows, and every symbol above the insertion point shifts.
|
|
|
|
WHY THIS IS A TOOL AND NOT A PARAGRAPH (P31 S72). `main` sat in exactly that state from Phase 7 to
|
|
Phase 31. The evidence was written down by hand in `config/splat.us.exe.yaml` on 2026-06-15 -- the
|
|
island's contents, the divider, the library tables -- and nothing ever compared it against the
|
|
subseg list. The cost of noticing late is not the delay, it is that a TU split costs a yaml edit at
|
|
0% matched and a declaration refactor at 94% (accelerators #20): `src/800.c` went from 13 externs /
|
|
0 typedefs to 2,378 / 175 in between. Eleven functions were meanwhile recorded as "PROVEN
|
|
gate-rejects" and ten of them banked byte-identical the moment the carve existed.
|
|
|
|
So this is the check that should run on day one of a decomp, and on every binary, forever:
|
|
|
|
tools/split_indicator.py # every binary
|
|
tools/split_indicator.py --binary main # one
|
|
tools/split_indicator.py --self-test # negative control (see below)
|
|
|
|
WHAT IT REPORTS
|
|
* **NEEDS SPLIT** -- a code subseg references raw tables in >= 2 non-adjacent spans. Names the
|
|
spans, the owning functions, and the address boundary to split at. Those functions cannot bank
|
|
until it is done. Exit 1.
|
|
* **OK** -- every subseg's raw tables are confined to one contiguous span.
|
|
* **REFUSED** -- a table it cannot attribute to any subseg. Reported loudly and counted as a
|
|
failure, never folded into OK (R43/R32: a checker that silently skips what it cannot parse
|
|
reports a true number about a narrower world than the reader believes).
|
|
|
|
WHAT IT DELIBERATELY DOES NOT DO. It does not propose splits on TU archaeology. A contiguous run of
|
|
tables IS one translation unit's rodata, so the spans reveal (some of) the original TU boundaries --
|
|
but a TU containing no `switch` emits no table and is invisible here, so the spans are a LOWER BOUND
|
|
on the original structure, never a reconstruction of it. Split where the BUILD forces a boundary and
|
|
nowhere else: the minimum that satisfies the constraint is the right answer, and extra splits cost
|
|
declaration duplication across the new TUs for nothing.
|
|
"""
|
|
import argparse, collections, functools, glob, os, re, subprocess, sys
|
|
|
|
print = functools.partial(print, flush=True)
|
|
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
JTBL = re.compile(r'jtbl_([0-9A-Fa-f]{8})')
|
|
|
|
|
|
def splat_yaml(binary):
|
|
p = os.path.join(REPO, 'config',
|
|
'splat.us.exe.yaml' if binary == 'main' else 'splat.%s.yaml' % binary)
|
|
return p if os.path.exists(p) else None
|
|
|
|
|
|
def file0_vram(text):
|
|
"""The vram matching byte 0 of the target file: `vram - start` for a segment with a header
|
|
(main's PS-X EXE), and plain `vram` for a flat overlay blob. Same derivation as
|
|
jtbl_rodata_pads._file0_vram -- yaml offsets and addresses must agree in ONE expression."""
|
|
m = re.search(r'-\s*name:\s*\w+\s*\n\s*type:\s*code\s*\n\s*start:\s*(0x[0-9A-Fa-f]+)'
|
|
r'\s*\n\s*vram:\s*(0x[0-9A-Fa-f]+)', text)
|
|
if m:
|
|
return int(m.group(2), 16) - int(m.group(1), 16)
|
|
m = re.search(r'^\s*vram:\s*(0x[0-9A-Fa-f]+)', text, re.M)
|
|
return int(m.group(1), 16) if m else None
|
|
|
|
|
|
def pieces(text, base):
|
|
"""-> (code_subsegs, rodata_carves) as {name: [(lo_vram, hi_vram)]}, from the yaml piece list."""
|
|
rows = [(int(a, 16), k, n) for a, k, n in
|
|
re.findall(r'^\s*- \[0x([0-9A-Fa-f]+), (\S+), (\S+?)\]', text, re.M)]
|
|
rows.sort()
|
|
code, rod = collections.defaultdict(list), collections.defaultdict(list)
|
|
for i, (off, kind, name) in enumerate(rows):
|
|
hi = base + rows[i + 1][0] if i + 1 < len(rows) else None
|
|
(code if kind == 'c' else rod if kind == '.rodata' else {}).setdefault(name, []) \
|
|
.append((base + off, hi)) if kind in ('c', '.rodata') else None
|
|
return code, rod
|
|
|
|
|
|
def raw_tables(binary):
|
|
"""Jump tables STILL IN RAW DATA (a migrated table lives in a .s and cannot conflict).
|
|
-> sorted [(lo, hi)] vram spans, one per table."""
|
|
d = os.path.join(REPO, 'asm', 'data') if binary == 'main' \
|
|
else os.path.join(REPO, 'asm', binary, 'data')
|
|
out = []
|
|
for f in sorted(glob.glob(os.path.join(d, '*.s'))):
|
|
cur, lo, hi = None, None, None
|
|
for ln in open(f, errors='replace'):
|
|
m = re.match(r'\s*dlabel\s+(jtbl_[0-9A-Fa-f]{8})', ln)
|
|
if m:
|
|
cur, lo, hi = m.group(1), None, None
|
|
continue
|
|
if cur is None:
|
|
continue
|
|
m = re.match(r'\s*/\*\s*[0-9A-Fa-f]+\s+([0-9A-Fa-f]{8})', ln)
|
|
if m:
|
|
a = int(m.group(1), 16)
|
|
lo = a if lo is None else lo
|
|
hi = a + 4
|
|
elif ln.strip().startswith('enddlabel') and lo is not None:
|
|
out.append((lo, hi)); cur = None
|
|
return sorted(out)
|
|
|
|
|
|
def spans(tables):
|
|
"""Merge tables that abut into contiguous spans -> [(lo, hi, count)]."""
|
|
out = []
|
|
for lo, hi in tables:
|
|
if out and out[-1][1] == lo:
|
|
out[-1][1], out[-1][2] = hi, out[-1][2] + 1
|
|
else:
|
|
out.append([lo, hi, 1])
|
|
return [tuple(s) for s in out]
|
|
|
|
|
|
def code_owner(code, addr):
|
|
"""The CURRENT code subseg containing `addr`, per the config. None if no piece contains it."""
|
|
best = None
|
|
for name, rng in code.items():
|
|
for lo, hi in rng:
|
|
if lo <= addr and (hi is None or addr < hi):
|
|
best = name
|
|
return best
|
|
|
|
|
|
def owners(binary, code):
|
|
"""-> ({table_addr: {(subseg, fn)}}, [stale note]) from every stub .s referencing a jtbl symbol.
|
|
|
|
THE SUBSEG IS DERIVED FROM THE CONFIG, BY ADDRESS — never from the .s path (P31 S74, R33).
|
|
`make extract` does not prune a re-homed subseg's `asm/<bin>/nonmatchings/<old>/` directory, so
|
|
after a §431 split BOTH the old and the new directory hold the moved function's stub. The first
|
|
version of this function read `os.path.basename(os.path.dirname(path))` and therefore attributed
|
|
the table to the STALE subseg: it printed **NEEDS SPLIT for a split that was already correct and
|
|
byte-green**, and three independent agents hit it in one session — on the one tool whose job is
|
|
to certify a split. `jtbl_carve.func_subseg` documents the identical hazard (a §8b isolation
|
|
leaves the same debris) and already defends against it exactly this way.
|
|
|
|
A stub whose name carries no address (a named function) still falls back to its path, but only
|
|
when that directory is a CURRENT subseg; otherwise it is counted as stale and skipped, and the
|
|
count is REPORTED rather than folded silently into the verdict (R32/R55).
|
|
"""
|
|
root = os.path.join(REPO, 'asm', 'nonmatchings') if binary == 'main' \
|
|
else os.path.join(REPO, 'asm', binary, 'nonmatchings')
|
|
out = collections.defaultdict(set)
|
|
stale = {} # path -> note, DEDUPED: one .s referencing two tables
|
|
# is one stale file, and a count is read as a count.
|
|
if not os.path.isdir(root):
|
|
return out, stale
|
|
r = subprocess.run(['grep', '-rEo', '--include=*.s', 'jtbl_[0-9A-Fa-f]{8}', root],
|
|
capture_output=True, text=True)
|
|
for line in r.stdout.splitlines():
|
|
path, _, sym = line.rpartition(':')
|
|
if not path:
|
|
continue
|
|
fn = os.path.basename(path)[:-2]
|
|
path_sub = os.path.basename(os.path.dirname(path))
|
|
m = re.match(r'func_([0-9A-Fa-f]{8})$', fn)
|
|
sub = code_owner(code, int(m.group(1), 16)) if m else None
|
|
if sub is None: # unaddressed name (or an addr outside every piece)
|
|
if path_sub not in code:
|
|
stale['%s/%s.s' % (path_sub, fn)] = None
|
|
continue
|
|
sub = path_sub
|
|
elif sub != path_sub:
|
|
stale['%s/%s.s (current owner: %s)' % (path_sub, fn, sub)] = None
|
|
out[int(sym[5:], 16)].add((sub, fn))
|
|
return out, list(stale)
|
|
|
|
|
|
def check(binary, remap=None, tabs_override=None, own_override=None):
|
|
"""-> (status, lines).
|
|
|
|
`tabs_override` / `own_override` inject a synthetic island so the DECISION LOGIC can be tested
|
|
against a case whose answer is known, independently of the current tree. The first version of
|
|
the self-test tried to simulate 'before the split' by merging subseg NAMES, and reported OK:
|
|
undoing the split would also mean un-carving spans B and C, and a carved table is no longer
|
|
raw, so there was nothing left to find. A control that cannot fail is not a control."""
|
|
y = splat_yaml(binary)
|
|
if not y:
|
|
return 'REFUSED', ['no splat config']
|
|
txt = open(y).read()
|
|
base = file0_vram(txt)
|
|
if base is None:
|
|
return 'REFUSED', ['no vram in the splat config']
|
|
code, _rod = pieces(txt, base)
|
|
# LINKED LIBRARY SUBSEGS CANNOT CONFLICT, AND FLAGGING THEM IS A FALSE POSITIVE. Their code
|
|
# comes from a PsyQ .a, not from our C, so cc1 never emits a table for them and their raw
|
|
# tables are inert forever. Measured: without this filter main reports NEEDS SPLIT on `libgs6`
|
|
# (5 spans) and `libmcrd1` (2) while the game subsegs are clean — the detector's own self-test
|
|
# caught it before it could be believed.
|
|
try:
|
|
sys.path.insert(0, os.path.join(REPO, 'tools'))
|
|
import progress
|
|
progress.set_binary(binary)
|
|
linked = set(progress.linked_subsegs())
|
|
except Exception:
|
|
linked = set()
|
|
tabs = raw_tables(binary) if tabs_override is None else tabs_override
|
|
if not tabs:
|
|
return 'OK', ['no raw jump tables']
|
|
sp = spans(tabs)
|
|
own, stale = (owners(binary, code) if own_override is None else (own_override, []))
|
|
by_sub, unattributed = collections.defaultdict(set), []
|
|
for i, (lo, hi, _n) in enumerate(sp):
|
|
hit = False
|
|
for addr, who in own.items():
|
|
if lo <= addr < hi:
|
|
for sub, fn in who:
|
|
if sub in linked:
|
|
hit = True
|
|
continue
|
|
by_sub[remap(sub) if remap else sub].add((i, fn))
|
|
hit = True
|
|
if not hit:
|
|
unattributed.append((lo, hi))
|
|
lines, bad = [], False
|
|
for sub, entries in sorted(by_sub.items()):
|
|
idxs = sorted({i for i, _ in entries})
|
|
if len(idxs) > 1:
|
|
bad = True
|
|
lines.append(f" NEEDS SPLIT — subseg `{sub}` owns raw tables in {len(idxs)} "
|
|
f"non-adjacent spans; only ONE can carve:")
|
|
for i in idxs:
|
|
lo, hi, n = sp[i]
|
|
fns = sorted(fn for j, fn in entries if j == i)
|
|
a = [f for f in fns if re.fullmatch(r'func_[0-9A-Fa-f]{8}', f)]
|
|
rng = (f"owners 0x{min(int(f[5:],16) for f in a):08x}.."
|
|
f"0x{max(int(f[5:],16) for f in a):08x}") if a else "owners: see below"
|
|
lines.append(f" span 0x{lo:08x}-0x{hi:08x} ({n} tables) — {rng}")
|
|
lines.append(f" {', '.join(fns[:6])}"
|
|
+ (f" … +{len(fns)-6}" if len(fns) > 6 else ""))
|
|
lines.append(f" -> split `{sub}` between those owner ranges (cookbook §431); "
|
|
f"until then those functions CANNOT bank.")
|
|
if stale:
|
|
# STALE ASM DEBRIS, REPORTED NOT SWALLOWED (P31 S74). `make extract` does not prune a
|
|
# re-homed subseg's `nonmatchings/<old>/` directory, so a split leaves the moved stubs in
|
|
# BOTH. The owner now comes from the config by address, so these files no longer steer the
|
|
# verdict — but they are named here, because the run that discovered this defect saw the
|
|
# tool assert NEEDS SPLIT off exactly this debris and believed it (R32/R55: silence about
|
|
# what you skipped is the defect, not the skipping).
|
|
lines.append(f" note: {len(stale)} stale stub .s under a non-current subseg dir "
|
|
f"(config-derived owner used; `rm -rf asm/{binary} && make extract` clears "
|
|
f"them): " + ", ".join(stale[:4])
|
|
+ (f" … +{len(stale)-4}" if len(stale) > 4 else ""))
|
|
if unattributed:
|
|
# A raw table that NO stub references is owned by linked library code. The reasoning is a
|
|
# derived invariant, not a guess (R33): the binary builds byte-identical, therefore every
|
|
# MATCHED function's table is already carved into its object; so a table still sitting raw
|
|
# and referenced by no stub cannot belong to code we compile. Reported, never hidden — but
|
|
# not counted as a failure, because there is nothing to split.
|
|
lines.append(f" note: {len(unattributed)} raw span(s) referenced by no stub — "
|
|
f"linked-library owned (the build is byte-identical, so every matched "
|
|
f"function's table is already carved): "
|
|
+ ", ".join(f"0x{lo:08x}-0x{hi:08x}" for lo, hi in unattributed[:4]))
|
|
return ('NEEDS SPLIT' if bad else 'OK'), (lines or [f"{len(sp)} span(s), each confined "
|
|
f"to one subseg"])
|
|
|
|
|
|
def self_test():
|
|
"""Known-true cases, both directions.
|
|
|
|
1. main as it stands must be OK — the S72 split fixed it.
|
|
2. main's island BEFORE that split (spans A+B+C all raw, all owned by subseg `800`) must be
|
|
flagged, with the boundary named. Fed synthetically, because the real tree no longer
|
|
contains that state."""
|
|
ok = True
|
|
st, _ = check('main')
|
|
print(f" main, as it is now -> {st:12s} {'OK' if st == 'OK' else 'FAIL'}")
|
|
ok &= (st == 'OK')
|
|
|
|
# The pre-S72 island: three separated game spans, every owner in subseg `800`.
|
|
tabs = [(0x80072A38, 0x80072C70), (0x80072E44, 0x80073140), (0x800732A0, 0x8007344C)]
|
|
own = {0x80072A7C: {('800', 'func_8001A114')}, 0x80072B88: {('800', 'func_80024448')},
|
|
0x80072E44: {('800', 'func_8002B0B4')}, 0x8007302C: {('800', 'func_8002EED8')},
|
|
0x800732A0: {('800', 'func_80035270')}, 0x80073420: {('800', 'func_80039C70')}}
|
|
st, lines = check('main', tabs_override=tabs, own_override=own)
|
|
fired = st == 'NEEDS SPLIT'
|
|
print(f" main's island BEFORE the S72 split -> {st:12s} {'OK' if fired else 'FAIL'}")
|
|
if fired:
|
|
print("\n".join(" " + l for l in lines[:5]))
|
|
ok &= fired
|
|
|
|
# And a control that must NOT fire: one span, many owners, one subseg.
|
|
st, _ = check('main', tabs_override=[(0x80072A38, 0x80072C70)],
|
|
own_override={0x80072A7C: {('800', 'func_8001A114')},
|
|
0x80072B88: {('800', 'func_80024448')}})
|
|
print(f" one span, one subseg (must NOT fire) -> {st:12s} {'OK' if st == 'OK' else 'FAIL'}")
|
|
ok &= (st == 'OK')
|
|
print('SELF-TEST', 'PASS' if ok else 'FAIL')
|
|
return 0 if ok else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
os.chdir(REPO)
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument('--binary')
|
|
ap.add_argument('--self-test', action='store_true')
|
|
ap.add_argument('--quiet', action='store_true', help='only print non-OK binaries')
|
|
a = ap.parse_args()
|
|
if a.self_test:
|
|
sys.exit(self_test())
|
|
# THE POPULATION IS DERIVED FROM THE YAMLS (R33/R36; P32 T2c). `.run/S70_bins_sorted.txt` used to BE the
|
|
# population when present — a stored list that went stale the moment a binary was onboarded (R51): tools-health
|
|
# printed "213 OK of 213" over a 218-binary fleet and nobody could tell. It is now an ORDER hint only; every
|
|
# yaml-derived binary it does not name is appended, and the denominator is asserted (R32).
|
|
derived = ['main'] + sorted(os.path.basename(p)[len('splat.'):-len('.yaml')]
|
|
for p in glob.glob('config/splat.*.yaml')
|
|
if 'us.exe' not in p and 'template' not in p)
|
|
if a.binary:
|
|
bins = [a.binary]
|
|
else:
|
|
order = open('.run/S70_bins_sorted.txt').read().split() if os.path.exists('.run/S70_bins_sorted.txt') else []
|
|
known = set(derived)
|
|
bins = [b for b in order if b in known] + [b for b in derived if b not in set(order)]
|
|
assert sorted(bins) == sorted(derived), "split_indicator: population != yaml-derived set (R32)"
|
|
bad = 0
|
|
for b in bins:
|
|
st, lines = check(b)
|
|
if st != 'OK':
|
|
bad += 1
|
|
# NOTES PRINT ON AN `OK` TOO (P31 S74). A `note:` line reports something the verdict does
|
|
# NOT cover — stale asm debris, an unattributed span — and hiding it behind `st != OK` is
|
|
# the same shape as the defect this run fixed: a true verdict about a narrower world than
|
|
# the reader believes. --quiet still suppresses everything but a failure.
|
|
notes = [l for l in lines if l.lstrip().startswith('note:')]
|
|
if st != 'OK' or not a.quiet:
|
|
print(f"{b}: {st}")
|
|
if st != 'OK':
|
|
print("\n".join(lines))
|
|
elif notes and not a.quiet:
|
|
print("\n".join(notes))
|
|
print(f"\nsplit_indicator: {len(bins)-bad} OK, {bad} needing attention, of {len(bins)} binaries")
|
|
sys.exit(1 if bad else 0)
|