#!/usr/bin/env python3 """fragment_check.py — refuse drafts that SUBSUME another symbol (the enclosing-function trap). WHY (P31 S52, cookbook §176i/§179). splat labels every address the symbol table names, including addresses that are not functions at all: a bare epilogue tail (`lw $ra; lw $s2..$s0; addiu $sp; jr $ra`), a jump-only branch target with no prologue, an interior label another function branches to. Those symbols live INSIDE a real function's address range. Convert that real function to C and the build breaks in two ways, neither of which any C-level check can see: * `symbol 'SYS_OBJ_8F4' is already defined` — the fragment's own .s is still assembled while the C body emits the same bytes, defining the symbol twice; * ``undefined reference `.L80050F24'`` — the reverse: a neighbour branches to a label inside the function you just replaced, and converting it deleted the label. Both are LINK/assemble-time and both cost a full rebuild to discover. This test costs milliseconds: a function is unsafe to convert if any OTHER symbol's address lies strictly inside `[addr, addr + 4*nins)`. Measured cost of not having it: one wave-Q gate spent ~3 HOURS bisecting a 45-draft slate in which NO subset could ever pass, because the failure was not draft-content at all. The bisect printed nothing the whole time, because it only reports when it isolates a single rejected draft. Usage: fragment_check.py [--binary main] [--prune out.json] Exit 0 = no draft subsumes a symbol; 1 = at least one does. """ import argparse, json, os, re, sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import corpus def interior_symbols(binary): """Return (stubs_by_name, addr->[names]) for the binary.""" stubs = {st.symbol: st for st in corpus.stubs(binary).values()} byaddr = {} for name, addr in corpus.symbols(binary).items(): byaddr.setdefault(addr, []).append(name) # splat-derived stub addresses are symbols too even when absent from the symbol files for st in stubs.values(): byaddr.setdefault(st.addr, []).append(st.symbol) return stubs, byaddr ASM_DEF = re.compile(r'^\s*"?\s*(?:\.globl|\.ent)\s+(\w+)|^\s*"\s*(\w+):', re.M) def defines_foreign_symbol(slate, binary='main'): """Drafts that DEFINE a symbol another stub owns — the second half of this trap, and the one that actually bit wave Q. An agent that works out that a trailing epilogue really belongs to ITS function (splat having attributed those bytes to the next symbol) may write raw inline asm that emits the label itself: `.globl SYS_OBJ_8F4` / `SYS_OBJ_8F4:`. `match_one` is happy -- the bytes are right -- but that symbol still has its OWN .s, so assembly fails with `symbol already defined`. The reasoning is sound and the fix is real; it just cannot ride in a batch while the fragment's stub still exists. Either bank the fragment and its parent together (removing the stub), or leave the function alone.""" stubs = {st.symbol for st in corpus.stubs(binary).values()} out = [] for e in slate: try: text = open(e['draft']).read() except OSError: continue defined = {m.group(1) or m.group(2) for m in ASM_DEF.finditer(text)} foreign = sorted(d for d in defined if d and d != e['fn'] and d in stubs) if foreign: out.append({'fn': e['fn'], 'defines': foreign}) return out LOCAL_LABEL = re.compile(r'\.L([0-9A-Fa-f]{8})\b') def branched_into(slate, binary='main'): """Drafts that own a `.L` label ANOTHER still-stubbed function branches to — the MIRROR of the subsume test, and the class that cost wave R two clean rebuilds (cookbook §181). `check()` above asks "does another symbol live inside MY range?". The reverse is just as fatal: a neighbour's .s contains `bnez $v0, .L8005B9B8` where `.L8005B9B8` is defined inside the function YOU are converting to C. Convert it and the label ceases to exist, so the neighbour's still-assembled .s fails at LINK time with `undefined reference to '.L8005B9B8'`. Nothing in the C-level ladder can see this: the draft is byte-perfect, its symbols are right, and the file it lands in is textually valid. Only the linker knows, and it charges a full rebuild to say so. Measured instances (wave R): `gfx2D_BG0_OBJ_4D8` (branched into by gfx2D_BG0_OBJ_1B4.s at .L80050D5C/.L80050D70) and `SYS_OBJ_26EC` (by SYS_OBJ_25C8.s at .L8005B9B8). Cost: one pass over the binary's still-stubbed .s files, seconds. It is deliberately scoped to STUBS ONLY -- a label referenced by an already-banked function is not a hazard, because that function no longer ships an .s that could reference it.""" stubs = {st.symbol: st for st in corpus.stubs(binary).values()} slate_names = {e['fn'] for e in slate} # every .L target referenced by a stub that is NOT itself being converted in this slate referenced = {} for name, st in stubs.items(): if name in slate_names: continue try: text = open(st.asm_path, errors='replace').read() except OSError: continue defined_here = {m.group(1).upper() for m in re.finditer(r'^\.L([0-9A-Fa-f]{8}):', text, re.M)} for m in LOCAL_LABEL.finditer(text): lab = m.group(1).upper() if lab in defined_here: continue # a purely internal label, harmless referenced.setdefault(int(lab, 16), set()).add(name) out = [] for e in slate: st = stubs.get(e['fn']) if not st: continue n = corpus.s_ins_count(st.asm_path) if hasattr(corpus, 's_ins_count') else None if not n: continue lo, hi = st.addr, st.addr + 4 * n hits = sorted((a, sorted(who)) for a, who in referenced.items() if lo <= a < hi) if hits: out.append({'fn': e['fn'], 'addr': '0x%08X' % lo, 'labels': [{'label': '.L%08X' % a, 'referenced_by': who} for a, who in hits]}) return out def check(slate, binary='main'): stubs, byaddr = interior_symbols(binary) findings = [] for e in slate: st = stubs.get(e['fn']) if not st: continue n = corpus.s_ins_count(st.asm_path) if hasattr(corpus, 's_ins_count') else None if not n: continue lo, hi = st.addr, st.addr + 4 * n inside = sorted((a, nm) for a, nms in byaddr.items() if lo < a < hi for nm in nms) if inside: findings.append({ 'fn': e['fn'], 'addr': '0x%08X' % lo, 'nins': n, 'subsumes': [{'addr': '0x%08X' % a, 'symbol': nm} for a, nm in inside], }) return findings def main(): ap = argparse.ArgumentParser() ap.add_argument('slate') ap.add_argument('--binary', default='main') ap.add_argument('--prune', help='write a slate with the offending drafts removed') a = ap.parse_args() slate = json.load(open(a.slate)) bad = check(slate, a.binary) foreign = defines_foreign_symbol(slate, a.binary) mirror = branched_into(slate, a.binary) print(f'{len(slate)} drafts checked against {a.binary} symbol space') for f in bad: syms = ', '.join(f"{s['symbol']}@{s['addr']}" for s in f['subsumes'][:4]) print(f" [FAIL] {f['fn']} ({f['addr']}, {f['nins']} ins) SUBSUMES {len(f['subsumes'])} " f"symbol(s): {syms}") for f in foreign: print(f" [FAIL] {f['fn']} DEFINES another stub's symbol in asm: {', '.join(f['defines'])} " f"— that symbol still has its own .s (assembler: 'already defined')") for f in mirror: first = f['labels'][0] print(f" [FAIL] {f['fn']} ({f['addr']}) OWNS {first['label']}, which " f"{', '.join(first['referenced_by'][:3])} branches to — converting it to C deletes " f"the label (linker: \"undefined reference to `{first['label']}'\")") if not bad and not foreign and not mirror: print(' clean — no draft subsumes, redefines, or owns a branched-into label') if a.prune: names = {f['fn'] for f in bad} | {f['fn'] for f in foreign} | {f['fn'] for f in mirror} json.dump([e for e in slate if e['fn'] not in names], open(a.prune, 'w'), indent=1) print(f' pruned slate ({len(slate) - len(names)} drafts) -> {a.prune}') sys.exit(1 if (bad or foreign or mirror) else 0) if __name__ == '__main__': main()