BFM-decomp/tools/main_diff_locate.py

#!/usr/bin/env python3
"""main_diff_locate.py -- turn a RED whole-binary gate into a NAMED list of divergent symbols.

WHY THIS EXISTS (P31 S72). A main gate's entire output is two hashes:

    [FAIL] build/us/SLUS_007.26
           got  817987d141d07ced0cc74e8bb0f8bb33eb41cfd1
           want 143dbb89f34491258bbc27810d0a12ec8b43a8dd

That is a correctness oracle with ZERO diagnostic content, and the campaign has been paying for
it. S71 substituted 11 main drafts one at a time, watched 7 of them come back with a different
hash, and recorded all 11 as "PROVEN gate-rejects -- §376 in its purest form". But a hash says
only THAT the image moved, never WHERE: a draft whose own body is byte-perfect still moves the
image if the substitution perturbed a CALLER (the classic §376 shape -- the TU keeps a stale
`extern void f(void*)` forward declaration while the new definition is `void f(s32)`, so every
call site's argument codegen changes). Attributing that to the drafted function is the R40
failure mode: blaming the subject for a harness effect, with no instrument that could tell them
apart.

WHAT IT DOES
  diff the built image against the retail reference, coalesce the differing bytes into runs, and
  name the symbol each run lands in using the linker map. The answer that matters is one line:
  is the divergence INSIDE the function you drafted, or somewhere else?

    * diff inside the drafted function only  -> a real body reject (and, if `match_one` said
      closeness 0, evidence of a match_one blind spot -- §405-A: it compares .text only, so a
      switch's .rodata jump table is invisible to it).
    * diff in a CALLER / elsewhere           -> a plumbing reject, not a body reject. Route to
      the §376/§378 chain (fix_arity_callers -> cast_self_callers -> --sync-decls), re-gate.
    * diff in both                           -> report both; the body verdict is unproven until
      the plumbing half is fixed and it is re-gated.

DERIVED, NOT HARDCODED (R33). The file-offset mapping comes from the map's own
`load address 0x...` on each output section, not from the PS-X EXE's 0x800 header constant, so a
resegmentation cannot silently skew every reported address by a fixed amount.

NEGATIVE CONTROL (R39, and `check-against-a-known-true-case`). `--self-test` flips one byte at a
caller-supplied address in a copy of the reference and asserts the tool names the containing
symbol, then asserts a byte-identical pair reports zero. A localizer that cannot be shown to
finger a KNOWN perturbation is not evidence about an unknown one.

Usage:
  tools/main_diff_locate.py                                  # build/ vs extracted/retail, table
  tools/main_diff_locate.py --focus func_8001A114            # verdict relative to one function
  tools/main_diff_locate.py --json                           # machine-readable
  tools/main_diff_locate.py --self-test 0x8001a114
"""
import argparse, bisect, functools, json, os, re, sys

print = functools.partial(print, flush=True)

REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
BUILT = 'build/us/SLUS_007.26'
REF = 'extracted/retail/SLUS_007.26'
MAP = 'build/us/SLUS_007.26.map'

# ld prints an output section as  `.main    0x80010000  0x64800 load address 0x00000800`, and the
# LMA is the only statement in the whole toolchain of where a vaddr lands in the FILE. Reading it
# beats restating the 0x800 PS-X header here (R33) -- and it is per-section, which a constant is not.
SECTION = re.compile(r'^(\.\S+)\s+0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+load address 0x([0-9a-fA-F]+)')
# A symbol line is an address and a name, indented, nothing else on the line. The `\S+$` anchor is
# what keeps input-section lines (`.text  0x80010000  0x23f0 build/src/boot.o`) out: those carry a
# size column, so they never reduce to one trailing token after the address.
SYMBOL = re.compile(r'^\s+0x([0-9a-fA-F]+)\s{2,}(\S+)$')
INPUT_SEC = re.compile(r'^\s(\.\S+)\s+0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\S+)$')


def parse_map(path):
    """-> (sections, syms) where sections is [(vma, size, lma)] and syms is sorted [(addr, name, obj)].

    Two map facts this has to survive:
      * splat emits `func_X.NON_MATCHING` and `func_X` at the SAME address for every stubbed
        function. Keeping both doubles every row of the report and makes an attribution look
        ambiguous when it is not, so the `.NON_MATCHING` alias is dropped in favour of the real
        name whenever both sit on one address.
      * `ld` has no per-symbol size here. A symbol's extent is [its address, the next DISTINCT
        address) -- the next-address rule is the only end marker available, and it is exact for
        contiguous code.
    """
    sections, raw, obj_of, cur_obj = [], {}, {}, None
    for line in open(path, errors='replace'):
        line = line.rstrip('\n')
        m = SECTION.match(line)
        if m:
            sections.append((int(m.group(2), 16), int(m.group(3), 16), int(m.group(4), 16)))
            continue
        m = INPUT_SEC.match(line)
        if m:
            cur_obj = '%s(%s)' % (m.group(4), m.group(1))
            continue
        m = SYMBOL.match(line)
        if m:
            addr, name = int(m.group(1), 16), m.group(2)
            if name.endswith('= .') or '=' in name:
                continue
            prev = raw.get(addr)
            # prefer the real name over splat's `.NON_MATCHING` alias at the same address
            if prev is None or (prev.endswith('.NON_MATCHING') and not name.endswith('.NON_MATCHING')):
                raw[addr] = name
                obj_of[addr] = cur_obj
    syms = sorted((a, n, obj_of.get(a)) for a, n in raw.items())
    return sections, syms


def off_to_addr(sections, off):
    """Map a file offset back to a RAM address, preferring the SMALLEST containing section.

    The output sections OVERLAP in file offsets here: `.main` spans 0x800..0x65000 and every
    linked PsyQ library block sits inside that range with its own `load address`. Both mappings
    agree today (the libs are contiguous inside .main), but "first match wins" would silently
    depend on map ordering the day they stop agreeing, so the tightest section wins."""
    best = None
    for vma, size, lma in sections:
        if lma <= off < lma + size and (best is None or size < best[1]):
            best = (vma, size, lma)
    return None if best is None else best[0] + (off - best[2])


def addr_to_sym(syms, addrs, addr):
    """Name the symbol containing `addr`, or None before the first symbol."""
    i = bisect.bisect_right(addrs, addr) - 1
    if i < 0:
        return None, None
    return syms[i][1], syms[i][2]


def diff_runs(a, b, gap=64):
    """Differing byte offsets, coalesced into runs separated by more than `gap` identical bytes.

    Regalloc drift scatters single differing WORDS across a whole function; emitting one run per
    word buries the answer in hundreds of rows. Merging across a small gap groups them back into
    the one region a human (or a router) actually reasons about, while `nbytes` keeps the honest
    count of bytes that actually differ (R41 -- the run count is not the magnitude)."""
    n = min(len(a), len(b))
    runs, start, last, ndiff = [], None, None, 0
    for i in range(n):
        if a[i] != b[i]:
            ndiff += 1
            if start is None:
                start, last = i, i
            elif i - last > gap:
                runs.append((start, last + 1))
                start = i
            last = i
    if start is not None:
        runs.append((start, last + 1))
    return runs, ndiff, (len(a) != len(b))


def attribute(built, ref, sections, syms, gap=64):
    """-> (per_symbol, total_diff_bytes, size_mismatch).

    Attribution is PER BYTE, not per run: a run that straddles two functions is credited to both
    in the right proportion. Crediting a whole run to the symbol its first byte lands in is how a
    one-byte spill into the next function gets reported as "two functions diverged"."""
    addrs = [s[0] for s in syms]
    runs, ndiff, size_mismatch = diff_runs(built, ref, gap)
    per = {}
    for lo, hi in runs:
        for off in range(lo, hi):
            if built[off] == ref[off]:
                continue
            addr = off_to_addr(sections, off)
            if addr is None:
                key, obj = '<outside any output section>', None
            else:
                name, obj = addr_to_sym(syms, addrs, addr)
                key = name or '<before first symbol>'
            e = per.setdefault(key, {'symbol': key, 'obj': obj, 'bytes': 0,
                                     'first_off': off, 'first_addr': addr, 'last_addr': addr})
            e['bytes'] += 1
            e['last_addr'] = addr
    return per, ndiff, size_mismatch


def report(focus=None, as_json=False, built_path=BUILT, ref_path=REF, map_path=MAP, gap=64):
    for p in (built_path, ref_path, map_path):
        if not os.path.exists(p):
            print(f"REFUSED — missing {p}", file=sys.stderr)
            return 2
    built, ref = open(built_path, 'rb').read(), open(ref_path, 'rb').read()
    sections, syms = parse_map(map_path)
    if not sections or not syms:
        print(f"REFUSED — {map_path} yielded {len(sections)} sections / {len(syms)} symbols; "
              f"the map format is not what this tool parses, and a localizer that silently "
              f"attributes nothing is worse than none (R43).", file=sys.stderr)
        return 2
    per, ndiff, size_mismatch = attribute(built, ref, sections, syms, gap)
    rows = sorted(per.values(), key=lambda e: -e['bytes'])

    if as_json:
        print(json.dumps({'diff_bytes': ndiff, 'file_bytes': len(ref),
                          'size_mismatch': size_mismatch, 'focus': focus,
                          'symbols': rows}, indent=1))
        return 0 if ndiff == 0 else 1

    if size_mismatch:
        print(f"!! SIZE MISMATCH — built {len(built)} bytes, reference {len(ref)} bytes. "
              f"Offsets past the shorter file are not compared.")
    if ndiff == 0:
        print(f"IDENTICAL — 0 differing bytes of {len(ref)}.")
        return 0
    print(f"{ndiff} differing bytes of {len(ref)} ({100.0*ndiff/len(ref):.4f}%), "
          f"across {len(rows)} symbol(s):\n")
    print(f"  {'bytes':>7}  {'first addr':>10}  symbol")
    for e in rows[:40]:
        a = f"0x{e['first_addr']:08x}" if e['first_addr'] is not None else f"@{e['first_off']}"
        print(f"  {e['bytes']:>7}  {a:>10}  {e['symbol']}"
              + (f"   [{e['obj']}]" if e['obj'] else ''))
    if len(rows) > 40:
        print(f"  ... and {len(rows)-40} more symbol(s) not listed "
              f"({sum(r['bytes'] for r in rows[40:])} bytes)")

    if focus:
        inside = per.get(focus, {}).get('bytes', 0)
        outside = ndiff - inside
        print()
        if inside and not outside:
            print(f"VERDICT — divergence is CONFINED TO {focus} ({inside} bytes). A real body "
                  f"reject. If match_one said closeness 0, suspect its .text-only blind spot "
                  f"(§405-A: a switch's .rodata jump table is invisible to it).")
        elif outside and not inside:
            print(f"VERDICT — {focus} is BYTE-IDENTICAL; all {outside} differing bytes are "
                  f"ELSEWHERE. This is a PLUMBING reject, not a body reject: the substitution "
                  f"perturbed other code (§376 -- a stale forward declaration changes caller "
                  f"codegen). Route to fix_arity_callers -> cast_self_callers -> --sync-decls.")
        elif inside and outside:
            print(f"VERDICT — MIXED: {inside} bytes inside {focus}, {outside} elsewhere. The body "
                  f"verdict is UNPROVEN until the {outside} outside bytes are fixed and it is "
                  f"re-gated (a perturbed caller can also perturb the callee's own codegen).")
        else:
            print(f"VERDICT — {focus} is not among the divergent symbols and neither is anything "
                  f"attributable to it; check the name (it must match the linker map exactly).")
    return 1


def self_test(addr_hex, map_path=MAP, ref_path=REF):
    """Flip one byte at a KNOWN address and assert the tool names the containing symbol.

    R39 / `check-against-a-known-true-case`. The identical-pair direction is asserted too: a
    localizer that reports a diff on identical inputs would make every verdict above worthless."""
    addr = int(addr_hex, 16)
    ref = open(ref_path, 'rb').read()
    sections, syms = parse_map(map_path)
    addrs = [s[0] for s in syms]
    want, _ = addr_to_sym(syms, addrs, addr)
    off = None
    for vma, size, lma in sections:
        if vma <= addr < vma + size:
            off = lma + (addr - vma)
    if off is None:
        print(f"SELF-TEST REFUSED — 0x{addr:08x} is in no output section")
        return 2
    ok = True

    # direction 1: identical inputs must report zero
    per, ndiff, _ = attribute(ref, ref, sections, syms)
    print(f"  identical-pair  -> {ndiff} differing bytes, {len(per)} symbols "
          f"{'OK' if ndiff == 0 and not per else 'FAIL'}")
    ok &= (ndiff == 0 and not per)

    # direction 2: a known one-byte perturbation must be attributed to the containing symbol
    mut = bytearray(ref)
    mut[off] ^= 0xFF
    per, ndiff, _ = attribute(bytes(mut), ref, sections, syms)
    got = list(per)
    hit = (ndiff == 1 and got == [want])
    print(f"  1-byte flip at 0x{addr:08x} (file offset {off}) -> {ndiff} byte(s) in {got}; "
          f"expected exactly ['{want}'] {'OK' if hit else 'FAIL'}")
    ok &= hit
    print('SELF-TEST', 'PASS' if ok else 'FAIL')
    return 0 if ok else 1


if __name__ == '__main__':
    os.chdir(REPO)
    ap = argparse.ArgumentParser()
    ap.add_argument('--built', default=BUILT)
    ap.add_argument('--ref', default=REF)
    ap.add_argument('--map', dest='map_path', default=MAP)
    ap.add_argument('--focus', help='the function you substituted; adds an attribution verdict')
    ap.add_argument('--gap', type=int, default=64, help='coalesce runs separated by <= N bytes')
    ap.add_argument('--json', action='store_true')
    ap.add_argument('--self-test', metavar='ADDR',
                    help='negative control: flip one byte at ADDR and assert attribution')
    a = ap.parse_args()
    if a.self_test:
        sys.exit(self_test(a.self_test, a.map_path, a.ref))
    sys.exit(report(a.focus, a.json, a.built, a.ref, a.map_path, a.gap))