BFM-decomp/tools/verbatim_target_s.py

#!/usr/bin/env python3
"""verbatim_target_s.py — regenerate a splat-format target `.s` for a function that is no longer a stub.

WHY THIS EXISTS (P31 S75). `tools/asm_in_c.py` found 147 GAME functions that are §265 verbatim
`__asm__` bodies — assembly pasted into a C string literal, byte-identical by construction and
completely undecompiled. They are real remaining work, and NONE of them can be worked on, because:

    splat emits `asm/nonmatchings/<subseg>/<fn>.s` only for functions that are still INCLUDE_ASM
    stubs. A verbatim body is not a stub, so splat stops emitting its `.s` — and `match_one` and
    `rtu_match` BOTH consume a `.s`. Measured: 1 of 147 had a target on disk.

So the entire class was unworkable, not because the information is missing but because it is in the
wrong FORM. This tool puts it back.

WHERE THE BYTES COME FROM, AND WHY IT MATTERS (R34). From the **extracted ROM image**, never from
the `__asm__` block in our own source. The block is the thing under test: regenerating a target from
it would produce an oracle that agrees with the candidate by construction, and a decompile verified
against it would prove only that we transcribed our own transcription. The image is independent.

Output is byte-compatible with what splat emits, so `match_one --asm-subdir` and `rtu_match` consume
it unchanged:

    /* <fileoff> <vaddr> <LEHEX> */  <mnemonic operands>

The mnemonic column comes from a real `objdump` disassembly (so `detect_o0`'s prologue sniffing and
any human reader get true text); the word column is the ground truth used for comparison.

Usage:
    tools/verbatim_target_s.py --binary main --fn SaveLoadRoutine
    tools/verbatim_target_s.py --all      # every DRAFTABLE unit in config/verbatim_manifest.json
    tools/verbatim_target_s.py --all      # default: .run/verbatim_targets/<binary>/<fn>.s
                                      # (NEVER under asm/ — the Makefile globs that tree)
"""
import argparse
import json
import os
import re
import struct
import subprocess
import sys
import tempfile

REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(REPO, 'tools'))

OBJDUMP = 'mipsel-linux-gnu-objdump'


def _fr():
    import family_remap
    return family_remap


def func_extent(binary, fn):
    """(vaddr, nins) for a function, from the binary's sig registry."""
    addr = None
    m = re.match(r'(?:func_|D_)([0-9A-Fa-f]{8})$', fn)
    if m:
        addr = int(m.group(1), 16)
    sig = os.path.join(REPO, '.run', f'sig.{binary}.jsonl')
    if not os.path.exists(sig):
        return None, None
    rows = {}
    for ln in open(sig):
        try:
            r = json.loads(ln)
        except Exception:
            continue
        rows[int(r['addr'], 16)] = r.get('nins')
    if addr is not None and addr in rows:
        return addr, rows[addr]
    # A NAMED function (SaveLoadRoutine, VectorNormal…) has no address in its name. Resolve it
    # through the symbol map rather than guessing — a wrong address silently produces a target for
    # the WRONG FUNCTION, which is the worst possible failure for a matching oracle (R43).
    for f in ('config/symbols.us.txt', f'config/symbols.{binary}.txt'):
        p = os.path.join(REPO, f)
        if not os.path.exists(p):
            continue
        for ln in open(p):
            mm = re.match(rf'\s*{re.escape(fn)}\s*=\s*(0x[0-9A-Fa-f]+)', ln)
            if mm:
                a = int(mm.group(1), 16)
                return a, rows.get(a)
    return None, None


def disassemble(data, vaddr):
    """[(word, text)] for a byte blob at `vaddr`, via a real objdump disassembly."""
    with tempfile.NamedTemporaryFile(suffix='.bin', delete=False) as fh:
        fh.write(data)
        tmp = fh.name
    try:
        # `-z` (--disassemble-zeroes) IS LOAD-BEARING. By default objdump ELIDES runs of zero bytes
        # as `...`, and a MIPS `nop` IS 0x00000000 — so every nop, and every nop-padded tail,
        # silently vanished from the disassembly. Measured across the verbatim class: func_80049610
        # (three nops) produced ZERO instructions, func_80047D3C 31 of 36, func_80049440 5 of 7.
        # The length assertion below caught all of them, which is the only reason this was not
        # shipped as ~30 quietly-truncated targets (R32 — the check is what makes the tool usable).
        r = subprocess.run([OBJDUMP, '-D', '-z', '-b', 'binary', '-m', 'mips:3000', '-EL',
                            f'--adjust-vma={vaddr:#x}', tmp],
                           capture_output=True, text=True, timeout=120)
        out = r.stdout
    except (OSError, subprocess.SubprocessError) as e:
        sys.exit(f'verbatim_target_s: {OBJDUMP} failed: {e}')
    finally:
        os.unlink(tmp)
    insns = []
    for ln in out.splitlines():
        m = re.match(r'\s*([0-9a-f]+):\s+([0-9a-f]{8})\s+(.*)$', ln)
        if m:
            insns.append((int(m.group(2), 16), m.group(3).strip()))
    return insns


_REG_RE = re.compile(r'(?<![\$\w.])(zero|at|v[01]|a[0-3]|t[0-9]|s[0-8]|k[01]|gp|sp|fp|ra)(?![\w])')
_BRANCHES = {'b', 'bal', 'beq', 'bne', 'beql', 'bnel', 'beqz', 'bnez', 'blez', 'bgtz', 'bltz', 'bgez', 'bltzal',
             'bgezal', 'blezl', 'bgtzl', 'bltzl', 'bgezl', 'bc1f', 'bc1t', 'bc2f', 'bc2t', 'j', 'jal'}


def _symtab(binary):
    """address -> name from config/symbols.us.txt (+ a per-binary symbols.us.<binary>.txt if one exists)."""
    tab = {}
    for p in (os.path.join(REPO, 'config/symbols.us.txt'), os.path.join(REPO, f'config/symbols.us.{binary}.txt')):
        if not os.path.exists(p):
            continue
        for ln in open(p):
            m = re.match(r'\s*([A-Za-z_]\w*)\s*=\s*(0x[0-9A-Fa-f]+)', ln)
            if m:
                tab.setdefault(int(m.group(2), 16), m.group(1))
    return tab


def to_gas(insns, vaddr, off, symtab):
    """The ASSEMBLABLE form of the objdump listing (P34 task 2, 2026-09-08). The splat-format listing above is a
    comparison oracle, never assembled by anything here: its bare register names resolve only through macro.inc and
    its branch/jump targets are absolute addresses. Pasted into decomp.me it failed with `invalid operands 'li a2,2'`
    (decomp.me's PS1 prelude defines glabel but not the register aliases). This form is what gets pasted:
    $-registers, `.L<va>` labels for in-function targets, a symbol name (or splat's func_<ADDR>) for external j/jal,
    and it carries the listing's own delay-slot nops, so the caller emits `.set noreorder` (else gas adds a second nop).
    The `/* off va LEHEX */` comment column is kept so the word oracle reads either form."""
    end = vaddr + 4 * len(insns)
    rows, need = [], set()
    for k, (word, text) in enumerate(insns):
        parts = text.split(None, 1)
        mnem, ops = parts[0], (parts[1].strip() if len(parts) > 1 else '')
        ops = _REG_RE.sub(r'$\1', ops)
        if mnem in _BRANCHES and ops:
            head, _, last = ops.rpartition(',')
            m = re.fullmatch(r'0x([0-9a-fA-F]+)', last.strip())
            if m:
                tgt = int(m.group(1), 16)
                if vaddr <= tgt < end and tgt % 4 == 0:
                    lab = f'.L{tgt:08X}'
                    need.add(tgt)
                else:
                    lab = symtab.get(tgt, f'func_{tgt:08X}')
                ops = f'{head},{lab}' if head else lab
        rows.append((vaddr + 4 * k, word, f'{mnem}\t{ops}' if ops else mnem))
    out = []
    for k, (va, word, text) in enumerate(rows):
        if va in need:
            out.append(f'.L{va:08X}:')
        le = struct.pack('<I', word).hex().upper()
        out.append(f'/* {off + 4*k:06X} {va:08X} {le} */  {text}')
    return out


def gas_roundtrip(rows, insns, fn):
    """[(row index, ROM word)] for every emitted instruction whose RE-ASSEMBLY does not reproduce the ROM's word.

    WHY (S99, and it invalidated a whole campaign before it was found): objdump prints the PSEUDO-instruction
    `move s2,a0` for `addu s2,a0,$zero` (0x00809021), and gas assembles `move` as `or` (0x00809025). Every `move` in
    the listing therefore came back a different word, so a target object built from this form differed from the ROM
    in every one of them — and the permuter, scoring against it, reported 28 for a body that IS byte-identical. A
    listing nothing ever assembled and compared is a claim, not an oracle (R98/DK-81 in a second place). Words that
    carry a RELOCATION (jal/j, HI16/LO16) are excluded: the linker fills those, and both sides mask them."""
    body = "\n".join([f".include \"macro.inc\"", ".set noat", ".set noreorder", ".section .text", f"glabel {fn}"] + rows)
    with tempfile.NamedTemporaryFile("w", suffix=".s", delete=False) as fh:
        fh.write(body + "\n")
        spath = fh.name
    opath = spath[:-2] + ".o"
    try:
        r = subprocess.run(["mipsel-linux-gnu-as", "-I", os.path.join(REPO, "include"), "-march=r3000", "-mtune=r3000",
                            "-no-pad-sections", "-O1", "-G0", spath, "-o", opath], capture_output=True, text=True)
        if r.returncode:
            return None, (r.stderr or "").strip().splitlines()[:3]
        sys.path.insert(0, os.path.join(REPO, "tools"))
        import masked_diff
        got = masked_diff.insns_from_object(opath, None)
    finally:
        for p in (spath, opath):
            if os.path.exists(p):
                os.unlink(p)
    if len(got) != len(insns):
        return None, [f"re-assembly produced {len(got)} instructions, the image has {len(insns)}"]
    bad = []
    for k, (g, (word, _text)) in enumerate(zip(got, insns)):
        if g["reloc_kind"]:
            continue
        if g["word"] != word:
            bad.append((k, word))
    return bad, None


def emit(binary, fn, outdir, quiet=False, gas=False):
    fr = _fr()
    vaddr, nins = func_extent(binary, fn)
    if vaddr is None:
        return None, f'{binary}:{fn}: no address (not in sig registry or symbols) — REFUSING to guess'
    if not nins:
        return None, f'{binary}:{fn}: address 0x{vaddr:08X} known but no nins in the sig registry'
    try:
        img = open(fr.img_path(binary), 'rb').read()
        base = fr.vram_of(binary)
    except Exception as e:
        return None, f'{binary}: cannot read image/vram ({e})'
    off = vaddr - base
    if off < 0 or off + nins * 4 > len(img):
        return None, (f'{binary}:{fn}: extent 0x{vaddr:08X}+{nins} lies outside the image '
                      f'(base 0x{base:08X}, {len(img)} bytes) — REFUSING')
    data = img[off:off + nins * 4]
    insns = disassemble(data, vaddr)
    if len(insns) != nins:
        return None, (f'{binary}:{fn}: objdump produced {len(insns)} instruction(s), sig says '
                      f'{nins} — REFUSING to emit a target that disagrees with the registry')
    os.makedirs(os.path.join(outdir, binary), exist_ok=True)
    if gas:
        path = os.path.join(outdir, binary, f'{fn}.gas.s')
        with open(path, 'w') as fh:
            fh.write(f'/* Assemblable target (gas syntax) regenerated by tools/verbatim_target_s.py --gas.\n')
            fh.write(f' * Source of truth: the EXTRACTED ROM IMAGE, not the __asm__ block in src/ — the block is the\n')
            fh.write(f' * thing under test. {nins} instructions at 0x{vaddr:08X}. $-registers, .L<addr> labels for\n')
            fh.write(f' * in-function targets, symbol names for external jumps; the listing carries its delay-slot nops\n')
            fh.write(f' * (hence .set noreorder). Paste WHOLE into decomp.me — its PS1 prelude defines glabel. */\n\n')
            rows = to_gas(insns, vaddr, off, _symtab(binary))
            # the listing must RE-ASSEMBLE to the image's own words, or it is not a target (S99). Each row that does
            # not is replaced by its `.word`, with the mnemonic kept in the comment for the reader; then it is
            # verified again, and a listing that still disagrees is REFUSED rather than emitted (R43).
            patched = 0
            for _ in range(3):
                bad, err = gas_roundtrip(rows, insns, fn)
                if bad is None:
                    return None, f'{binary}:{fn}: the gas form does not assemble ({err})'
                if not bad:
                    break
                # `rows` interleaves `.L<addr>:` label lines with instructions; `bad` counts INSTRUCTIONS. Indexing rows
                # by an instruction index rewrote a label into a comment and the next round refused to assemble.
                ins_rows = [i for i, r in enumerate(rows) if r.startswith('/*')]
                for k, word in bad:
                    head, _, text = rows[ins_rows[k]].partition('*/')
                    rows[ins_rows[k]] = f'{head}*/  .word 0x{word:08X}  /* {text.strip()} */'
                patched += len(bad)
            else:
                return None, (f'{binary}:{fn}: {len(bad)} instruction(s) still do not re-assemble to the image '
                              f'(first at +{bad[0][0] * 4:#x}) — REFUSING to emit a target that is not the bytes')
            fh.write('.set noat\n.set noreorder\n\n.section .text\n\n')
            fh.write(f'glabel {fn}\n')
            for ln in rows:
                fh.write(ln + '\n')
        if not quiet:
            print(f'  {binary:14s} {fn:26s} {nins:5d} ins @ 0x{vaddr:08X} -> {os.path.relpath(path, REPO)} (gas form'
                  + (f', {patched} word-patched)' if patched else ')'))
        return path, None
    path = os.path.join(outdir, binary, f'{fn}.s')
    with open(path, 'w') as fh:
        fh.write(f'.include "macro.inc"\n\n')
        fh.write(f'/* Regenerated target for a §265 verbatim body by tools/verbatim_target_s.py.\n')
        fh.write(f' * Source of truth: the EXTRACTED ROM IMAGE, not the __asm__ block in src/ —\n')
        fh.write(f' * the block is the thing under test. {nins} instructions at 0x{vaddr:08X}. */\n\n')
        fh.write('.section .text\n\n')
        fh.write(f'glabel {fn}\n')
        for k, (word, text) in enumerate(insns):
            va = vaddr + 4 * k
            # BYTE-ORDER hex, not value-order. splat writes the four bytes as they sit in the
            # image (`C8FFBD27` for the instruction 0x27BDFFC8) and masked_diff.insns_from_s reads
            # the column with `struct.unpack("<I", bytes.fromhex(...))`. objdump prints the VALUE,
            # so reversing here double-swaps and every word comes out wrong: measured 91 of 1139
            # words agreeing with splat's own .s for the same function, which is what a
            # known-true cross-check is for (the length matched perfectly, so nothing else caught it).
            le = struct.pack('<I', word).hex().upper()
            fh.write(f'/* {off + 4*k:06X} {va:08X} {le} */  {text}\n')
    if not quiet:
        print(f'  {binary:14s} {fn:26s} {nins:5d} ins @ 0x{vaddr:08X} -> {os.path.relpath(path, REPO)}')
    return path, None


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument('--binary')
    ap.add_argument('--fn')
    ap.add_argument('--all', action='store_true',
                    help='every DRAFTABLE unit in config/verbatim_manifest.json (fragments, permanent-verbatim and not-code are SKIPPED — a target for those is a trap)')
    ap.add_argument('--game-only', action='store_true', default=True)
    # NOT under asm/ — `build/asm/%.o: asm/%.s` globs that tree, so targets written there are
    # picked up as BUILD OBJECTS and the binary goes red (I did exactly that, P31 S75).
    ap.add_argument('--out', default=os.path.join(REPO, '.run/verbatim_targets'))
    ap.add_argument('--gas', action='store_true',
                    help='emit the ASSEMBLABLE form <fn>.gas.s ($-registers, .L labels, .set noat/noreorder, no .include) — '
                         'the form to paste into decomp.me; proven per function by tools/decompme_replica.sh step D')
    a = ap.parse_args()

    targets = []
    if a.all:
        # TARGETS COME FROM THE MANIFEST, NEVER FROM A SCAN (P31 S75). The first version enumerated
        # every verbatim SYMBOL, and 62 of them are NOT FUNCTIONS — fragments of a split function,
        # bare epilogue tails, padding, trampolines. Emitting a per-symbol target for those is what
        # sent two drafting bursts at things no C function can express: a bare epilogue tail has no
        # prologue, so no compiler can produce it in isolation. `config/verbatim_manifest.json`
        # carries the derived taxonomy (cookbook §452), and only these dispositions name a real,
        # standalone function that a drafter can legitimately be pointed at.
        DRAFTABLE = {'DECOMPILE-NOW', 'DECOMPILE-LOW-VALUE', 'UNCERTAIN'}
        mp = os.path.join(REPO, 'config/verbatim_manifest.json')
        if not os.path.exists(mp):
            sys.exit('verbatim_target_s --all: config/verbatim_manifest.json is missing. This tool '
                     'no longer derives its own target list (it emitted targets for 62 non-functions '
                     'when it did). Run tools/verbatim_check.py first.')
        man = json.load(open(mp))
        skipped = 0
        for r in man['rows']:
            if r.get('disposition') in DRAFTABLE:
                targets.append((r['binary'], r['fn']))
            else:
                skipped += 1
        print(f"targets from the manifest: {len(targets)} draftable; {skipped} skipped "
              f"(fragments / permanent-verbatim / not-code — a target for those is a trap)")
    elif a.binary and a.fn:
        targets = [(a.binary, a.fn)]
    else:
        ap.error('give --binary and --fn, or --all')

    ok, refused = 0, []
    for b, fn in sorted(set(targets)):
        p, err = emit(b, fn, a.out, gas=a.gas)
        if p:
            ok += 1
        else:
            refused.append(err)
    print(f'\nemitted {ok} target(s) -> {os.path.relpath(a.out, REPO)}/<binary>/<fn>{".gas" if a.gas else ""}.s')
    if refused:
        # R32/R43: a refusal is REPORTED, never a silent skip — a missing target is why this whole
        # class was unworkable in the first place.
        print(f'REFUSED {len(refused)} (reported, not skipped):')
        for e in refused[:15]:
            print(f'   {e}')


if __name__ == '__main__':
    main()