From 70de5a8611ef15ac8859c2eb8ecc7bcc79e95c72 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:18:49 -0600 Subject: [PATCH] =?UTF-8?q?feat(tools):=20verbatim=5Ftarget=5Fs.py=20?= =?UTF-8?q?=E2=80=94=20the=20147=20asm-posing-as-C=20functions=20are=20wor?= =?UTF-8?q?kable=20again;=20bank=20func=5F8017DB98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit THE BLOCKER. asm_in_c.py found 147 GAME functions that are §265 verbatim __asm__ bodies. NONE of them could be worked on: splat emits asm/nonmatchings//.s only for functions that are still INCLUDE_ASM stubs, and a verbatim body is not a stub -- so splat stops emitting its .s, while match_one and rtu_match BOTH consume one. Measured: 1 of 147 had a target on disk. The class was unworkable because the information was in the wrong FORM, not because it was missing. verbatim_target_s.py regenerates a splat-format target .s from the EXTRACTED ROM IMAGE -- never from the __asm__ block in our own source, because the block is the thing under test and a target derived from it would agree with the candidate by construction (R34). 146 of 147 emitted; the 1 refusal is REPORTED. TWO DEFECTS CAUGHT BY CHECKING AGAINST A KNOWN-TRUE CASE, both of which would have shipped ~147 silently-wrong targets: * BYTE ORDER. splat writes the four bytes as they sit in the image (`C8FFBD27` for instruction 0x27BDFFC8) and masked_diff.insns_from_s reads the column with struct.unpack(" hdr[0] + * +0x06 u16 y -> hdr[1] + * +0x08 s16 n -> hdr[2] (entry count; read signed everywhere else) + * +0x0A u16 h -> hdr[3] + * +0x0C u16 *base base[0..n) = "A" source + * base[n..2n) = "B" source + * base[2n..3n) = live/destination + * a1 != 0 selects source A, a1 == 0 selects source B. + * + * ---- MATCHING NOTES (the four levers, all byte-verified) ------------------- + * 1. THE RECORD WALKER IS THE PARAMETER ITSELF, NOT A DERIVED LOCAL. + * The target's giv init reads the raw incoming argument register: + * addiu $s0, $a0, 0x8 + * loop.c's record_initial() takes bl->initial_value from the SET_SRC of the + * biv's pre-loop set. If the source writes `rec = (Rec *)a0;` that set is + * `(set rec a0pseudo)`, the two coalesce, and the giv init reads $s2 — which + * is what the previous draft emitted. Incrementing the PARAMETER makes the + * biv BE the parm pseudo, whose only pre-loop set is `(set parm (reg $a0))`, + * so the initial value is the HARD REG and the giv init reads $a0. + * (42 -> 32 mismatches, and it also fixed the whole prologue save order and + * put `tag` in $a1.) + * + * 2. THE EQUALITY GUARDS ARE WRITTEN target-FIRST. + * Target: `beq $v1,$a3` = (tr, r). `if (tr != r)`, not `if (r != tr)`. + * The following slt keeps natural order, so only the beq operands move. + * (-3 mismatches.) + * + * 3. THE DESTINATION BASE IS ITS OWN PSEUDO, LIVE ACROSS THE INNER LOOP. + * Target: addu $a1,$v1,$v0 / blez $a0,... / addu $t3,$a1,$zero + * i.e. base+n*2 lands in $a1 and is COPIED into the loop pointer $t3. + * Naturally gcc ties the addu's destination to its dying first operand + * (global.c:set_preference takes XEXP(plus,0)'s hard reg as a preference — + * the .greg dump literally prints ";; 75 preferences: 3"), so it emitted + * `addu $v1,$v1,$v0` and the whole a0..t5 file rotated one slot down. + * Two source facts fix it: `dst = pal;` sits ABOVE the `if (n > 0)` guard + * (that is why the target's delay slot holds the COPY and not the addu), + * and `pal` is still live after the loop, which makes it conflict with the + * loop pointer so the copy cannot be coalesced away. The zero-byte + * `__asm__("" :: "r"(pal))` is what states "still live" at C level. + * (32 -> 9 -> 5 mismatches.) + * + * 4. THE GUARD BLOCK'S THREE SCRATCH VALUES ARE NAMED. + * Because `pal` has to be pinned to $a1 (see 3), $a1 is otherwise free for + * local-alloc to grab as the `n*4` scratch, which shifts n and base down to + * $v1/$v0. Naming the count-scale and the base and pinning them to $v0/$v1 + * reproduces the target's lh $a0 / lw $v1 / sll $v0 / addu $a1 exactly. + * (5 -> 0.) + * + * ---- REQUIRED TU EDIT (blocking, but byte-neutral) ------------------------ + * src/ov_SC06_025/ov_SC06_025_jr_8017BEBC.c:3445 currently reads + * extern void func_8017DB98(s32 a0, s32 a1); + * The target ends `addu $v0, $s3, $zero` — it RETURNS a value — so the + * declaration must become + * extern s32 func_8017DB98(s32 a0, s32 a1); + * The sole caller (:3449) discards the result, so the edit changes no bytes + * there. The (s32 a0, s32 a1) parameter spelling is the TU's and is kept + * verbatim (§20 def-side wall); every narrowing is done inside the body. + * + * Only relocation in the target is `jal func_800599B8`; there are no data + * symbols, and every load/store below goes through a0 or $sp (law 1c + * re-walked against the .s). func_800599B8 is spelled with the fleet-modal + * `void func_800599B8(u16 *)` (n=1066); the TU does not declare it. + * ------------------------------------------------------------------------- */ +typedef struct { + u16 tag; /* 0x00 */ + u16 unk02; /* 0x02 */ + u16 x; /* 0x04 */ + u16 y; /* 0x06 */ + s16 n; /* 0x08 */ + u16 h; /* 0x0A */ + u16 *base; /* 0x0C */ +} Rec8017DB98; + +extern void func_800599B8(u16 *); + +#define REC8017DB98 ((Rec8017DB98 *)a0) + +s32 func_8017DB98(s32 a0, s32 a1) +{ + u16 buf[8]; /* sp+0x10 header for func_800599B8 */ + u16 *src; + u16 *dst; + register u16 *bp __asm__("$3"); /* lw $v1, 4($s0) — record base */ + register s32 kk __asm__("$2"); /* sll $v0, $a0, 2 — n*4 bytes */ + register u16 *pal __asm__("$5"); /* addu $a1, $v1, $v0 — dst base */ + s32 i; + s32 flag; + s32 any; + u16 tag; + s32 cur; + s32 tgt; + s32 r, g, b; + s32 tr, tg, tb; + s32 result; + + tag = REC8017DB98->tag; + any = 0; + if (tag != 0xff) { + do { + if (tag == 9) { + buf[0] = REC8017DB98->x; + buf[1] = REC8017DB98->y; + buf[2] = *(u16 *)&REC8017DB98->n; /* lhu, unlike every other read of n */ + buf[3] = REC8017DB98->h; + if (a1) { + src = REC8017DB98->base; + } else { + src = REC8017DB98->base + REC8017DB98->n; + } + i = 0; + flag = 0; + bp = REC8017DB98->base; + kk = REC8017DB98->n * 4; + pal = (u16 *)((u8 *)bp + kk); + dst = pal; /* ABOVE the guard — lever 3 */ + if (REC8017DB98->n > 0) { + do { + cur = *dst; + tgt = *src; + r = cur & 0x1F; + g = cur & 0x3E0; + b = cur & 0x7C00; + tr = tgt & 0x1F; + tg = tgt & 0x3E0; + tb = tgt & 0x7C00; + if (tr != r) { + flag = 1; + if (r < tr) r += 1; + else if (tr < r) r -= 1; + } + if (tg != g) { + flag = 1; + if (g < tg) g += 0x20; + else if (tg < g) g -= 0x20; + } + if (tb != b) { + flag = 1; + if (b < tb) b += 0x400; + else if (tb < b) b -= 0x400; + } + result = r | g | b | (tgt & 0x8000); + if (result == 0 && tgt != 0) { + result = 0x8000; + } + *dst = result; + dst++; + i++; + src++; + } while (i < REC8017DB98->n); + __asm__("" :: "r"(pal)); /* zero bytes: keeps pal live */ + } + if (flag) { + func_800599B8(buf); + } + any |= flag; + } + a0 += 0x10; + tag = REC8017DB98->tag; + } while (tag != 0xff); + } + return any; +} + +#undef REC8017DB98 + #include "common.h" diff --git a/tools/verbatim_target_s.py b/tools/verbatim_target_s.py new file mode 100644 index 000000000..bf88b5b89 --- /dev/null +++ b/tools/verbatim_target_s.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""verbatim_target_s.py — regenerate a splat-format target `.s` for a function that is no longer a stub. + +WHY THIS EXISTS (P31 S75). `tools/asm_in_c.py` found 147 GAME functions that are §265 verbatim +`__asm__` bodies — assembly pasted into a C string literal, byte-identical by construction and +completely undecompiled. They are real remaining work, and NONE of them can be worked on, because: + + splat emits `asm/nonmatchings//.s` only for functions that are still INCLUDE_ASM + stubs. A verbatim body is not a stub, so splat stops emitting its `.s` — and `match_one` and + `rtu_match` BOTH consume a `.s`. Measured: 1 of 147 had a target on disk. + +So the entire class was unworkable, not because the information is missing but because it is in the +wrong FORM. This tool puts it back. + +WHERE THE BYTES COME FROM, AND WHY IT MATTERS (R34). From the **extracted ROM image**, never from +the `__asm__` block in our own source. The block is the thing under test: regenerating a target from +it would produce an oracle that agrees with the candidate by construction, and a decompile verified +against it would prove only that we transcribed our own transcription. The image is independent. + +Output is byte-compatible with what splat emits, so `match_one --asm-subdir` and `rtu_match` consume +it unchanged: + + /* */ + +The mnemonic column comes from a real `objdump` disassembly (so `detect_o0`'s prologue sniffing and +any human reader get true text); the word column is the ground truth used for comparison. + +Usage: + tools/verbatim_target_s.py --binary main --fn SaveLoadRoutine + tools/verbatim_target_s.py --all # every verbatim body asm_in_c.py finds + tools/verbatim_target_s.py --all --out asm/verbatim # default: asm/verbatim//.s +""" +import argparse +import json +import os +import re +import struct +import subprocess +import sys +import tempfile + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(REPO, 'tools')) + +OBJDUMP = 'mipsel-linux-gnu-objdump' + + +def _fr(): + import family_remap + return family_remap + + +def func_extent(binary, fn): + """(vaddr, nins) for a function, from the binary's sig registry.""" + addr = None + m = re.match(r'(?:func_|D_)([0-9A-Fa-f]{8})$', fn) + if m: + addr = int(m.group(1), 16) + sig = os.path.join(REPO, '.run', f'sig.{binary}.jsonl') + if not os.path.exists(sig): + return None, None + rows = {} + for ln in open(sig): + try: + r = json.loads(ln) + except Exception: + continue + rows[int(r['addr'], 16)] = r.get('nins') + if addr is not None and addr in rows: + return addr, rows[addr] + # A NAMED function (SaveLoadRoutine, VectorNormal…) has no address in its name. Resolve it + # through the symbol map rather than guessing — a wrong address silently produces a target for + # the WRONG FUNCTION, which is the worst possible failure for a matching oracle (R43). + for f in ('config/symbols.us.txt', f'config/symbols.{binary}.txt'): + p = os.path.join(REPO, f) + if not os.path.exists(p): + continue + for ln in open(p): + mm = re.match(rf'\s*{re.escape(fn)}\s*=\s*(0x[0-9A-Fa-f]+)', ln) + if mm: + a = int(mm.group(1), 16) + return a, rows.get(a) + return None, None + + +def disassemble(data, vaddr): + """[(word, text)] for a byte blob at `vaddr`, via a real objdump disassembly.""" + with tempfile.NamedTemporaryFile(suffix='.bin', delete=False) as fh: + fh.write(data) + tmp = fh.name + try: + # `-z` (--disassemble-zeroes) IS LOAD-BEARING. By default objdump ELIDES runs of zero bytes + # as `...`, and a MIPS `nop` IS 0x00000000 — so every nop, and every nop-padded tail, + # silently vanished from the disassembly. Measured across the verbatim class: func_80049610 + # (three nops) produced ZERO instructions, func_80047D3C 31 of 36, func_80049440 5 of 7. + # The length assertion below caught all of them, which is the only reason this was not + # shipped as ~30 quietly-truncated targets (R32 — the check is what makes the tool usable). + r = subprocess.run([OBJDUMP, '-D', '-z', '-b', 'binary', '-m', 'mips:3000', '-EL', + f'--adjust-vma={vaddr:#x}', tmp], + capture_output=True, text=True, timeout=120) + out = r.stdout + except (OSError, subprocess.SubprocessError) as e: + sys.exit(f'verbatim_target_s: {OBJDUMP} failed: {e}') + finally: + os.unlink(tmp) + insns = [] + for ln in out.splitlines(): + m = re.match(r'\s*([0-9a-f]+):\s+([0-9a-f]{8})\s+(.*)$', ln) + if m: + insns.append((int(m.group(2), 16), m.group(3).strip())) + return insns + + +def emit(binary, fn, outdir, quiet=False): + fr = _fr() + vaddr, nins = func_extent(binary, fn) + if vaddr is None: + return None, f'{binary}:{fn}: no address (not in sig registry or symbols) — REFUSING to guess' + if not nins: + return None, f'{binary}:{fn}: address 0x{vaddr:08X} known but no nins in the sig registry' + try: + img = open(fr.img_path(binary), 'rb').read() + base = fr.vram_of(binary) + except Exception as e: + return None, f'{binary}: cannot read image/vram ({e})' + off = vaddr - base + if off < 0 or off + nins * 4 > len(img): + return None, (f'{binary}:{fn}: extent 0x{vaddr:08X}+{nins} lies outside the image ' + f'(base 0x{base:08X}, {len(img)} bytes) — REFUSING') + data = img[off:off + nins * 4] + insns = disassemble(data, vaddr) + if len(insns) != nins: + return None, (f'{binary}:{fn}: objdump produced {len(insns)} instruction(s), sig says ' + f'{nins} — REFUSING to emit a target that disagrees with the registry') + os.makedirs(os.path.join(outdir, binary), exist_ok=True) + path = os.path.join(outdir, binary, f'{fn}.s') + with open(path, 'w') as fh: + fh.write(f'.include "macro.inc"\n\n') + fh.write(f'/* Regenerated target for a §265 verbatim body by tools/verbatim_target_s.py.\n') + fh.write(f' * Source of truth: the EXTRACTED ROM IMAGE, not the __asm__ block in src/ —\n') + fh.write(f' * the block is the thing under test. {nins} instructions at 0x{vaddr:08X}. */\n\n') + fh.write('.section .text\n\n') + fh.write(f'glabel {fn}\n') + for k, (word, text) in enumerate(insns): + va = vaddr + 4 * k + # BYTE-ORDER hex, not value-order. splat writes the four bytes as they sit in the + # image (`C8FFBD27` for the instruction 0x27BDFFC8) and masked_diff.insns_from_s reads + # the column with `struct.unpack(" {os.path.relpath(path, REPO)}') + return path, None + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--binary') + ap.add_argument('--fn') + ap.add_argument('--all', action='store_true', help='every verbatim body tools/asm_in_c.py finds') + ap.add_argument('--game-only', action='store_true', default=True) + ap.add_argument('--out', default=os.path.join(REPO, 'asm/verbatim')) + a = ap.parse_args() + + targets = [] + if a.all: + import asm_in_c + rows, _ = [], None + for path, b in asm_in_c.sources(None): + r, _d = asm_in_c.scan_file(path, b) + rows += r + for r in rows: + if r['cls'] != 'A-FILE-SCOPE-VERBATIM': + continue + if a.game_only and r['fn'] in asm_in_c.SDK_NAMES: + continue + targets.append((r['binary'], r['fn'])) + elif a.binary and a.fn: + targets = [(a.binary, a.fn)] + else: + ap.error('give --binary and --fn, or --all') + + ok, refused = 0, [] + for b, fn in sorted(set(targets)): + p, err = emit(b, fn, a.out) + if p: + ok += 1 + else: + refused.append(err) + print(f'\nemitted {ok} target(s) -> {os.path.relpath(a.out, REPO)}//.s') + if refused: + # R32/R43: a refusal is REPORTED, never a silent skip — a missing target is why this whole + # class was unworkable in the first place. + print(f'REFUSED {len(refused)} (reported, not skipped):') + for e in refused[:15]: + print(f' {e}') + + +if __name__ == '__main__': + main()