mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-10-08 01:34:14 -04:00
feat(phase-12): T3 — +2 (typedef-strip recovery) + per-function match tool (REAL 87->89)
- tools/match_one.py (new): compile ONE function's C standalone (pinned triple), mask relocations (jal/HI16/LO16 like psyq_identify), compare to the target bytes in its .s. Isolated temp dir -> parallelizable; a real per-function asm-differ loop for agents to iterate against without a shared build. Validated: a matched fn -> MATCH, residuals -> informative diffs. - harvest_verify.py + match_one.py: strip inline scalar-typedef redefinitions (some drafts inlined `typedef unsigned char u8;` -> C89 redefinition error vs common.h = a COMPILE fail, not a byte mismatch). Re-gate recovered func_800CFBCC + func_800D1938. - REAL 89 (62.76% byte-identical); resident 8e17e02f, main 143dbb89 (R22 clean). - 54 stubs remain (scheduling/regalloc residuals + big GPU/dispatch fns) -> match_one-driven iteration pass next.
This commit is contained in:
Drew T
@@ -1,19 +1,17 @@
|
||||
# Unmatched difficulty inventory (generated by tools/difficulty.py — harvest queue)
|
||||
|
||||
unmatched functions : 56
|
||||
unmatched functions : 54
|
||||
trivial (<=5 ins) : 1
|
||||
non-jtbl leaves : 12 (best harvest targets)
|
||||
non-jtbl leaves : 11 (best harvest targets)
|
||||
jump-table funcs : 5 (deferred — need the rodata-island workflow, Task 2')
|
||||
|
||||
## Easiest 56 unmatched (score asc) — the work queue
|
||||
## Easiest 54 unmatched (score asc) — the work queue
|
||||
| score | name | nins | br | calls | jtbl | leaf |
|
||||
|---|---|---|---|---|---|---|
|
||||
| 4 | func_800CFBE8 | 4 | 0 | 0 | - | Y |
|
||||
| 7 | func_800CFBCC | 7 | 0 | 0 | - | Y |
|
||||
| 15 | func_800D1028 | 11 | 0 | 2 | - | - |
|
||||
| 21 | func_800D2E20 | 19 | 0 | 1 | - | - |
|
||||
| 23 | func_800D05E8 | 18 | 1 | 1 | - | - |
|
||||
| 25 | func_800D1938 | 19 | 0 | 3 | - | - |
|
||||
| 26 | func_800D0EC4 | 18 | 2 | 1 | - | - |
|
||||
| 27 | func_800CEF5C | 21 | 0 | 3 | - | - |
|
||||
| 29 | func_800D0F8C | 21 | 2 | 1 | - | - |
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
# BFM matching progress (generated by tools/progress.py — authoritative)
|
||||
|
||||
REAL substantive matches : 87 <- the Gen1-exit >=25 bar counts THIS
|
||||
REAL substantive matches : 89 <- the Gen1-exit >=25 bar counts THIS
|
||||
LINKED real PsyQ objects : 0 <- byte-identical via linked SDK objects
|
||||
NON_MATCHING (near-miss) : 0
|
||||
splat-auto empty no-ops : 2
|
||||
INCLUDE_ASM stubs : 56
|
||||
INCLUDE_ASM stubs : 54
|
||||
data blobs (excluded) : 0
|
||||
----------------------------------------
|
||||
matchable functions : 145
|
||||
REAL / matchable : 87 / 145 = 60.00%
|
||||
byte-identical/ matchable: 89 / 145 = 61.38% (REAL+LINKED+empties)
|
||||
REAL / matchable : 89 / 145 = 61.38%
|
||||
byte-identical/ matchable: 91 / 145 = 62.76% (REAL+LINKED+empties)
|
||||
|
||||
LINKED subsegs: (0 fns)
|
||||
REAL matches: DsMix func_800CEDFC func_800CEE40 func_800CEE84 func_800CEF04 func_800CEF34 func_800CEFB0 func_800CF370 func_800CF398 func_800CF47C func_800CF4D4 func_800CF510 func_800CF5D4 func_800CF714 func_800CF764 func_800CF7A4 func_800CF804 func_800CF818 func_800CF82C func_800CF840 func_800CF854 func_800CF864 func_800CF8B4 func_800CF8CC func_800CF94C func_800CFBBC func_800CFBF8 func_800CFD68 func_800CFDA4 func_800CFDE8 func_800D02C0 func_800D0630 func_800D065C func_800D0694 func_800D06CC func_800D0C48 func_800D0C74 func_800D0CA0 func_800D0F0C func_800D0F4C func_800D0FE0 func_800D1000 func_800D1054 func_800D1078 func_800D109C func_800D10C8 func_800D1714 func_800D1724 func_800D1734 func_800D1744 func_800D18DC func_800D19DC func_800D1B10 func_800D1D24 func_800D1D94 func_800D1F58 func_800D1F90 func_800D222C func_800D22E4 func_800D2318 func_800D2468 func_800D25FC func_800D2624 func_800D2DAC func_800D2DFC func_800D2EE8 func_800D2F0C func_800D2F48 func_800D2F8C func_800D2FB0 func_800D2FEC func_800D30A8 func_800D30C8 func_800D3104 func_800D3120 func_800D319C func_800D31C0 func_800D31FC func_800D3238 func_800D3274 func_800D32D4 func_800D3328 func_800D3348 func_800D3384 func_800D3398 func_800D33B8 func_800D33E0
|
||||
REAL matches: DsMix func_800CEDFC func_800CEE40 func_800CEE84 func_800CEF04 func_800CEF34 func_800CEFB0 func_800CF370 func_800CF398 func_800CF47C func_800CF4D4 func_800CF510 func_800CF5D4 func_800CF714 func_800CF764 func_800CF7A4 func_800CF804 func_800CF818 func_800CF82C func_800CF840 func_800CF854 func_800CF864 func_800CF8B4 func_800CF8CC func_800CF94C func_800CFBBC func_800CFBCC func_800CFBF8 func_800CFD68 func_800CFDA4 func_800CFDE8 func_800D02C0 func_800D0630 func_800D065C func_800D0694 func_800D06CC func_800D0C48 func_800D0C74 func_800D0CA0 func_800D0F0C func_800D0F4C func_800D0FE0 func_800D1000 func_800D1054 func_800D1078 func_800D109C func_800D10C8 func_800D1714 func_800D1724 func_800D1734 func_800D1744 func_800D18DC func_800D1938 func_800D19DC func_800D1B10 func_800D1D24 func_800D1D94 func_800D1F58 func_800D1F90 func_800D222C func_800D22E4 func_800D2318 func_800D2468 func_800D25FC func_800D2624 func_800D2DAC func_800D2DFC func_800D2EE8 func_800D2F0C func_800D2F48 func_800D2F8C func_800D2FB0 func_800D2FEC func_800D30A8 func_800D30C8 func_800D3104 func_800D3120 func_800D319C func_800D31C0 func_800D31FC func_800D3238 func_800D3274 func_800D32D4 func_800D3328 func_800D3348 func_800D3384 func_800D3398 func_800D33B8 func_800D33E0
|
||||
NON_MATCHING:
|
||||
|
||||
build SHA1: 8e17e02ff8954d07c979449198f7e1645046b353 (byte-identical)
|
||||
|
||||
+55
-2
@@ -305,7 +305,24 @@ void func_800CFBBC(void) {
|
||||
D_80114E70 = 0;
|
||||
}
|
||||
|
||||
INCLUDE_ASM("asm/resident/nonmatchings/resident", func_800CFBCC);
|
||||
/* func_800CFBCC — trivial two-halfword setter.
|
||||
* addiu v0,zero,0x1 ; sh v0,D_80114E70 ; sh a0,D_80114E74 ; jr ra
|
||||
*
|
||||
* D_80114E70 is the SAME global already matched as `u16` in src/resident/resident.c
|
||||
* (func_800CFBBC sets D_80114E70 = 0). Both stores are `sh` (halfword) so the global /
|
||||
* arg signedness does NOT change the emitted store — width is fixed at 16. Kept u16 to
|
||||
* mirror the proven type. Stores are in source order (E70 first, E74 second), with the
|
||||
* `1` materialised once into v0.
|
||||
*
|
||||
* Self-contained per drafts3 rules: NO #include; typedef inlined.
|
||||
*/
|
||||
extern u16 D_80114E70;
|
||||
extern u16 D_80114E74;
|
||||
|
||||
void func_800CFBCC(u16 arg0) {
|
||||
D_80114E70 = 1;
|
||||
D_80114E74 = arg0;
|
||||
}
|
||||
|
||||
INCLUDE_ASM("asm/resident/nonmatchings/resident", func_800CFBE8);
|
||||
|
||||
@@ -675,7 +692,43 @@ void func_800D18DC(void) {
|
||||
local.fn[idx](&D_800AE6A8);
|
||||
}
|
||||
|
||||
INCLUDE_ASM("asm/resident/nonmatchings/resident", func_800D1938);
|
||||
/* ANALYSIS (asm 0x4C):
|
||||
* s0 = arg0 (saved in func_800D1F58's delay slot)
|
||||
* func_800D1F58(); // void, no args set -> a0 still holds arg0 (incidental)
|
||||
* func_801285D4(); // void, nop in delay slot
|
||||
* v0 = (u8)field 0x14 (lbu)
|
||||
* a0 = arg0
|
||||
* field 0x15 = 0 (sb)
|
||||
* v0 = v0 + 1
|
||||
* field 0x14 = (u8)v0 (sb, in jal delay slot)
|
||||
* func_800D1984(arg0);
|
||||
* return 0; // addu v0,zero,zero -> plain s32 0
|
||||
*
|
||||
* field0x14 read with lbu (u8), the +1 result stored back with sb (so the store
|
||||
* truncates to 8 bits naturally). field0x15 is a u8 store of 0. The store order
|
||||
* in asm is: sb 0x15 (=0), then sb 0x14 (=field+1) -- but the 0x14 store sits in
|
||||
* the func_800D1984 delay slot. Writing field0x15=0 first then field0x14++ keeps
|
||||
* gcc's natural schedule (the 0x14 store hoists into the call delay slot).
|
||||
*/
|
||||
|
||||
typedef struct {
|
||||
u8 pad[0x14];
|
||||
u8 f14; /* 0x14 */
|
||||
u8 f15; /* 0x15 */
|
||||
} S800D1938;
|
||||
|
||||
extern void func_800D1F58(void);
|
||||
extern void func_801285D4(void);
|
||||
extern void func_800D1984(S800D1938 *arg0);
|
||||
|
||||
s32 func_800D1938(S800D1938 *arg0) {
|
||||
func_800D1F58();
|
||||
func_801285D4();
|
||||
arg0->f15 = 0;
|
||||
arg0->f14 = arg0->f14 + 1;
|
||||
func_800D1984(arg0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
INCLUDE_ASM("asm/resident/nonmatchings/resident", func_800D1984);
|
||||
|
||||
|
||||
+10
-1
@@ -28,6 +28,15 @@ a = ap.parse_args()
|
||||
|
||||
STUB = 'INCLUDE_ASM("' + a.asm_subdir + '", {fn});' # {fn} filled per function
|
||||
|
||||
# Some drafts inline `typedef unsigned char u8;` etc. ("self-contained") -> when placed in a
|
||||
# .c that already #includes common.h, gcc-2.7.2 (C89) errors on the redefinition. Strip those
|
||||
# lines so common.h provides the types (a compile error is NOT a byte mismatch).
|
||||
_TD = re.compile(r'^[ \t]*typedef\b.*\b(u8|u16|u32|u64|s8|s16|s32|s64|f32|f64)[ \t]*;[ \t]*\n', re.M)
|
||||
|
||||
|
||||
def strip_typedefs(c):
|
||||
return _TD.sub('', c)
|
||||
|
||||
|
||||
def sha1(path):
|
||||
return hashlib.sha1(open(path, 'rb').read()).hexdigest() if os.path.exists(path) else None
|
||||
@@ -56,7 +65,7 @@ for cf in sorted(glob.glob(a.drafts + '/*.c')):
|
||||
w = open(cp).read().strip().lower().split()
|
||||
if w and w[0] in ('high', 'medium', 'low'):
|
||||
conf = w[0]
|
||||
drafts[fn] = {'c': open(cf).read(), 'conf': conf}
|
||||
drafts[fn] = {'c': strip_typedefs(open(cf).read()), 'conf': conf}
|
||||
|
||||
order = {'high': 0, 'medium': 1, 'low': 2}
|
||||
items = sorted(drafts, key=lambda fn: (order[drafts[fn]['conf']], len(drafts[fn]['c'])))
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Per-function matching check: compile ONE function's C standalone (the pinned triple),
|
||||
mask relocations (jal/j 26-bit + HI16/LO16 immediates, like psyq_identify), and compare to
|
||||
the target bytes embedded in its splat .s. Fully isolated (own temp dir) so many run in
|
||||
PARALLEL with no shared build -- a real asm-differ loop for an agent to iterate against.
|
||||
|
||||
Prints "MATCH (N ins)" on a byte-exact (relocation-masked) match, else a per-instruction diff.
|
||||
gcc-2.7.2 -O2 compiles each global function independently (no cross-TU inlining of externs),
|
||||
so a standalone match == the whole-binary byte-gate for that function.
|
||||
|
||||
python3 tools/match_one.py func_800D1DB0 # uses .run/drafts3|2|/<fn>.c
|
||||
python3 tools/match_one.py func_800D1DB0 --c /path/to.c # explicit C file
|
||||
"""
|
||||
import subprocess, re, sys, os, struct, argparse
|
||||
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument('fn')
|
||||
ap.add_argument('--c', help='C file (externs + the function def). Default: search .run/drafts3|2|/')
|
||||
ap.add_argument('--asm-subdir', default='asm/resident/nonmatchings/resident')
|
||||
ap.add_argument('--work', default='.run/match')
|
||||
a = ap.parse_args()
|
||||
|
||||
CPP = 'mipsel-linux-gnu-cpp'; CC1 = 'tools/bin/gcc-2.7.2-psx/cc1'
|
||||
MASPSX = 'tools/maspsx/maspsx.py'; AS = 'mipsel-linux-gnu-as'; PY = '.venv/bin/python'
|
||||
OBJDUMP = 'mipsel-linux-gnu-objdump'
|
||||
CPPFLAGS = '-lang-c -Iinclude -undef -Wall -fno-builtin -Dmips -D__GNUC__=2 -D__OPTIMIZE__ -Dpsx -D_PSYQ -D_MIPSEL -D_LANGUAGE_C'.split()
|
||||
CC1FLAGS = '-quiet -O2 -G0 -mips1 -mcpu=3000 -mgas -msoft-float -fgnu-linker'.split()
|
||||
ASFLAGS = '-Iinclude -march=r3000 -mtune=r3000 -no-pad-sections -O1 -G0'.split()
|
||||
|
||||
cfile = a.c
|
||||
if not cfile:
|
||||
for d in ('.run/drafts3', '.run/drafts2', '.run/drafts'):
|
||||
if os.path.exists('%s/%s.c' % (d, a.fn)):
|
||||
cfile = '%s/%s.c' % (d, a.fn); break
|
||||
if not cfile or not os.path.exists(cfile):
|
||||
print('FAIL: no C file for', a.fn); sys.exit(2)
|
||||
|
||||
wd = '%s/%s' % (a.work, a.fn); os.makedirs(wd, exist_ok=True)
|
||||
src = open(cfile).read()
|
||||
# strip inline scalar-typedef redefinitions (common.h provides them; C89 rejects the dup)
|
||||
src = re.sub(r'^[ \t]*typedef\b.*\b(u8|u16|u32|u64|s8|s16|s32|s64|f32|f64)[ \t]*;[ \t]*\n', '', src, flags=re.M)
|
||||
if '#include "common.h"' not in src:
|
||||
src = '#include "common.h"\n' + src
|
||||
open('%s/t.c' % wd, 'w').write(src)
|
||||
|
||||
|
||||
def pipe(cmd, data=None):
|
||||
return subprocess.run(cmd, input=data, capture_output=True)
|
||||
|
||||
|
||||
p = pipe([CPP] + CPPFLAGS + ['%s/t.c' % wd])
|
||||
if p.returncode: print('CPP FAIL\n' + p.stderr.decode()[-1500:]); sys.exit(1)
|
||||
p = pipe([CC1] + CC1FLAGS, p.stdout)
|
||||
if p.returncode: print('CC1 FAIL\n' + p.stderr.decode()[-1800:]); sys.exit(1)
|
||||
p = pipe([PY, MASPSX, '--aspsx-version=2.56', '--expand-div'], p.stdout)
|
||||
if p.returncode: print('MASPSX FAIL\n' + p.stderr.decode()[-1500:]); sys.exit(1)
|
||||
p = pipe([AS] + ASFLAGS + ['-o', '%s/t.o' % wd], p.stdout)
|
||||
if p.returncode: print('AS FAIL\n' + p.stderr.decode()[-1500:]); sys.exit(1)
|
||||
|
||||
|
||||
def my_insns(obj, fn):
|
||||
"""objdump the function; return (words, masks, mnemonics). mask=0 on jal/j 26-bit,
|
||||
0xFFFF0000 on HI16/LO16 reloc immediates, else 0xFFFFFFFF."""
|
||||
d = subprocess.run([OBJDUMP, '-dr', '-j', '.text', obj], capture_output=True, text=True).stdout
|
||||
words, masks, mnem = [], [], []
|
||||
infn = False
|
||||
for line in d.splitlines():
|
||||
h = re.match(r'^[0-9a-f]+ <([^>]+)>:', line)
|
||||
if h:
|
||||
infn = (h.group(1) == fn); continue
|
||||
if not infn:
|
||||
continue
|
||||
mi = re.match(r'\s+([0-9a-f]+):\s+([0-9a-f]{8})\s+(.*)', line)
|
||||
if mi:
|
||||
w = int(mi.group(2), 16)
|
||||
words.append(w); mnem.append(mi.group(3).strip())
|
||||
masks.append(0 if (w >> 26) in (2, 3) else 0xFFFFFFFF)
|
||||
elif 'R_MIPS' in line and words:
|
||||
masks[-1] = 0 if '_26' in line else 0xFFFF0000
|
||||
return words, masks, mnem
|
||||
|
||||
|
||||
def target_insns(fn):
|
||||
"""parse the splat .s: each /* off vaddr LEHEX */ mnemonic line -> (word, mnemonic)."""
|
||||
words, mnem = [], []
|
||||
for line in open('%s/%s.s' % (a.asm_subdir, fn)):
|
||||
mi = re.match(r'\s*/\*\s*[0-9A-Fa-f]+\s+[0-9A-Fa-f]+\s+([0-9A-Fa-f]{8})\s*\*/\s+(.*)', line)
|
||||
if mi:
|
||||
words.append(struct.unpack('<I', bytes.fromhex(mi.group(1)))[0])
|
||||
mnem.append(re.sub(r'\s+', ' ', mi.group(2).strip()))
|
||||
return words, mnem
|
||||
|
||||
|
||||
mw, mask, mmn = my_insns('%s/t.o' % wd, a.fn)
|
||||
tw, tmn = target_insns(a.fn)
|
||||
|
||||
if not mw:
|
||||
print('FAIL: my object has no function', a.fn, '(compile produced nothing?)'); sys.exit(1)
|
||||
|
||||
n = max(len(mw), len(tw))
|
||||
diffs = []
|
||||
for i in range(n):
|
||||
me = (mw[i] & mask[i]) if i < len(mw) else None
|
||||
tg = (tw[i] & (mask[i] if i < len(mask) else 0xFFFFFFFF)) if i < len(tw) else None
|
||||
if me != tg:
|
||||
diffs.append((i,
|
||||
('%08x %s' % (mw[i], mmn[i])) if i < len(mw) else '--',
|
||||
('%08x %s' % (tw[i], tmn[i])) if i < len(tw) else '--'))
|
||||
|
||||
if not diffs and len(mw) == len(tw):
|
||||
print('MATCH (%d ins) %s' % (len(mw), a.fn))
|
||||
sys.exit(0)
|
||||
print('DIFF %s mine=%d ins, target=%d ins, %d mismatched' % (a.fn, len(mw), len(tw), len(diffs)))
|
||||
print(' idx | MINE | TARGET')
|
||||
for i, me, tg in diffs[:40]:
|
||||
print(' %3d | %-28s | %s' % (i, me, tg))
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user