From cbf5bae0437af950d1dd9754ea04f6a1e12cc2e6 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:56:28 -0600 Subject: [PATCH] =?UTF-8?q?feat(main):=20unblock=20main's=20switch=20funct?= =?UTF-8?q?ions=20=E2=80=94=20the=20rodata=20span=20carve=20+=20derived=20?= =?UTF-8?q?jtbl=20pads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit main's gate could only ever say "got X want Y". S71 read 7 such verdicts as body rejects and recorded 11 functions as "PROVEN gate-rejects, §376 in its purest form". They are not: all 11 are switch functions, and the blocker is that main has had exactly ONE rodata carve since Phase 7 (LZSS's jtbl_80072A38). Every other main jump table stayed raw in the tail data, so a drafted switch DOUBLE-EMITTED its table, the image grew (+28/+52/+76/+84 measured), and all 238 symbols above 0x80072A4C shifted. * tools/main_diff_locate.py (NEW) — turns a red image into a named list of divergent symbols via the linker map; per-byte attribution, self-test flips a byte at a known address and asserts the containing symbol (plus the identical-pair direction). * gate_main.py — PRESERVES the red image + map before the R40 baseline control rebuilds over it, and auto-localizes: BODY REJECT vs PLUMBING REJECT vs MIXED. Also -j on the build (was single-threaded) and the §376 drop list written to .run/gate_main_dropped.json with the reconciliation chain. * splat.us.exe.yaml — the .rodata carve extends from the LZSS table alone to the whole contiguous game-jtbl span 0x80072A38-0x80072C70 (12 tables, one 800.o run). Byte-neutral with no drafts substituted (probed first). * jtbl_rodata_pads.py — --derive now works for main: one file-0-vram expression makes both address->bytes and yaml-piece->address correct for the EXE's 0x800 header and leaves flat overlays unchanged. Makefile arms it for BINARY=main. Banked byte-identical: func_8001A114, func_8001AAD0, func_8001AF34 — three of the eleven. 25 of main's 59 frontier functions (6,215 of 12,912 instructions) are in this class; the remaining spans need src/800.c split at the TU boundaries the spans reveal. --- Makefile | 4 +- config/splat.us.exe.yaml | 17 ++- src/800.c | 264 +++++++++++++++++++++++++++++++++- tools/gate_main.py | 95 ++++++++++++- tools/jtbl_rodata_pads.py | 37 ++++- tools/main_diff_locate.py | 289 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 693 insertions(+), 13 deletions(-) create mode 100644 tools/main_diff_locate.py diff --git a/Makefile b/Makefile index 5d5699a29..a95813f38 100644 --- a/Makefile +++ b/Makefile @@ -659,7 +659,7 @@ ifeq ($(BINARY),main) # 6324C.data. Idempotent; keyed off splat's exact output (re-run = no-op). # EXE-only (overlays have no rodata island) — gated to BINARY=main; --front/--tail # name the sandwich .data objects (cookbook §8). - $(PYTHON) tools/ld_interleave.py --front 53198.data.o --tail 6324C.data.o $(LD_SCRIPT) + $(PYTHON) tools/ld_interleave.py --front 53198.data.o --tail 63470.data.o $(LD_SCRIPT) endif # Phase-26 §8: overlays that carve a jr-function's jtbl into a dotted .rodata subseg run # ld_interleave to place the migrated .rodata between the pre/post data-tail chunks (the @@ -718,7 +718,7 @@ ASFLAGS_REORDER := -Iinclude -march=r3000 -mtune=r3000 -no-pad-sections -O2 -G0 build/src/%.o: src/%.c @mkdir -p $(dir $@) @echo " CC $@" - @set -o pipefail; $(CPP) $(CPPFLAGS) -MMD -MP -MT $@ -MF $(@:.o=.d) $< | $(CC1_PSX) $(CC1FLAGS) | $(if $(filter $*,$(REORDER_TUS)),$(VENV_PY) tools/reorder_passthrough.py | $(AS) $(ASFLAGS_REORDER) -o $@,$(VENV_PY) $(MASPSX) --aspsx-version=$(ASPSX_VERSION) $(MASPSX_FLAGS) $(if $(JTBL_PADS),| $(VENV_PY) tools/jtbl_rodata_pads.py --pads $(JTBL_PADS),$(if $(filter md_%,$(BINARY)),| $(VENV_PY) tools/jtbl_rodata_pads.py --derive $(BINARY) --tu $(notdir $*))) | $(AS) $(ASFLAGS) -o $@) + @set -o pipefail; $(CPP) $(CPPFLAGS) -MMD -MP -MT $@ -MF $(@:.o=.d) $< | $(CC1_PSX) $(CC1FLAGS) | $(if $(filter $*,$(REORDER_TUS)),$(VENV_PY) tools/reorder_passthrough.py | $(AS) $(ASFLAGS_REORDER) -o $@,$(VENV_PY) $(MASPSX) --aspsx-version=$(ASPSX_VERSION) $(MASPSX_FLAGS) $(if $(JTBL_PADS),| $(VENV_PY) tools/jtbl_rodata_pads.py --pads $(JTBL_PADS),$(if $(filter md_% main,$(BINARY)),| $(VENV_PY) tools/jtbl_rodata_pads.py --derive $(BINARY) --tu $(notdir $*))) | $(AS) $(ASFLAGS) -o $@) # Per-module optimization override (SETUP §5.5 — per-module compiler mixing). The boot/ # main/game-mode-dispatch module (src/boot.c, vram 0x80010000-0x800123F0) was compiled at diff --git a/config/splat.us.exe.yaml b/config/splat.us.exe.yaml index 1773a5fa5..221fa5b49 100644 --- a/config/splat.us.exe.yaml +++ b/config/splat.us.exe.yaml @@ -204,6 +204,19 @@ segments: # splat mis-detect it as func_80062998 (shadowing D_80062998). Carving it as the head of the # front-data subseg forces the data labels deterministically. (ld_interleave FRONT_DATA matches.) - [0x53198, data, 53198] # data table + front data (vram 0x80062998-0x80072A38) - - [0x63238, .rodata, 800] # LZSS jtbl_80072A38 ONLY (vram 0x80072A38-0x80072A4C) -> migrates into LzssDecodeSector - - [0x6324C, data, 6324C] # tail data: rest of island (raw) + globals (vram 0x80072A4C-0x80074800) + # P31 S72 — SPAN EXTENSION. The Phase-7 carve stopped at the LZSS table because a FULL + # island migration hits the interleaved game data and the library jtbls. But the island + # opens with a CONTIGUOUS run of game tables owned entirely by subseg 800: + # 0x80072A38 jtbl (LzssDecodeSector, matched, cc1-emitted) + # 0x80072A4C A7C A94 AB4 ADC B0C B24 B3C B64 B88 BFC (11 tables, 8 stubbed owners) + # 0x80072C70 loadDestPtrTable <- first non-table datum, the span's hard end + # One code object may contribute exactly ONE contiguous .rodata run, and this whole run is + # 800.o's, in address order = src/800.c source order. So the span carves as one piece and + # the 3-piece data->rodata->data sandwich is unchanged in SHAPE, only in where it splits. + # This is what blocked every main switch function: gcc emits the drafted function's table + # into .rodata while the raw copy stayed here, so the image GREW (measured +28/+52/+76/+84 + # on four drafts) and all 238 symbols above 0x80072A4C shifted. 25 of main's 59 frontier + # functions (6,215 of 12,912 instructions) are in that class. + - [0x63238, .rodata, 800] # LZSS jtbl + the 11 contiguous game jtbls (vram 0x80072A38-0x80072C70) + - [0x63470, data, 63470] # tail data: loadDestPtrTable onward (vram 0x80072C70-0x80074800) - [0x65000] diff --git a/src/800.c b/src/800.c index aafdfb1e4..073b4527f 100644 --- a/src/800.c +++ b/src/800.c @@ -7054,7 +7054,120 @@ void func_8001A0FC(void) { D_800AE70C = 0; } -INCLUDE_ASM("asm/nonmatchings/800", func_8001A114); + +/* CD error-recovery state machine (0x8001A114), stepped from CdReadStateMachine's state 10. + * One step per call; returns 1 only when the path table has been re-resolved (recovery done). + * 0: CdFlush-ish reset + CdlNop -> 1 1: CdSync poll (2 -> advance, 0x10 -> restart) + * 2: CdlSetmode(0x80) 3: burn 3 frames 4: CdSync poll + * 5: re-run CdSearchFile on the path entry (up to 16 tries) -> done / restart + * + * §ADD-8 (re-tie barrier): the two constant arguments of the state-0 CdControl must issue + * BEFORE the `la $s0,cdReq_cdResult`; sched1 otherwise ranks the address load first (it feeds + * $a2 and so has the longer chain). The zero-byte `__volatile__` re-ties pin them to source + * order. §20/§243 (held-pointer): cdReq_retry in state 3 and D_800AE6F4 on the shared + * "advance" tail are spelled through a named pointer — that is what turns their %hi/%lo pairs + * into a single base register, and keeping the two tails textually distinct is what stops + * cross-jumping from merging .L8001A260 with .L8001A2AC. */ + +extern void func_800434BC(void); +extern int func_80043830(int com, u8 *param, u8 *result); +extern int func_80046630(int mode); +extern int func_8004674C(void); + +extern s32 D_800AE6F4; /* recovery state */ +extern u8 cdReq_cdResult; /* CdControl status byte (bit 0x10 = error) */ +extern u8 D_800AE740; /* CdlSetmode mode-byte buffer; cdReq_cdResult is at -8 */ +extern int cdReq_retry; +extern CdlFILE D_80063028; /* CdPathTable entry; its name string sits at -0x14 */ + +int func_8001A114(void) { + int ret; + u8 *p; + int r; + int i; + CdlFILE *fp; + int *rp; + s32 *sp; + int c0; + int c1; + + ret = 0; + switch (D_800AE6F4) { + case 0: + func_800434BC(); + c0 = 1; + __asm__ __volatile__("" : "=r"(c0) : "0"(c0)); + c1 = 0; + __asm__ __volatile__("" : "=r"(c1) : "0"(c1)); + p = &cdReq_cdResult; + func_80043830(c0, (u8 *)c1, p); + if ((*p & 0x10) != 0) { + break; + } + D_800AE6F4 = D_800AE6F4 + 1; + /* fallthrough */ + case 1: + r = func_80046630(0); + if (r == 2) { + goto bump; + } + if (r != 0x10) { + break; + } + reset: + D_800AE6F4 = 0; + break; + case 2: + p = &D_800AE740; + *p = 0x80; + if (func_80043830(0xE, p, p - 8) == 0) { + break; + } + if ((p[-8] & 0x10) != 0) { + goto reset; + } + cdReq_retry = 0; + D_800AE6F4 = D_800AE6F4 + 1; + break; + case 3: + rp = &cdReq_retry; + *rp = *rp + 1; + if (*rp < 3) { + break; + } + *rp = 0; + D_800AE6F4 = D_800AE6F4 + 1; + break; + case 4: + r = func_8004674C(); + if ((r == 1) || (r == 0x10) || (r == 0)) { + D_800AE6F4 = 0; + } + if (r != 2) { + break; + } + bump: + sp = &D_800AE6F4; + *sp = *sp + 1; + break; + case 5: + i = 0; + fp = &D_80063028; + do { + r = (int)CdSearchFile(fp, (char *)fp - 0x14); + if (r != -1) { + break; + } + i = i + 1; + } while (i < 0x10); + if ((u32)(r + 1) < 2) { + goto reset; + } + ret = 1; + break; + } + return ret; +} #ifdef NON_MATCHING typedef struct { short x, y, w, h; } RECT; /* libgpu RECT (VRAM rectangle) */ @@ -7312,7 +7425,61 @@ void func_8001AAA0(s32 arg0) { func_8001ABBC(1, arg0, 0, 0, 0); } -INCLUDE_ASM("asm/nonmatchings/800", func_8001AAD0); +extern s32 D_800BA1B4; +extern u8 D_80062C38; +extern s32 func_8001ABBC(s32 a0, s32 a1, s32 a2, s32 a3, s32 a4); + +void func_8001AAD0(s32 arg0, s32 arg1) { + s32 idx; + + switch (arg0) { + case 0: + idx = 9; + break; + case 1: + idx = 10; + break; + case 2: + idx = 11; + break; + case 3: + idx = 12; + break; + case 4: + idx = 13; + break; + case 5: + idx = 14; + break; + case 6: + idx = 15; + break; + case 7: + idx = 16; + break; + case 8: + idx = 17; + break; + case 9: + idx = 18; + break; + case 10: + idx = 19; + break; + case 11: + idx = 20; + break; + default: + idx = 0; + break; + } + + if (D_800BA1B4 == 5) { + D_800BA1B4 = 0; + } + + func_8001ABBC(3, arg1, (s32)(&D_80062C38 + idx * 0x30), 0, 0); +} void func_8001ABB4(void) { } @@ -7463,7 +7630,98 @@ s32 func_8001AF04(void) { return 2; } -INCLUDE_ASM("asm/nonmatchings/800", func_8001AF34); +extern s32 D_800BA1B4; +extern s32 D_800AE6E4; +extern s32 D_800BA318; +extern s32 D_800C6D2C; +extern s32 D_800A6550; +extern s32 cdReq_curSector; +extern void *cdReq_dest; +extern s32 cdReq_size; +extern void *cdReq_cdlFile; +extern s32 D_800AE724; +extern s32 D_800AE720; +extern s32 cdReq_result; +extern s32 D_800AE640; +extern s32 D_800A6430; +extern s16 D_800A6430_h __asm__("D_800A6430"); +extern s32 D_800747E4; +extern s32 CdQueueBusy(void); +extern void CdReadStateMachine(int); +extern s32 func_8001B394(s32); +extern s32 func_8001B7C4(void *); +extern s32 func_8001B0D4(void *, s32); + +void func_8001AF34(void) { + s32 *cdlFile; + s32 dest; + s32 size; + s32 mode; + s32 sector; + s32 ret; + + CdQueueBusy(); + switch (D_800BA1B4) { + case 0: + return; + case 1: + cdlFile = (s32 *)D_800AE6E4; + dest = D_800BA318; + size = D_800C6D2C; + mode = D_800A6550; + if (CdQueueBusy() != 0) { + ret = 0; + goto chk; + } + if (cdReq_curSector == 0) { + sector = *cdlFile; + } else { + sector = *cdlFile; + if (sector != cdReq_curSector) { + ret = 0; + goto chk; + } + } + cdReq_dest = (void *)dest; + cdReq_size = size; + cdReq_cdlFile = (void *)cdlFile; + D_800AE724 = mode; + D_800AE720 = 0; + cdReq_curSector = sector; + if (mode == 0) { + D_800AE720 = 1; + } + CdReadStateMachine(0); + ret = cdReq_result; + chk: + if (ret == 0) { + return; + } + D_800BA1B4 = 3; + return; + case 2: + if (func_8001B394(D_800AE640) == 0) { + return; + } + D_800BA1B4 = 3; + return; + case 3: + return; + case 4: + if (func_8001B7C4((void *)D_800AE6E4) == 0) { + return; + } + D_800BA1B4 = 3; + return; + case 5: + if (func_8001B0D4((void *)D_800AE6E4, D_800A6430_h) == 0) { + return; + } + D_800747E4 = 0; + D_800BA1B4 = 3; + return; + } +} INCLUDE_ASM("asm/nonmatchings/800", func_8001B0D4); diff --git a/tools/gate_main.py b/tools/gate_main.py index e31f3de40..c5c786970 100644 --- a/tools/gate_main.py +++ b/tools/gate_main.py @@ -563,7 +563,12 @@ def clean_build(): can pass without building is worse than no verifier.""" run("rm -f build/us/SLUS_007.26") run("make extract BINARY=main") - r = run("make build BINARY=main") + # `-j`. `make build BINARY=main` without it is SINGLE-THREADED on a 32-core box; the + # Makefile's own JOBS knob is parallelism ACROSS binaries, which a one-binary build never + # reaches (memory `pass-j-to-every-build`, measured 6.1x elsewhere and byte-identical). A gate + # is run hundreds of times a session, so this is the difference between a probe you take and a + # probe you talk yourself out of. + r = run(f"make build BINARY=main -j{os.cpu_count() or 8}") if r.returncode != 0: # `make build BINARY=main` runs the SHA check itself, so rc!=0 does NOT mean "no # binary": a linked-but-MISMATCHED build also exits nonzero. Returning None here routed @@ -578,12 +583,83 @@ def clean_build(): return None, r # build truly failed -> no hash, and never a pass return sha(), r +FAILDIR = '.run/gate_main_fail' + + +def _preserve_and_localize(entries, got): + """Snapshot the RED image + its map, then name the symbols that actually diverged. + + WHY THIS EXISTS (P31 S72). Every red verdict this gate has ever produced was two hashes and + nothing else -- and the R40 baseline control that runs immediately after a failure REBUILDS + THE TREE GREEN, overwriting `build/us/SLUS_007.26` and its map. The one artifact that could + say WHERE the image moved was destroyed, every time, before anyone could look at it. + + That is not a cosmetic gap. S71 substituted 11 main drafts one at a time, saw 7 come back with + a different hash, and recorded all 11 as "PROVEN gate-rejects". A hash cannot distinguish + "your body is wrong" from "your body is perfect and the substitution changed a CALLER" -- the + §376 shape, where the TU keeps a stale `extern void f(void*)` while the definition is + `void f(s32)`, so every call site's argument codegen moves. Six of those eleven are that + class, and this gate's own pre-check names them (see resolve_conflicts) -- but the four that + reached a build were judged with no instrument that could tell the two apart. + + So: copy the image and the map aside FIRST, attribute per byte, and print the verdict. With a + single-entry slate the verdict is the routing decision (body reject vs plumbing reject).""" + try: + import main_diff_locate as MDL + except Exception as e: # never let diagnostics sink a gate + print(f" (diff localization unavailable: {e})") + return + tag = entries[0]['fn'] if len(entries) == 1 else f"batch{len(entries)}" + d = os.path.join(FAILDIR, f"{tag}_{(got or 'nobin')[:8]}") + os.makedirs(d, exist_ok=True) + for f in ('build/us/SLUS_007.26', 'build/us/SLUS_007.26.map'): + if os.path.exists(f): + run(f"cp {f} {d}/") + built = os.path.join(d, 'SLUS_007.26') + mp = os.path.join(d, 'SLUS_007.26.map') + if not (os.path.exists(built) and os.path.exists(mp) and os.path.exists(MDL.REF)): + print(f" (red image preserved at {d}, but localization inputs are incomplete)") + return + try: + sections, syms = MDL.parse_map(mp) + per, ndiff, _sz = MDL.attribute(open(built, 'rb').read(), open(MDL.REF, 'rb').read(), + sections, syms) + except Exception as e: + print(f" (red image preserved at {d}; localization failed: {e})") + return + rows = sorted(per.values(), key=lambda x: -x['bytes']) + print(f" RED IMAGE PRESERVED -> {d}") + print(f" {ndiff} differing byte(s) across {len(rows)} symbol(s):") + for e in rows[:12]: + a = f"0x{e['first_addr']:08x}" if e['first_addr'] is not None else '?' + print(f" {e['bytes']:>6} {a} {e['symbol']}") + if len(rows) > 12: + print(f" ... {len(rows)-12} more ({sum(x['bytes'] for x in rows[12:])} bytes)") + if len(entries) == 1: + fn = entries[0]['fn'] + inside = per.get(fn, {}).get('bytes', 0) + outside = ndiff - inside + if inside and not outside: + print(f" VERDICT {fn}: BODY REJECT — divergence confined to the function itself.") + elif outside and not inside: + print(f" VERDICT {fn}: PLUMBING REJECT — the function is BYTE-IDENTICAL; all " + f"{outside} differing bytes are elsewhere. Route to the §376/§378 chain " + f"(fix_arity_callers --any-proto -> cast_self_callers -> re-gate); do NOT " + f"record this as a body reject.") + elif inside and outside: + print(f" VERDICT {fn}: MIXED — {inside} bytes inside, {outside} outside. The body " + f"verdict is UNPROVEN until the outside bytes are fixed and it is re-gated.") + + def try_batch(entries): run("git checkout -- " + " ".join(main_tus())) run("make extract BINARY=main") # regenerate .s for the reverted stubs (hazard 2) substitute(entries) got, r = clean_build() - return got == GOOD, got, r + ok = got == GOOD + if not ok and got is not None and entries: + _preserve_and_localize(entries, got) + return ok, got, r def main(): ap = argparse.ArgumentParser() @@ -694,6 +770,21 @@ def main(): if dropped: print(" (dropped drafts are usually CORRECT -- recover with a cast-at-use: adopt the") print(" other declaration verbatim and adapt at the use site, e.g. (&D_x)[i].)") + # A DROP IS A ROUTE, NOT A VERDICT (P31 S72). This list is the §376 pile: the draft's + # definition disagrees with a forward declaration the TU already carries, which is a + # PLUMBING problem with a named fix chain -- not evidence about the body. Printed-only, + # it kept getting read as a rejection: S71 recorded six of these as "PROVEN gate-rejects, + # §376 in its purest form -- do not re-slate", and they were never re-slated. Writing it + # to disk with the chain spelled out makes the recovery the obvious next command instead + # of a paragraph someone has to remember. + json.dump(dropped, open('.run/gate_main_dropped.json', 'w'), indent=1) + print(f" -> .run/gate_main_dropped.json ({len(dropped)} to reconcile). The chain is:") + print(f" tools/fix_arity_callers.py --apply --any-proto --funcs " + f"{','.join(d['fn'] for d in dropped)} \\\n" + f" --drafts --journal .run//arity.json") + print(f" tools/cast_self_callers.py --binary main --funcs --drafts " + f"--apply --journal .run//cast.json") + print(f" tools/gate_main.py --apply # the byte-gate arbitrates") if not a.apply: print("\nDRY RUN. Re-run with --apply to substitute and clean-rebuild.") return diff --git a/tools/jtbl_rodata_pads.py b/tools/jtbl_rodata_pads.py index 95d86a101..9bd219ba7 100644 --- a/tools/jtbl_rodata_pads.py +++ b/tools/jtbl_rodata_pads.py @@ -121,9 +121,38 @@ def run(pads, lines, out): # --------------------------------------------------------------------------------------------- import os, struct +def _splat_yaml(binary): + """main's config is `splat.us.exe.yaml`; every other binary is `splat..yaml`. + + `corpus.splat_config` says the same thing, but this filter sits in the hot `build/src/%.o` + recipe (once per object, every build), so it stays free of the corpus layer's import cost. + Kept to one expression so the two cannot drift apart in shape.""" + return "config/splat.us.exe.yaml" if binary == "main" else "config/splat.%s.yaml" % binary + + +def _file0_vram(y): + """The vram that byte 0 of the target file corresponds to. + + For a flat overlay blob this IS the segment vram: the payload starts at file 0. main is a + PS-X EXE whose code segment starts at FILE offset 0x800 (the header), so the vram matching + raw[0] is `vram - start` = 0x80010000 - 0x800. Returning the FILE-0 vram rather than the + segment vram is what makes both `raw[a - vram]` (address -> bytes) and `vram + ` + (yaml piece -> address) correct in BOTH shapes with one expression instead of two code paths. + Derived from the same yaml the build reads (R33); `family_remap.vram_of` derives it the same + way for the family engine.""" + m = re.search(r"-\s*name:\s*\w+\s*\n\s*type:\s*code\s*\n\s*start:\s*(0x[0-9A-Fa-f]+)" + r"\s*\n\s*vram:\s*(0x[0-9A-Fa-f]+)", y) + if m: + return int(m.group(2), 16) - int(m.group(1), 16) + m = re.search(r"^\s*vram:\s*(0x[0-9A-Fa-f]+)", y, re.M) + if not m: + sys.exit("jtbl_rodata_pads: no vram in the splat config (R32 — refusing a default)") + return int(m.group(1), 16) + + def _module_target(binary): - y = open("config/splat.%s.yaml" % binary).read() - vram = int(re.search(r"^\s*vram:\s*(0x[0-9A-Fa-f]+)", y, re.M).group(1), 16) + y = open(_splat_yaml(binary)).read() + vram = _file0_vram(y) tgt = re.search(r"^\s*(?:target_)?path:\s*(\S+)", y, re.M).group(1) return vram, open(tgt, "rb").read() @@ -204,8 +233,8 @@ def _tu_piece(binary, tu): stream has no anchor before its first C table (an isolated §260 object).""" if not tu: return None - y = open("config/splat.%s.yaml" % binary).read() - vram = int(re.search(r"^\s*vram:\s*(0x[0-9A-Fa-f]+)", y, re.M).group(1), 16) + y = open(_splat_yaml(binary)).read() + vram = _file0_vram(y) segs = [(int(a, 16), k, n) for a, k, n in re.findall(r"^\s*- \[0x([0-9A-Fa-f]+), (\S+), (\S+?)\]", y, re.M)] for i, (a, k, n) in enumerate(segs): if k == ".rodata" and n == tu: diff --git a/tools/main_diff_locate.py b/tools/main_diff_locate.py new file mode 100644 index 000000000..b4cc8e959 --- /dev/null +++ b/tools/main_diff_locate.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +"""main_diff_locate.py -- turn a RED whole-binary gate into a NAMED list of divergent symbols. + +WHY THIS EXISTS (P31 S72). A main gate's entire output is two hashes: + + [FAIL] build/us/SLUS_007.26 + got 817987d141d07ced0cc74e8bb0f8bb33eb41cfd1 + want 143dbb89f34491258bbc27810d0a12ec8b43a8dd + +That is a correctness oracle with ZERO diagnostic content, and the campaign has been paying for +it. S71 substituted 11 main drafts one at a time, watched 7 of them come back with a different +hash, and recorded all 11 as "PROVEN gate-rejects -- §376 in its purest form". But a hash says +only THAT the image moved, never WHERE: a draft whose own body is byte-perfect still moves the +image if the substitution perturbed a CALLER (the classic §376 shape -- the TU keeps a stale +`extern void f(void*)` forward declaration while the new definition is `void f(s32)`, so every +call site's argument codegen changes). Attributing that to the drafted function is the R40 +failure mode: blaming the subject for a harness effect, with no instrument that could tell them +apart. + +WHAT IT DOES + diff the built image against the retail reference, coalesce the differing bytes into runs, and + name the symbol each run lands in using the linker map. The answer that matters is one line: + is the divergence INSIDE the function you drafted, or somewhere else? + + * diff inside the drafted function only -> a real body reject (and, if `match_one` said + closeness 0, evidence of a match_one blind spot -- §405-A: it compares .text only, so a + switch's .rodata jump table is invisible to it). + * diff in a CALLER / elsewhere -> a plumbing reject, not a body reject. Route to + the §376/§378 chain (fix_arity_callers -> cast_self_callers -> --sync-decls), re-gate. + * diff in both -> report both; the body verdict is unproven until + the plumbing half is fixed and it is re-gated. + +DERIVED, NOT HARDCODED (R33). The file-offset mapping comes from the map's own +`load address 0x...` on each output section, not from the PS-X EXE's 0x800 header constant, so a +resegmentation cannot silently skew every reported address by a fixed amount. + +NEGATIVE CONTROL (R39, and `check-against-a-known-true-case`). `--self-test` flips one byte at a +caller-supplied address in a copy of the reference and asserts the tool names the containing +symbol, then asserts a byte-identical pair reports zero. A localizer that cannot be shown to +finger a KNOWN perturbation is not evidence about an unknown one. + +Usage: + tools/main_diff_locate.py # build/ vs extracted/retail, table + tools/main_diff_locate.py --focus func_8001A114 # verdict relative to one function + tools/main_diff_locate.py --json # machine-readable + tools/main_diff_locate.py --self-test 0x8001a114 +""" +import argparse, bisect, functools, json, os, re, sys + +print = functools.partial(print, flush=True) + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +BUILT = 'build/us/SLUS_007.26' +REF = 'extracted/retail/SLUS_007.26' +MAP = 'build/us/SLUS_007.26.map' + +# ld prints an output section as `.main 0x80010000 0x64800 load address 0x00000800`, and the +# LMA is the only statement in the whole toolchain of where a vaddr lands in the FILE. Reading it +# beats restating the 0x800 PS-X header here (R33) -- and it is per-section, which a constant is not. +SECTION = re.compile(r'^(\.\S+)\s+0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+load address 0x([0-9a-fA-F]+)') +# A symbol line is an address and a name, indented, nothing else on the line. The `\S+$` anchor is +# what keeps input-section lines (`.text 0x80010000 0x23f0 build/src/boot.o`) out: those carry a +# size column, so they never reduce to one trailing token after the address. +SYMBOL = re.compile(r'^\s+0x([0-9a-fA-F]+)\s{2,}(\S+)$') +INPUT_SEC = re.compile(r'^\s(\.\S+)\s+0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\S+)$') + + +def parse_map(path): + """-> (sections, syms) where sections is [(vma, size, lma)] and syms is sorted [(addr, name, obj)]. + + Two map facts this has to survive: + * splat emits `func_X.NON_MATCHING` and `func_X` at the SAME address for every stubbed + function. Keeping both doubles every row of the report and makes an attribution look + ambiguous when it is not, so the `.NON_MATCHING` alias is dropped in favour of the real + name whenever both sit on one address. + * `ld` has no per-symbol size here. A symbol's extent is [its address, the next DISTINCT + address) -- the next-address rule is the only end marker available, and it is exact for + contiguous code. + """ + sections, raw, obj_of, cur_obj = [], {}, {}, None + for line in open(path, errors='replace'): + line = line.rstrip('\n') + m = SECTION.match(line) + if m: + sections.append((int(m.group(2), 16), int(m.group(3), 16), int(m.group(4), 16))) + continue + m = INPUT_SEC.match(line) + if m: + cur_obj = '%s(%s)' % (m.group(4), m.group(1)) + continue + m = SYMBOL.match(line) + if m: + addr, name = int(m.group(1), 16), m.group(2) + if name.endswith('= .') or '=' in name: + continue + prev = raw.get(addr) + # prefer the real name over splat's `.NON_MATCHING` alias at the same address + if prev is None or (prev.endswith('.NON_MATCHING') and not name.endswith('.NON_MATCHING')): + raw[addr] = name + obj_of[addr] = cur_obj + syms = sorted((a, n, obj_of.get(a)) for a, n in raw.items()) + return sections, syms + + +def off_to_addr(sections, off): + """Map a file offset back to a RAM address, preferring the SMALLEST containing section. + + The output sections OVERLAP in file offsets here: `.main` spans 0x800..0x65000 and every + linked PsyQ library block sits inside that range with its own `load address`. Both mappings + agree today (the libs are contiguous inside .main), but "first match wins" would silently + depend on map ordering the day they stop agreeing, so the tightest section wins.""" + best = None + for vma, size, lma in sections: + if lma <= off < lma + size and (best is None or size < best[1]): + best = (vma, size, lma) + return None if best is None else best[0] + (off - best[2]) + + +def addr_to_sym(syms, addrs, addr): + """Name the symbol containing `addr`, or None before the first symbol.""" + i = bisect.bisect_right(addrs, addr) - 1 + if i < 0: + return None, None + return syms[i][1], syms[i][2] + + +def diff_runs(a, b, gap=64): + """Differing byte offsets, coalesced into runs separated by more than `gap` identical bytes. + + Regalloc drift scatters single differing WORDS across a whole function; emitting one run per + word buries the answer in hundreds of rows. Merging across a small gap groups them back into + the one region a human (or a router) actually reasons about, while `nbytes` keeps the honest + count of bytes that actually differ (R41 -- the run count is not the magnitude).""" + n = min(len(a), len(b)) + runs, start, last, ndiff = [], None, None, 0 + for i in range(n): + if a[i] != b[i]: + ndiff += 1 + if start is None: + start, last = i, i + elif i - last > gap: + runs.append((start, last + 1)) + start = i + last = i + if start is not None: + runs.append((start, last + 1)) + return runs, ndiff, (len(a) != len(b)) + + +def attribute(built, ref, sections, syms, gap=64): + """-> (per_symbol, total_diff_bytes, size_mismatch). + + Attribution is PER BYTE, not per run: a run that straddles two functions is credited to both + in the right proportion. Crediting a whole run to the symbol its first byte lands in is how a + one-byte spill into the next function gets reported as "two functions diverged".""" + addrs = [s[0] for s in syms] + runs, ndiff, size_mismatch = diff_runs(built, ref, gap) + per = {} + for lo, hi in runs: + for off in range(lo, hi): + if built[off] == ref[off]: + continue + addr = off_to_addr(sections, off) + if addr is None: + key, obj = '', None + else: + name, obj = addr_to_sym(syms, addrs, addr) + key = name or '' + e = per.setdefault(key, {'symbol': key, 'obj': obj, 'bytes': 0, + 'first_off': off, 'first_addr': addr, 'last_addr': addr}) + e['bytes'] += 1 + e['last_addr'] = addr + return per, ndiff, size_mismatch + + +def report(focus=None, as_json=False, built_path=BUILT, ref_path=REF, map_path=MAP, gap=64): + for p in (built_path, ref_path, map_path): + if not os.path.exists(p): + print(f"REFUSED — missing {p}", file=sys.stderr) + return 2 + built, ref = open(built_path, 'rb').read(), open(ref_path, 'rb').read() + sections, syms = parse_map(map_path) + if not sections or not syms: + print(f"REFUSED — {map_path} yielded {len(sections)} sections / {len(syms)} symbols; " + f"the map format is not what this tool parses, and a localizer that silently " + f"attributes nothing is worse than none (R43).", file=sys.stderr) + return 2 + per, ndiff, size_mismatch = attribute(built, ref, sections, syms, gap) + rows = sorted(per.values(), key=lambda e: -e['bytes']) + + if as_json: + print(json.dumps({'diff_bytes': ndiff, 'file_bytes': len(ref), + 'size_mismatch': size_mismatch, 'focus': focus, + 'symbols': rows}, indent=1)) + return 0 if ndiff == 0 else 1 + + if size_mismatch: + print(f"!! SIZE MISMATCH — built {len(built)} bytes, reference {len(ref)} bytes. " + f"Offsets past the shorter file are not compared.") + if ndiff == 0: + print(f"IDENTICAL — 0 differing bytes of {len(ref)}.") + return 0 + print(f"{ndiff} differing bytes of {len(ref)} ({100.0*ndiff/len(ref):.4f}%), " + f"across {len(rows)} symbol(s):\n") + print(f" {'bytes':>7} {'first addr':>10} symbol") + for e in rows[:40]: + a = f"0x{e['first_addr']:08x}" if e['first_addr'] is not None else f"@{e['first_off']}" + print(f" {e['bytes']:>7} {a:>10} {e['symbol']}" + + (f" [{e['obj']}]" if e['obj'] else '')) + if len(rows) > 40: + print(f" ... and {len(rows)-40} more symbol(s) not listed " + f"({sum(r['bytes'] for r in rows[40:])} bytes)") + + if focus: + inside = per.get(focus, {}).get('bytes', 0) + outside = ndiff - inside + print() + if inside and not outside: + print(f"VERDICT — divergence is CONFINED TO {focus} ({inside} bytes). A real body " + f"reject. If match_one said closeness 0, suspect its .text-only blind spot " + f"(§405-A: a switch's .rodata jump table is invisible to it).") + elif outside and not inside: + print(f"VERDICT — {focus} is BYTE-IDENTICAL; all {outside} differing bytes are " + f"ELSEWHERE. This is a PLUMBING reject, not a body reject: the substitution " + f"perturbed other code (§376 -- a stale forward declaration changes caller " + f"codegen). Route to fix_arity_callers -> cast_self_callers -> --sync-decls.") + elif inside and outside: + print(f"VERDICT — MIXED: {inside} bytes inside {focus}, {outside} elsewhere. The body " + f"verdict is UNPROVEN until the {outside} outside bytes are fixed and it is " + f"re-gated (a perturbed caller can also perturb the callee's own codegen).") + else: + print(f"VERDICT — {focus} is not among the divergent symbols and neither is anything " + f"attributable to it; check the name (it must match the linker map exactly).") + return 1 + + +def self_test(addr_hex, map_path=MAP, ref_path=REF): + """Flip one byte at a KNOWN address and assert the tool names the containing symbol. + + R39 / `check-against-a-known-true-case`. The identical-pair direction is asserted too: a + localizer that reports a diff on identical inputs would make every verdict above worthless.""" + addr = int(addr_hex, 16) + ref = open(ref_path, 'rb').read() + sections, syms = parse_map(map_path) + addrs = [s[0] for s in syms] + want, _ = addr_to_sym(syms, addrs, addr) + off = None + for vma, size, lma in sections: + if vma <= addr < vma + size: + off = lma + (addr - vma) + if off is None: + print(f"SELF-TEST REFUSED — 0x{addr:08x} is in no output section") + return 2 + ok = True + + # direction 1: identical inputs must report zero + per, ndiff, _ = attribute(ref, ref, sections, syms) + print(f" identical-pair -> {ndiff} differing bytes, {len(per)} symbols " + f"{'OK' if ndiff == 0 and not per else 'FAIL'}") + ok &= (ndiff == 0 and not per) + + # direction 2: a known one-byte perturbation must be attributed to the containing symbol + mut = bytearray(ref) + mut[off] ^= 0xFF + per, ndiff, _ = attribute(bytes(mut), ref, sections, syms) + got = list(per) + hit = (ndiff == 1 and got == [want]) + print(f" 1-byte flip at 0x{addr:08x} (file offset {off}) -> {ndiff} byte(s) in {got}; " + f"expected exactly ['{want}'] {'OK' if hit else 'FAIL'}") + ok &= hit + print('SELF-TEST', 'PASS' if ok else 'FAIL') + return 0 if ok else 1 + + +if __name__ == '__main__': + os.chdir(REPO) + ap = argparse.ArgumentParser() + ap.add_argument('--built', default=BUILT) + ap.add_argument('--ref', default=REF) + ap.add_argument('--map', dest='map_path', default=MAP) + ap.add_argument('--focus', help='the function you substituted; adds an attribution verdict') + ap.add_argument('--gap', type=int, default=64, help='coalesce runs separated by <= N bytes') + ap.add_argument('--json', action='store_true') + ap.add_argument('--self-test', metavar='ADDR', + help='negative control: flip one byte at ADDR and assert attribution') + a = ap.parse_args() + if a.self_test: + sys.exit(self_test(a.self_test, a.map_path, a.ref)) + sys.exit(report(a.focus, a.json, a.built, a.ref, a.map_path, a.gap))