From a8ae190fa3459be4b5eea9d9dc83d8bf99fbd1a4 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Tue, 16 Jun 2026 00:56:35 -0600 Subject: [PATCH] =?UTF-8?q?feat(phase-11):=20T3=20=E2=80=94=20intra-binary?= =?UTF-8?q?=20collapse=20PROOF=20(one=20body,=20two=20addresses,=20byte-id?= =?UTF-8?q?entical)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - src/shared/clearTbl40.h: CLEAR_TBL40 macro = the matched byte-clear loop body, authored ONCE; instantiated at both func_80037004 and func_80037334 in src/800.c (one source -> two vrams) - matched first try; clean rebuild main -> 143dbb89 BYTE-IDENTICAL WITH the shared C AND WITHOUT it (INCLUDE_ASM stub fallback) -> dual invariant proven (R22 clean rebuilds, both states) - config/dedup.us.yaml: I0_clearTbl40 group registered (h_exact a0744d60…); dedup_integrate --check validates it; negative tests (corrupt hash / wrong vram) fail-closed (exit 1) - tools/progress.py: count dedup-shared members as REAL via the registry (the macro form isn't a parseable function def); REAL 52 -> 54, byte-identical 50.24% -> 50.33%; honest measurement (P9) - tools/dedup_integrate.py: display vram in hex in diagnostics - the machinery half of the Phase-11 milestone is proven on the byte-verified EXE --- config/dedup.us.yaml | 14 +++++++++++++- docs/difficulty.md | 4 ++-- docs/progress.md | 11 ++++++----- phase-ends/CURRENT_PHASE.md | 11 ++++++++--- src/800.c | 5 +++-- src/shared/clearTbl40.h | 21 +++++++++++++++++++++ tools/dedup_integrate.py | 7 ++++--- tools/progress.py | 22 ++++++++++++++++++++++ 8 files changed, 79 insertions(+), 16 deletions(-) create mode 100644 src/shared/clearTbl40.h diff --git a/config/dedup.us.yaml b/config/dedup.us.yaml index 59f5242f7..69e3acdc3 100644 --- a/config/dedup.us.yaml +++ b/config/dedup.us.yaml @@ -46,4 +46,16 @@ # - { binary: ovSC01_077, vram: 0x8012XXXX, name: FUN_8012XXXX } # ============================================================================= version: 1 -groups: [] +groups: + # I0 — Phase-11 intra-binary proof: the byte-identical pair func_80037004 / func_80037334 + # (9-ins byte-clear loop) matched ONCE in src/shared/clearTbl40.h, instantiated at both sites + # in src/800.c. Clean rebuild of `main` stays 143dbb89 with the shared C and with INCLUDE_ASM + # stubs (both byte-identical). Proves the source-level share machinery + the byte-honesty gate. + - id: I0_clearTbl40 + tier: h_exact + hash: a0744d60bfa18d153690d4932282d5e9c703ccdd + source: src/shared/clearTbl40.h + func: CLEAR_TBL40 + members: + - { binary: main, vram: 0x80037004, name: func_80037004 } + - { binary: main, vram: 0x80037334, name: func_80037334 } diff --git a/docs/difficulty.md b/docs/difficulty.md index e39a801b6..8ad969cec 100644 --- a/docs/difficulty.md +++ b/docs/difficulty.md @@ -1,8 +1,8 @@ # Unmatched difficulty inventory (generated by tools/difficulty.py — harvest queue) -unmatched functions : 1995 +unmatched functions : 1993 trivial (<=5 ins) : 291 -non-jtbl leaves : 1117 (best harvest targets) +non-jtbl leaves : 1115 (best harvest targets) jump-table funcs : 80 (deferred — need the rodata-island workflow, Task 2') ## Easiest 120 unmatched (score asc) — the work queue diff --git a/docs/progress.md b/docs/progress.md index 2df82c520..a7846755d 100644 --- a/docs/progress.md +++ b/docs/progress.md @@ -1,18 +1,19 @@ # BFM matching progress (generated by tools/progress.py — authoritative) -REAL substantive matches : 52 <- the Gen1-exit >=25 bar counts THIS +REAL substantive matches : 54 <- the Gen1-exit >=25 bar counts THIS + (of which dedup-shared : 2 one body -> N sites, config/dedup.us.yaml) LINKED real PsyQ objects : 959 <- byte-identical via linked SDK objects NON_MATCHING (near-miss) : 7 splat-auto empty no-ops : 42 -INCLUDE_ASM stubs : 1036 +INCLUDE_ASM stubs : 1034 data blobs (excluded) : 2 ---------------------------------------- matchable functions : 2096 -REAL / matchable : 52 / 2096 = 2.48% -byte-identical/ matchable: 1053 / 2096 = 50.24% (REAL+LINKED+empties) +REAL / matchable : 54 / 2096 = 2.58% +byte-identical/ matchable: 1055 / 2096 = 50.33% (REAL+LINKED+empties) LINKED subsegs: apicard1 apicard2 apicard3 apicard4 libc2_1 libc2_2 libcd1 libcd2 libetc libgpu libgs1 libgs2 libgs3 libgs4 libgs5 libgs6 libgte1 libgte10 libgte11 libgte12 libgte13 libgte14 libgte15 libgte16 libgte17 libgte18 libgte19 libgte2 libgte20 libgte21 libgte22 libgte3 libgte4 libgte5 libgte6 libgte7 libgte8 libgte9 libmcrd1 libmcrd2 snd1 snd2 snd3 snd4 snd5 snd6 snd7 snd8 snd9 (959 fns) -REAL matches: CdQueueBusy CdReadRequest DebugMenuHandler GameModeDispatch LoaderResetReadState LzssDecodeSector ResourceGetCdLoc func_80012AB0 func_800168B4 func_80018F20 func_80019018 func_80019198 func_80019378 func_80019388 func_80019398 func_800193A8 func_8001AA78 func_8001AA88 func_8001B22C func_8001B374 func_8001B384 func_8001B85C func_8001BE20 func_8001BFA0 func_8001BFE8 func_8001D0E8 func_80029254 func_80029264 func_80029504 func_80029514 func_8002953C func_8002954C func_8002957C func_8002958C func_80029FD4 func_8002A26C func_8002A27C func_8002A4B8 func_8002A4C8 func_8002A728 func_8002A738 func_8002A998 func_8002A9A8 func_8002AEF8 func_8002AF08 func_8002AF60 func_8002D4B8 func_8002D7FC func_8002D834 func_8002F648 func_80037358 func_80037CC8 +REAL matches: CdQueueBusy CdReadRequest DebugMenuHandler GameModeDispatch LoaderResetReadState LzssDecodeSector ResourceGetCdLoc func_80012AB0 func_800168B4 func_80018F20 func_80019018 func_80019198 func_80019378 func_80019388 func_80019398 func_800193A8 func_8001AA78 func_8001AA88 func_8001B22C func_8001B374 func_8001B384 func_8001B85C func_8001BE20 func_8001BFA0 func_8001BFE8 func_8001D0E8 func_80029254 func_80029264 func_80029504 func_80029514 func_8002953C func_8002954C func_8002957C func_8002958C func_80029FD4 func_8002A26C func_8002A27C func_8002A4B8 func_8002A4C8 func_8002A728 func_8002A738 func_8002A998 func_8002A9A8 func_8002AEF8 func_8002AF08 func_8002AF60 func_8002D4B8 func_8002D7FC func_8002D834 func_8002F648 func_80037004 func_80037334 func_80037358 func_80037CC8 NON_MATCHING: CdReadSectorReadyCB CdReadStateMachine LoaderInitFileTable ResourceLoadStateMachine StreamLoadStateMachine func_80015A74 func_80016714 build SHA1: 143dbb89f34491258bbc27810d0a12ec8b43a8dd (byte-identical) diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index bccd7e534..dc368fbba 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -23,9 +23,10 @@ - [x] **T2** [Max] — `config/dedup.us.yaml` schema + `tools/dedup_integrate.py` (byte-honesty VALIDATOR, fail-closed on hash drift) + `make report` dedup-check gate. **DONE.** Verified: --check passes on empty registry; clean rebuild main → `143dbb89` (no-op proven, R22). *Refinement below.* -- [ ] **T3** [xHigh→Max] — intra-binary collapse PROOF (`func_80037004`→shared body, fill both addrs via - `src/shared/` macro). Verify clean rebuild `143dbb89` WITH the shared-C match AND WITHOUT (INCLUDE_ASM stub - fallback); dedup-check validates the registered group. ← **CURRENT TASK** +- [x] **T3** [xHigh] — intra-binary collapse PROOF. **DONE.** `src/shared/clearTbl40.h` macro `CLEAR_TBL40` + instantiated at both `func_80037004`/`func_80037334` in `src/800.c`; matched first try. Clean rebuild + `143dbb89` WITH shared-C AND WITHOUT (stub) — dual invariant. I0 in `dedup.us.yaml`; dedup-check PASS; + negative tests (wrong hash/vram) fail-closed. progress REAL 52→54 (2 dedup-shared), 50.33%. - [ ] **T4** [Max] — `tools/sig_image.py`: `h_exact` + boundary detection. Verify ≥99% h_exact vs `.run/sig.resident.jsonl`, zero UNEXPLAINED. - [ ] **T5** [Max] — `sig_image.py`: `h_norm` normToken replica + resident acceptance gate (≥98% non-GTE; GTE/noncontig @@ -62,3 +63,7 @@ change; 0 NON_MATCHING in default build. EXE↔resident cross groups = 0. - 2026-06-16: **T2 done** — `dedup.us.yaml` registry + `dedup_integrate.py` validator + `make report` gate; clean rebuild main `143dbb89` (no-op, R22). Recorded deviation D1 (source-level share). Next: T3 proof. +- 2026-06-16: **T3 done** — machinery half of the milestone PROVEN. Shared body `CLEAR_TBL40` + (`src/shared/clearTbl40.h`) → `func_80037004`+`func_80037334`; matched first try; clean rebuild `143dbb89` + WITH shared-C and WITHOUT (stub); I0 validated, fail-closed negatives pass; REAL 52→54. `progress.py` taught to + count dedup members (registry source-of-truth). Next: T4 (sig_image h_exact) — the report half's tooling. diff --git a/src/800.c b/src/800.c index 7fcbb9e02..41cbd810b 100644 --- a/src/800.c +++ b/src/800.c @@ -1,5 +1,6 @@ #include "common.h" #include "psyq/libcd.h" +#include "shared/clearTbl40.h" /* dedup group I0: func_80037004 / func_80037334 share one body */ INCLUDE_ASM("asm/nonmatchings/800", func_800123F0); @@ -2715,13 +2716,13 @@ INCLUDE_ASM("asm/nonmatchings/800", func_80036F98); INCLUDE_ASM("asm/nonmatchings/800", func_80036FB0); -INCLUDE_ASM("asm/nonmatchings/800", func_80037004); +CLEAR_TBL40(func_80037004) /* dedup I0: shared body (src/shared/clearTbl40.h) */ INCLUDE_ASM("asm/nonmatchings/800", func_80037028); INCLUDE_ASM("asm/nonmatchings/800", func_80037144); -INCLUDE_ASM("asm/nonmatchings/800", func_80037334); +CLEAR_TBL40(func_80037334) /* dedup I0: shared body (src/shared/clearTbl40.h) */ extern s32 D_8007623C; void func_80037358(int posInt) { diff --git a/src/shared/clearTbl40.h b/src/shared/clearTbl40.h new file mode 100644 index 000000000..c979e5d66 --- /dev/null +++ b/src/shared/clearTbl40.h @@ -0,0 +1,21 @@ +/* src/shared/clearTbl40.h — shared body for the byte-identical pair func_80037004 / func_80037334 + * (Phase 11 dedup proof, group I0). The body is authored ONCE here as a macro and instantiated at + * each member site in src/800.c, so the same bytes land at both vrams (0x80037004 + 0x80037334). + * Registered in config/dedup.us.yaml; byte-honesty validated by tools/dedup_integrate.py. + * + * Clears 5 bytes of a table: D_80076251[i] = 0 for i = 0x40,0x30,0x20,0x10,0x00 (down-counting). */ +#ifndef SHARED_CLEARTBL40_H +#define SHARED_CLEARTBL40_H +#include "common.h" + +extern u8 D_80076251; + +#define CLEAR_TBL40(name) \ + void name(void) { \ + s32 i; \ + for (i = 0x40; i >= 0; i -= 0x10) { \ + (&D_80076251)[i] = 0; \ + } \ + } + +#endif /* SHARED_CLEARTBL40_H */ diff --git a/tools/dedup_integrate.py b/tools/dedup_integrate.py index 36c983414..a59671d9e 100644 --- a/tools/dedup_integrate.py +++ b/tools/dedup_integrate.py @@ -84,16 +84,17 @@ def check(groups, binary_filter=None): vram = _addr(m["vram"]) except (KeyError, ValueError): print(f"[FAIL] {gid}: bad/absent vram for {name} in {b}"); failures += 1; ok = False; continue + vhex = f"0x{vram:08x}" idx = sig_for(b) if idx is None: - print(f"[WARN] {gid}: {b} has no sig yet — cannot validate {name} @ {m.get('vram')}") + print(f"[WARN] {gid}: {b} has no sig yet — cannot validate {name} @ {vhex}") warnings += 1; ok = False; continue row = idx.get(vram) if row is None: - print(f"[FAIL] {gid}: {b}:{m.get('vram')} ({name}) not found in {b} signature"); failures += 1; ok = False; continue + print(f"[FAIL] {gid}: {b}:{vhex} ({name}) not found in {b} signature"); failures += 1; ok = False; continue got = row.get(tier) if got != want: - print(f"[FAIL] {gid}: {b}:{m.get('vram')} ({name}) {tier}={got} != recorded {want} — SHARE DRIFTED") + print(f"[FAIL] {gid}: {b}:{vhex} ({name}) {tier}={got} != recorded {want} — SHARE DRIFTED") failures += 1; ok = False if ok: validated += 1 diff --git a/tools/progress.py b/tools/progress.py index c44dfdd05..e3baca4a1 100644 --- a/tools/progress.py +++ b/tools/progress.py @@ -70,6 +70,22 @@ def linked_subsegs(): LINKED_SEGS = linked_subsegs() +def dedup_members(binary): + """Function names matched-once-and-shared via config/dedup.us.yaml for this binary. They are + hand-matched byte-identical C, but instantiated from a shared body (a macro in src/shared/), + so classify()'s function-definition scan does NOT recognize the macro form — count them REAL + via the registry instead (the source of truth for code shares, Phase 11). Returns a set.""" + p = ROOT / "config/dedup.us.yaml" + if not p.exists(): + return set() + try: + import yaml + data = yaml.safe_load(p.read_text()) or {} + return {m["name"] for g in (data.get("groups") or []) + for m in (g.get("members") or []) if m.get("binary") == binary} + except Exception: + return set() + def find_s(name): """Locate .s in any asm/nonmatchings// subdir (segments: boot, 800, ...).""" for p in sorted(ASM_ROOT.glob(f"*/{name}.s")): @@ -161,6 +177,10 @@ def main(): audit = '--audit' in sys.argv check = '--check' in sys.argv real, empty, nonmatching, stubs, blobs, linked = classify() + # Code-shared functions (dedup.us.yaml) are REAL byte-identical matches whose macro-instantiated + # form classify() doesn't parse — fold them in (dedup-safe) so the count stays honest (P9). + shared = sorted(dedup_members(BINARY) - set(real)) + real = sorted(set(real) | set(shared)) matchable = len(real) + len(empty) + len(nonmatching) + len(stubs) + len(linked) byteident = len(real) + len(linked) + len(empty) # all byte-identical in the build @@ -168,6 +188,8 @@ def main(): out.append("# BFM matching progress (generated by tools/progress.py — authoritative)") out.append("") out.append(f"REAL substantive matches : {len(real):5d} <- the Gen1-exit >=25 bar counts THIS") + if shared: + out.append(f" (of which dedup-shared : {len(shared):5d} one body -> N sites, config/dedup.us.yaml)") out.append(f"LINKED real PsyQ objects : {len(linked):5d} <- byte-identical via linked SDK objects") out.append(f"NON_MATCHING (near-miss) : {len(nonmatching):5d}") out.append(f"splat-auto empty no-ops : {len(empty):5d}")