From bb65d36341c666d3a9ac484b2e9cb0e08bc8a110 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Tue, 14 Jul 2026 02:49:51 -0600 Subject: [PATCH] =?UTF-8?q?fix(phase-26a):=20A1=20=E2=80=94=20dedup=5Finte?= =?UTF-8?q?grate=20was=20a=20gate=20that=20could=20print=20a=20FALSE=20GRE?= =?UTF-8?q?EN?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit's priority #1: a fail-closed byte-honesty validator whose silent skips nothing downstream can catch. Three false-green paths, all measured, all now fail-closed with negative controls. R33 FIRST (derive, don't re-derive). The registry makes two claims; the tool only ever checked one, and mis-described that one: C1 EQUIVALENCE ("these vrams hold the same code in the ORIGINAL") — checked against the sigs, which sign the ORIGINAL bytes. KEPT. But the docstring claimed it also caught SOURCE drift: it cannot. A sig is a property of the ROM, immutable w.r.t. src/. Source drift is caught by the BUILD. Docstring corrected (P9). C2 BANK ("matched once in the source header, instantiated at every member") — NEVER CHECKED. Now DERIVED from the build invariant: INCLUDE_ASM pastes the ORIGINAL asm, so a member NOT wrapped in it is byte-exact, and one that IS wrapped is not banked — whatever the registry says. C2a: the group's macro token must occur in its source file. C2b: no member may still be an INCLUDE_ASM stub. THE THREE FALSE GREENS 1. 1808 groups claimed a DEFINE_func_* macro; only 1801 exist. The 7 ghosts printed [ OK ] — hiding 532 member-instances / 22,344 instructions of REAL, UNBANKED work (4 fns matched in ov_SC01_077, still INCLUDE_ASM in the other 133 overlays). 2. An absent .run/sig..jsonl degraded to "0 validated, 0 failed" and EXIT 0. On a fresh clone the gate validated NOTHING and passed. Now fails; --allow-unsigned is the escape. 3. The bank claim was never checked at all. THE CAUSAL CHAIN (the audit's thesis in one example). 3 of the 4 hidden fns are defined in ov_SC01_077_jr_8012ACE0.c — a _jr_* split file. dedup_propagate.overlay_files allowlists only ("_a","_o0","_o0b","_after"), so the propagator could not SEE them; the group was registered anyway; dedup_integrate greenlit the lie. TWO silent-skip bugs compounding: one created the hole, the other hid it. Harvest fuel -> .run/audit/a1_harvest_fuel.json, banked in A5. BLAST RADIUS, MEASURED NOT PREDICTED (R14). Headline metrics UNCHANGED to the decimal (instr-weighted 66.5%, distinct-code 46.8%) — weighted_metrics() derives from the invariant and was structurally immune to the lying registry. FLEET REAL substantive unchanged (282,466): progress.py had already been taught to distrust it (commit:0574). Only dedup_integrate still believed it. A null result that CONFIRMS R33: the tool that refused to re-derive was the one that was right. - registry repaired: 1813 -> 1806 groups (7 ghosts removed; instances 223,725 -> 222,787) - make report GREEN end-to-end: 1806 validated, 0 failed | C1 coverage 222,787/222,787 signed - negative controls: stubbed member -> exit 1; missing sig -> exit 1; --allow-unsigned -> exit 0 - report-only tool: no compiled artifact depends on it, so no R22 clean-fleet is owed here --- config/dedup.us.yaml | 49 ---------- docs/progress.fleet.md | 2 +- tools/dedup_integrate.py | 194 ++++++++++++++++++++++++++++++++------- 3 files changed, 162 insertions(+), 83 deletions(-) diff --git a/config/dedup.us.yaml b/config/dedup.us.yaml index 987abbea67..e45760a994 100644 --- a/config/dedup.us.yaml +++ b/config/dedup.us.yaml @@ -11503,55 +11503,6 @@ groups: func: DEFINE_func_80166054 vram: 0x80166054 binaries: [ov_SC01_077, ov_SC01_005, ov_SC01_006, ov_SC03_001, ov_SC01_000, ov_SC01_001, ov_SC02_000, ov_SC02_003, ov_SC03_002, ov_SC03_006, ov_SC04_000, ov_SC04_018, ov_SC04_019, ov_SC05_000, ov_SC06_000, ov_SC07_000, ov_SC01_004, ov_SC01_008, ov_SC01_009, ov_SC01_074, ov_SC01_080, ov_SC01_084, ov_SC02_004, ov_SC02_005, ov_SC02_011, ov_SC02_015, ov_SC02_016, ov_SC02_017, ov_SC02_021, ov_SC02_026, ov_SC02_027, ov_SC02_028, ov_SC02_031, ov_SC02_035, ov_SC02_039, ov_SC02_041, ov_SC03_003, ov_SC03_007, ov_SC03_010, ov_SC03_011, ov_SC03_012, ov_SC03_013, ov_SC03_014, ov_SC03_015, ov_SC03_023, ov_SC03_024, ov_SC03_028, ov_SC03_029, ov_SC03_030, ov_SC03_031, ov_SC03_089, ov_SC03_090, ov_SC03_091, ov_SC03_092, ov_SC03_093, ov_SC03_094, ov_SC03_095, ov_SC03_096, ov_SC03_097, ov_SC03_098, ov_SC03_099, ov_SC03_100, ov_SC03_101, ov_SC03_102, ov_SC03_103, ov_SC03_104, ov_SC03_105, ov_SC03_108, ov_SC03_109, ov_SC03_110, ov_SC03_111, ov_SC03_112, ov_SC03_113, ov_SC03_114, ov_SC03_115, ov_SC03_116, ov_SC03_117, ov_SC03_118, ov_SC03_119, ov_SC03_121, ov_SC03_124, ov_SC03_125, ov_SC03_126, ov_SC04_002, ov_SC04_003, ov_SC04_004, ov_SC04_005, ov_SC04_006, ov_SC04_007, ov_SC04_008, ov_SC04_009, ov_SC04_010, ov_SC04_011, ov_SC04_012, ov_SC04_015, ov_SC04_016, ov_SC04_020, ov_SC04_021, ov_SC05_001, ov_SC05_002, ov_SC05_003, ov_SC05_004, ov_SC05_005, ov_SC05_006, ov_SC05_007, ov_SC05_008, ov_SC05_009, ov_SC05_010, ov_SC05_011, ov_SC05_017, ov_SC05_018, ov_SC05_019, ov_SC06_006, ov_SC06_008, ov_SC06_010, ov_SC06_011, ov_SC06_013, ov_SC06_014, ov_SC06_015, ov_SC06_016, ov_SC06_018, ov_SC06_020, ov_SC06_022, ov_SC06_024, ov_SC06_025, ov_SC06_027, ov_SC06_029, ov_SC06_030, ov_SC06_032, ov_SC06_033, ov_SC07_001, ov_SC07_002, ov_SC07_008, ov_SC07_009] - - id: E_func_8012C098 - tier: h_exact - hash: df79b18a903e7a59b72fd5274a1e4cfc9b716cfd - source: src/shared/engine_core.h - func: DEFINE_func_8012C098 - vram: 0x8012C098 - binaries: [ov_SC01_077, ov_SC01_005, ov_SC01_006, ov_SC03_001, ov_SC01_000, ov_SC01_001, ov_SC02_000, ov_SC02_003, ov_SC03_002, ov_SC03_006, ov_SC04_000, ov_SC04_018, ov_SC04_019, ov_SC05_000, ov_SC06_000, ov_SC07_000, ov_SC01_004, ov_SC01_008, ov_SC01_009, ov_SC01_074, ov_SC01_080, ov_SC01_084, ov_SC02_004, ov_SC02_005, ov_SC02_011, ov_SC02_015, ov_SC02_016, ov_SC02_017, ov_SC02_021, ov_SC02_026, ov_SC02_027, ov_SC02_028, ov_SC02_031, ov_SC02_035, ov_SC02_039, ov_SC02_041, ov_SC03_003, ov_SC03_007, ov_SC03_010, ov_SC03_011, ov_SC03_012, ov_SC03_013, ov_SC03_014, ov_SC03_015, ov_SC03_023, ov_SC03_024, ov_SC03_028, ov_SC03_029, ov_SC03_030, ov_SC03_031, ov_SC03_089, ov_SC03_090, ov_SC03_091, ov_SC03_092, ov_SC03_093, ov_SC03_094, ov_SC03_095, ov_SC03_096, ov_SC03_097, ov_SC03_098, ov_SC03_099, ov_SC03_100, ov_SC03_101, ov_SC03_102, ov_SC03_103, ov_SC03_104, ov_SC03_105, ov_SC03_108, ov_SC03_109, ov_SC03_110, ov_SC03_111, ov_SC03_112, ov_SC03_113, ov_SC03_114, ov_SC03_115, ov_SC03_116, ov_SC03_117, ov_SC03_118, ov_SC03_119, ov_SC03_121, ov_SC03_124, ov_SC03_125, ov_SC03_126, ov_SC04_002, ov_SC04_003, ov_SC04_004, ov_SC04_005, ov_SC04_006, ov_SC04_007, ov_SC04_008, ov_SC04_009, ov_SC04_010, ov_SC04_011, ov_SC04_012, ov_SC04_015, ov_SC04_016, ov_SC04_020, ov_SC04_021, ov_SC05_001, ov_SC05_002, ov_SC05_003, ov_SC05_004, ov_SC05_005, ov_SC05_006, ov_SC05_007, ov_SC05_008, ov_SC05_009, ov_SC05_010, ov_SC05_011, ov_SC05_017, ov_SC05_018, ov_SC05_019, ov_SC06_006, ov_SC06_008, ov_SC06_010, ov_SC06_011, ov_SC06_013, ov_SC06_014, ov_SC06_015, ov_SC06_016, ov_SC06_018, ov_SC06_020, ov_SC06_022, ov_SC06_024, ov_SC06_025, ov_SC06_027, ov_SC06_029, ov_SC06_030, ov_SC06_032, ov_SC06_033, ov_SC07_001, ov_SC07_002, ov_SC07_008, ov_SC07_009] - - id: E_func_8012F14C - tier: h_exact - hash: 9a2982b698fb11ab38b82a0cc962f161f4fc4aae - source: src/shared/engine_core.h - func: DEFINE_func_8012F14C - vram: 0x8012F14C - binaries: [ov_SC01_077, ov_SC01_005, ov_SC01_006, ov_SC03_001, ov_SC01_000, ov_SC01_001, ov_SC02_000, ov_SC02_003, ov_SC03_002, ov_SC03_006, ov_SC04_000, ov_SC04_018, ov_SC04_019, ov_SC05_000, ov_SC06_000, ov_SC07_000, ov_SC01_004, ov_SC01_008, ov_SC01_009, ov_SC01_074, ov_SC01_080, ov_SC01_084, ov_SC02_004, ov_SC02_005, ov_SC02_011, ov_SC02_015, ov_SC02_016, ov_SC02_017, ov_SC02_021, ov_SC02_026, ov_SC02_027, ov_SC02_028, ov_SC02_031, ov_SC02_035, ov_SC02_039, ov_SC02_041, ov_SC03_003, ov_SC03_007, ov_SC03_010, ov_SC03_011, ov_SC03_012, ov_SC03_013, ov_SC03_014, ov_SC03_015, ov_SC03_023, ov_SC03_024, ov_SC03_028, ov_SC03_029, ov_SC03_030, ov_SC03_031, ov_SC03_089, ov_SC03_090, ov_SC03_091, ov_SC03_092, ov_SC03_093, ov_SC03_094, ov_SC03_095, ov_SC03_096, ov_SC03_097, ov_SC03_098, ov_SC03_099, ov_SC03_100, ov_SC03_101, ov_SC03_102, ov_SC03_103, ov_SC03_104, ov_SC03_105, ov_SC03_108, ov_SC03_109, ov_SC03_110, ov_SC03_111, ov_SC03_112, ov_SC03_113, ov_SC03_114, ov_SC03_115, ov_SC03_116, ov_SC03_117, ov_SC03_118, ov_SC03_119, ov_SC03_121, ov_SC03_124, ov_SC03_125, ov_SC03_126, ov_SC04_002, ov_SC04_003, ov_SC04_004, ov_SC04_005, ov_SC04_006, ov_SC04_007, ov_SC04_008, ov_SC04_009, ov_SC04_010, ov_SC04_011, ov_SC04_012, ov_SC04_015, ov_SC04_016, ov_SC04_020, ov_SC04_021, ov_SC05_001, ov_SC05_002, ov_SC05_003, ov_SC05_004, ov_SC05_005, ov_SC05_006, ov_SC05_007, ov_SC05_008, ov_SC05_009, ov_SC05_010, ov_SC05_011, ov_SC05_017, ov_SC05_018, ov_SC05_019, ov_SC06_006, ov_SC06_008, ov_SC06_010, ov_SC06_011, ov_SC06_013, ov_SC06_014, ov_SC06_015, ov_SC06_016, ov_SC06_018, ov_SC06_020, ov_SC06_022, ov_SC06_024, ov_SC06_025, ov_SC06_027, ov_SC06_029, ov_SC06_030, ov_SC06_032, ov_SC06_033, ov_SC07_001, ov_SC07_002, ov_SC07_008, ov_SC07_009] - - id: E_func_8012F038 - tier: h_exact - hash: b4078382239b7b561f91de91820e9adda6a0ebee - source: src/shared/engine_core.h - func: DEFINE_func_8012F038 - vram: 0x8012F038 - binaries: [ov_SC01_077, ov_SC01_005, ov_SC01_006, ov_SC03_001, ov_SC01_000, ov_SC01_001, ov_SC02_000, ov_SC02_003, ov_SC03_002, ov_SC03_006, ov_SC04_000, ov_SC04_018, ov_SC04_019, ov_SC05_000, ov_SC06_000, ov_SC07_000, ov_SC01_004, ov_SC01_008, ov_SC01_009, ov_SC01_074, ov_SC01_080, ov_SC01_084, ov_SC02_004, ov_SC02_005, ov_SC02_011, ov_SC02_015, ov_SC02_016, ov_SC02_017, ov_SC02_021, ov_SC02_026, ov_SC02_027, ov_SC02_028, ov_SC02_031, ov_SC02_035, ov_SC02_039, ov_SC02_041, ov_SC03_003, ov_SC03_007, ov_SC03_010, ov_SC03_011, ov_SC03_012, ov_SC03_013, ov_SC03_014, ov_SC03_015, ov_SC03_023, ov_SC03_024, ov_SC03_028, ov_SC03_029, ov_SC03_030, ov_SC03_031, ov_SC03_089, ov_SC03_090, ov_SC03_091, ov_SC03_092, ov_SC03_093, ov_SC03_094, ov_SC03_095, ov_SC03_096, ov_SC03_097, ov_SC03_098, ov_SC03_099, ov_SC03_100, ov_SC03_101, ov_SC03_102, ov_SC03_103, ov_SC03_104, ov_SC03_105, ov_SC03_108, ov_SC03_109, ov_SC03_110, ov_SC03_111, ov_SC03_112, ov_SC03_113, ov_SC03_114, ov_SC03_115, ov_SC03_116, ov_SC03_117, ov_SC03_118, ov_SC03_119, ov_SC03_121, ov_SC03_124, ov_SC03_125, ov_SC03_126, ov_SC04_002, ov_SC04_003, ov_SC04_004, ov_SC04_005, ov_SC04_006, ov_SC04_007, ov_SC04_008, ov_SC04_009, ov_SC04_010, ov_SC04_011, ov_SC04_012, ov_SC04_015, ov_SC04_016, ov_SC04_020, ov_SC04_021, ov_SC05_001, ov_SC05_002, ov_SC05_003, ov_SC05_004, ov_SC05_005, ov_SC05_006, ov_SC05_007, ov_SC05_008, ov_SC05_009, ov_SC05_010, ov_SC05_011, ov_SC05_017, ov_SC05_018, ov_SC05_019, ov_SC06_006, ov_SC06_008, ov_SC06_010, ov_SC06_011, ov_SC06_013, ov_SC06_014, ov_SC06_015, ov_SC06_016, ov_SC06_018, ov_SC06_020, ov_SC06_022, ov_SC06_024, ov_SC06_025, ov_SC06_027, ov_SC06_029, ov_SC06_030, ov_SC06_032, ov_SC06_033, ov_SC07_001, ov_SC07_002, ov_SC07_008, ov_SC07_009] - - id: E_func_8012C0EC - tier: h_exact - hash: e05b7bff0ee2e0e138192fdd29894c97ebaa971e - source: src/shared/engine_core.h - func: DEFINE_func_8012C0EC - vram: 0x8012C0EC - binaries: [ov_SC01_077, ov_SC01_005, ov_SC01_006, ov_SC03_001, ov_SC01_000, ov_SC01_001, ov_SC02_000, ov_SC02_003, ov_SC03_002, ov_SC03_006, ov_SC04_000, ov_SC04_018, ov_SC04_019, ov_SC05_000, ov_SC06_000, ov_SC07_000, ov_SC01_004, ov_SC01_008, ov_SC01_009, ov_SC01_074, ov_SC01_080, ov_SC01_084, ov_SC02_004, ov_SC02_005, ov_SC02_011, ov_SC02_015, ov_SC02_016, ov_SC02_017, ov_SC02_021, ov_SC02_026, ov_SC02_027, ov_SC02_028, ov_SC02_031, ov_SC02_035, ov_SC02_039, ov_SC02_041, ov_SC03_003, ov_SC03_007, ov_SC03_010, ov_SC03_011, ov_SC03_012, ov_SC03_013, ov_SC03_014, ov_SC03_015, ov_SC03_023, ov_SC03_024, ov_SC03_028, ov_SC03_029, ov_SC03_030, ov_SC03_031, ov_SC03_089, ov_SC03_090, ov_SC03_091, ov_SC03_092, ov_SC03_093, ov_SC03_094, ov_SC03_095, ov_SC03_096, ov_SC03_097, ov_SC03_098, ov_SC03_099, ov_SC03_100, ov_SC03_101, ov_SC03_102, ov_SC03_103, ov_SC03_104, ov_SC03_105, ov_SC03_108, ov_SC03_109, ov_SC03_110, ov_SC03_111, ov_SC03_112, ov_SC03_113, ov_SC03_114, ov_SC03_115, ov_SC03_116, ov_SC03_117, ov_SC03_118, ov_SC03_119, ov_SC03_121, ov_SC03_124, ov_SC03_125, ov_SC03_126, ov_SC04_002, ov_SC04_003, ov_SC04_004, ov_SC04_005, ov_SC04_006, ov_SC04_007, ov_SC04_008, ov_SC04_009, ov_SC04_010, ov_SC04_011, ov_SC04_012, ov_SC04_015, ov_SC04_016, ov_SC04_020, ov_SC04_021, ov_SC05_001, ov_SC05_002, ov_SC05_003, ov_SC05_004, ov_SC05_005, ov_SC05_006, ov_SC05_007, ov_SC05_008, ov_SC05_009, ov_SC05_010, ov_SC05_011, ov_SC05_017, ov_SC05_018, ov_SC05_019, ov_SC06_006, ov_SC06_008, ov_SC06_010, ov_SC06_011, ov_SC06_013, ov_SC06_014, ov_SC06_015, ov_SC06_016, ov_SC06_018, ov_SC06_020, ov_SC06_022, ov_SC06_024, ov_SC06_025, ov_SC06_027, ov_SC06_029, ov_SC06_030, ov_SC06_032, ov_SC06_033, ov_SC07_001, ov_SC07_002, ov_SC07_008, ov_SC07_009] - - id: E_func_8012E5CC - tier: h_exact - hash: 62ab8c1ddf81e7b3724acc9eadaeaeb9daf60331 - source: src/shared/engine_core.h - func: DEFINE_func_8012E5CC - vram: 0x8012E5CC - binaries: [ov_SC01_077, ov_SC01_005, ov_SC01_006, ov_SC03_001, ov_SC01_000, ov_SC01_001, ov_SC02_000, ov_SC02_003, ov_SC03_002, ov_SC03_006, ov_SC04_000, ov_SC04_018, ov_SC04_019, ov_SC05_000, ov_SC06_000, ov_SC07_000, ov_SC01_004, ov_SC01_008, ov_SC01_009, ov_SC01_074, ov_SC01_080, ov_SC01_084, ov_SC02_004, ov_SC02_005, ov_SC02_011, ov_SC02_015, ov_SC02_016, ov_SC02_017, ov_SC02_021, ov_SC02_026, ov_SC02_027, ov_SC02_028, ov_SC02_031, ov_SC02_035, ov_SC02_039, ov_SC02_041, ov_SC03_003, ov_SC03_007, ov_SC03_010, ov_SC03_011, ov_SC03_012, ov_SC03_013, ov_SC03_014, ov_SC03_015, ov_SC03_023, ov_SC03_024, ov_SC03_028, ov_SC03_029, ov_SC03_030, ov_SC03_031, ov_SC03_089, ov_SC03_090, ov_SC03_091, ov_SC03_092, ov_SC03_093, ov_SC03_094, ov_SC03_095, ov_SC03_096, ov_SC03_097, ov_SC03_098, ov_SC03_099, ov_SC03_100, ov_SC03_101, ov_SC03_102, ov_SC03_103, ov_SC03_104, ov_SC03_105, ov_SC03_108, ov_SC03_109, ov_SC03_110, ov_SC03_111, ov_SC03_112, ov_SC03_113, ov_SC03_114, ov_SC03_115, ov_SC03_116, ov_SC03_117, ov_SC03_118, ov_SC03_119, ov_SC03_121, ov_SC03_124, ov_SC03_125, ov_SC03_126, ov_SC04_002, ov_SC04_003, ov_SC04_004, ov_SC04_005, ov_SC04_006, ov_SC04_007, ov_SC04_008, ov_SC04_009, ov_SC04_010, ov_SC04_011, ov_SC04_012, ov_SC04_015, ov_SC04_016, ov_SC04_020, ov_SC04_021, ov_SC05_001, ov_SC05_002, ov_SC05_003, ov_SC05_004, ov_SC05_005, ov_SC05_006, ov_SC05_007, ov_SC05_008, ov_SC05_009, ov_SC05_010, ov_SC05_011, ov_SC05_017, ov_SC05_018, ov_SC05_019, ov_SC06_006, ov_SC06_008, ov_SC06_010, ov_SC06_011, ov_SC06_013, ov_SC06_014, ov_SC06_015, ov_SC06_016, ov_SC06_018, ov_SC06_020, ov_SC06_022, ov_SC06_024, ov_SC06_025, ov_SC06_027, ov_SC06_029, ov_SC06_030, ov_SC06_032, ov_SC06_033, ov_SC07_001, ov_SC07_002, ov_SC07_008, ov_SC07_009] - - id: E_func_8012C750 - tier: h_exact - hash: 0789df8727b76a56d8b21a76eec096717ba81b84 - source: src/shared/engine_core.h - func: DEFINE_func_8012C750 - vram: 0x8012C750 - binaries: [ov_SC01_077, ov_SC01_005, ov_SC01_006, ov_SC03_001, ov_SC01_000, ov_SC01_001, ov_SC02_000, ov_SC02_003, ov_SC03_002, ov_SC03_006, ov_SC04_000, ov_SC04_018, ov_SC04_019, ov_SC05_000, ov_SC06_000, ov_SC07_000, ov_SC01_004, ov_SC01_008, ov_SC01_009, ov_SC01_074, ov_SC01_080, ov_SC01_084, ov_SC02_004, ov_SC02_005, ov_SC02_011, ov_SC02_015, ov_SC02_016, ov_SC02_017, ov_SC02_021, ov_SC02_026, ov_SC02_027, ov_SC02_028, ov_SC02_031, ov_SC02_035, ov_SC02_039, ov_SC02_041, ov_SC03_003, ov_SC03_007, ov_SC03_010, ov_SC03_011, ov_SC03_012, ov_SC03_013, ov_SC03_014, ov_SC03_015, ov_SC03_023, ov_SC03_024, ov_SC03_028, ov_SC03_029, ov_SC03_030, ov_SC03_031, ov_SC03_089, ov_SC03_090, ov_SC03_091, ov_SC03_092, ov_SC03_093, ov_SC03_094, ov_SC03_095, ov_SC03_096, ov_SC03_097, ov_SC03_098, ov_SC03_099, ov_SC03_100, ov_SC03_101, ov_SC03_102, ov_SC03_103, ov_SC03_104, ov_SC03_105, ov_SC03_108, ov_SC03_109, ov_SC03_110, ov_SC03_111, ov_SC03_112, ov_SC03_113, ov_SC03_114, ov_SC03_115, ov_SC03_116, ov_SC03_117, ov_SC03_118, ov_SC03_119, ov_SC03_121, ov_SC03_124, ov_SC03_125, ov_SC03_126, ov_SC04_002, ov_SC04_003, ov_SC04_004, ov_SC04_005, ov_SC04_006, ov_SC04_007, ov_SC04_008, ov_SC04_009, ov_SC04_010, ov_SC04_011, ov_SC04_012, ov_SC04_015, ov_SC04_016, ov_SC04_020, ov_SC04_021, ov_SC05_001, ov_SC05_002, ov_SC05_003, ov_SC05_004, ov_SC05_005, ov_SC05_006, ov_SC05_007, ov_SC05_008, ov_SC05_009, ov_SC05_010, ov_SC05_011, ov_SC05_017, ov_SC05_018, ov_SC05_019, ov_SC06_006, ov_SC06_008, ov_SC06_010, ov_SC06_011, ov_SC06_013, ov_SC06_014, ov_SC06_015, ov_SC06_016, ov_SC06_018, ov_SC06_020, ov_SC06_022, ov_SC06_024, ov_SC06_025, ov_SC06_027, ov_SC06_029, ov_SC06_030, ov_SC06_032, ov_SC06_033, ov_SC07_001, ov_SC07_002, ov_SC07_008, ov_SC07_009] - - id: E_func_80128ED8 - tier: h_exact - hash: e40952016218d0585ccde06ec4d19bcca09a80b1 - source: src/shared/engine_core.h - func: DEFINE_func_80128ED8 - vram: 0x80128ED8 - binaries: [ov_SC01_077, ov_SC01_005, ov_SC01_006, ov_SC03_001, ov_SC01_000, ov_SC01_001, ov_SC02_000, ov_SC02_003, ov_SC03_002, ov_SC03_006, ov_SC04_000, ov_SC04_018, ov_SC04_019, ov_SC05_000, ov_SC06_000, ov_SC07_000, ov_SC01_004, ov_SC01_008, ov_SC01_009, ov_SC01_074, ov_SC01_080, ov_SC01_084, ov_SC02_004, ov_SC02_005, ov_SC02_011, ov_SC02_015, ov_SC02_016, ov_SC02_017, ov_SC02_021, ov_SC02_026, ov_SC02_027, ov_SC02_028, ov_SC02_031, ov_SC02_035, ov_SC02_039, ov_SC02_041, ov_SC03_003, ov_SC03_007, ov_SC03_010, ov_SC03_011, ov_SC03_012, ov_SC03_013, ov_SC03_014, ov_SC03_015, ov_SC03_023, ov_SC03_024, ov_SC03_028, ov_SC03_029, ov_SC03_030, ov_SC03_031, ov_SC03_089, ov_SC03_090, ov_SC03_091, ov_SC03_092, ov_SC03_093, ov_SC03_094, ov_SC03_095, ov_SC03_096, ov_SC03_097, ov_SC03_098, ov_SC03_099, ov_SC03_100, ov_SC03_101, ov_SC03_102, ov_SC03_103, ov_SC03_104, ov_SC03_105, ov_SC03_108, ov_SC03_109, ov_SC03_110, ov_SC03_111, ov_SC03_112, ov_SC03_113, ov_SC03_114, ov_SC03_115, ov_SC03_116, ov_SC03_117, ov_SC03_118, ov_SC03_119, ov_SC03_121, ov_SC03_124, ov_SC03_125, ov_SC03_126, ov_SC04_002, ov_SC04_003, ov_SC04_004, ov_SC04_005, ov_SC04_006, ov_SC04_007, ov_SC04_008, ov_SC04_009, ov_SC04_010, ov_SC04_011, ov_SC04_012, ov_SC04_015, ov_SC04_016, ov_SC04_020, ov_SC04_021, ov_SC05_001, ov_SC05_002, ov_SC05_003, ov_SC05_004, ov_SC05_005, ov_SC05_006, ov_SC05_007, ov_SC05_008, ov_SC05_009, ov_SC05_010, ov_SC05_011, ov_SC05_017, ov_SC05_018, ov_SC05_019, ov_SC06_006, ov_SC06_008, ov_SC06_010, ov_SC06_011, ov_SC06_013, ov_SC06_014, ov_SC06_015, ov_SC06_016, ov_SC06_018, ov_SC06_020, ov_SC06_022, ov_SC06_024, ov_SC06_025, ov_SC06_027, ov_SC06_029, ov_SC06_030, ov_SC06_032, ov_SC06_033, ov_SC07_001, ov_SC07_002, ov_SC07_008, ov_SC07_009] - id: E_func_8017CE24 tier: h_exact hash: 511a8b68802b6800ebb271e89b28604ee53e3401 diff --git a/docs/progress.fleet.md b/docs/progress.fleet.md index 7740e33bf4..4a46eab21f 100644 --- a/docs/progress.fleet.md +++ b/docs/progress.fleet.md @@ -8,7 +8,7 @@ FLEET fn-count byte-ident: 283995 / 343774 = 82.61% (REAL+LINKED+empties; FUNC FLEET instr-weighted : 8448285 / 12707182 = 66.5% (shipped .text across resident+134 overlays; the decomp.dev-DISPLAY number) FLEET distinct-code(uniq): 2530637 / 5410077 = 46.8% (51834/84996 unique fns; the DISTINCT-RE number; main EXE not sig'd) -FLEET REAL substantive : 282466 (of which dedup-shared 222743 via 1813 groups / 223725 instances) +FLEET REAL substantive : 282466 (of which dedup-shared 222743 via 1806 groups / 222787 instances) FLEET LINKED PsyQ objs : 959 FLEET NON_MATCHING : 7 (0 in any default build — G4) FLEET INCLUDE_ASM stubs : 59772 diff --git a/tools/dedup_integrate.py b/tools/dedup_integrate.py index 22bb8f8380..288ec97a1b 100644 --- a/tools/dedup_integrate.py +++ b/tools/dedup_integrate.py @@ -1,27 +1,50 @@ #!/usr/bin/env python3 """Cross-binary code-share registry validator (Phase 11, G7 "match once, share"). -Validates config/dedup.us.yaml — the registry of functions matched ONCE and shared across ->=2 duplicate sites/binaries via a source-level body (see config/dedup.us.yaml for the why). +Validates config/dedup.us.yaml — the registry of functions matched ONCE and shared across >=2 +duplicate sites/binaries via a source-level body (a macro in src/shared/), instantiated at each +member site so the same bytes land at each member vram. -The share itself is SOURCE-LEVEL: the matched body lives once in `source` (a macro header) -and is instantiated at each member site in that binary's game-code .c, so the same bytes land -at each member vram. The REAL byte-gate is the existing per-binary `make check BINARY=` -(the image is byte-identical or it is not) — this tool adds the BYTE-HONESTY check that makes a -share's claim trustworthy: every member's CURRENT signature hash must still equal the recorded -`hash`. If a shared function drifts (someone edits it and one site diverges), --check FAILS so -the stale share can never silently mislead (P9/G3). h_exact = guaranteed byte-match; h_norm = -candidate, accepted only when every claiming binary still builds byte-identical. +WHAT THE REGISTRY CLAIMS, AND HOW EACH CLAIM IS CHECKED (Phase 26-A tooling audit; R32/R33) +-------------------------------------------------------------------------------------------- +C1 EQUIVALENCE — "these member vrams hold the same code IN THE ORIGINAL BINARY." + Checked against .run/sig..jsonl, which signs the ORIGINAL bytes. A genuine EXTERNAL + oracle: it catches a group registered with a wrong hash, or a member vram whose original + bytes are not in the claimed equivalence class. + It does NOT — and cannot — detect SOURCE drift: a sig is a property of the ROM, so it is + immutable with respect to src/. (The pre-audit docstring claimed it did. Source drift is + caught by the BUILD, `make check`, which is the real byte-gate. P9.) -This is NOT a fork of psyq_integrate's object-swap: that works only for separate library -SUBSEGMENT stubs; game-code functions are interior to one object per binary, so the linker -cannot swap them — sharing is source-level. The .ld interpose stays the library mechanism. +C2 BANK — "the body is matched once in `source` (named `func`) and INSTANTIATED at every member." + DERIVED from the build invariant rather than re-parsed (R33): the fleet builds byte-identical, + and INCLUDE_ASM pastes the ORIGINAL assembly — therefore a member site NOT wrapped in + INCLUDE_ASM is byte-exact, and one that IS wrapped is NOT banked, whatever the registry says. + C2a the `func` token must actually occur in `source` (no group may name a macro nobody wrote) + C2b no member may still be an INCLUDE_ASM stub (no group may claim work never done) -Usage: tools/dedup_integrate.py [--check] [--binary ] [--dedup config/dedup.us.yaml] - (--apply is reserved for future per-site boilerplate generation; shares are source-level, - so there is no build-time action — --apply currently just validates.) +COVERAGE (R32) — the run reports found-vs-candidates and REFUSES to print a green it did not earn. + A member whose binary has no signature file is UNVALIDATED for C1, and the run FAILS unless + --allow-unsigned. Pre-audit this printed [WARN] and still exited 0 — so on a fresh clone, where + .run/sig.*.jsonl does not exist at all, the gate validated NOTHING and passed. + +WHY THIS TOOL IS AUDITED FIRST +------------------------------ +It is the last line of `make report` and is fail-closed by design, so a silent skip here is a FALSE +GREEN FROM A GATE and nothing downstream can catch it. The audit found three: + * C2a was greenlighting 7 groups whose `DEFINE_func_*` macro was never written (1808 claimed, + 1801 defined) — hiding 532 member-instances of real, unbanked work in 4 functions that are + matched in ov_SC01_077 and still INCLUDE_ASM in the other 133 overlays; + * a missing sig file degraded silently to "0 validated, 0 failed" and exit 0; + * the bank claim itself was never checked at all. + +This is NOT a fork of psyq_integrate's object-swap: that works only for separate library SUBSEGMENT +stubs; game-code functions are interior to one object per binary, so the linker cannot swap them — +sharing is source-level. The .ld interpose stays the library mechanism. + +Usage: tools/dedup_integrate.py [--check] [--binary ] [--allow-unsigned] + [--dedup config/dedup.us.yaml] """ -import argparse, json, pathlib, sys +import argparse, json, pathlib, re, sys ROOT = pathlib.Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT / "tools")) @@ -29,6 +52,11 @@ from dup_report import BINARIES # single source of truth for per-binary .run/si TIERS = ("h_exact", "h_norm") +# OVER-APPROXIMATING by design (R32): any C identifier, not just func_ — a curated symbol +# (e.g. listCdBuffer) is a stub too, and a `func_`-only pattern silently misses it. +INCLUDE_ASM_RE = re.compile(r'INCLUDE_ASM\([^)]*,\s*([A-Za-z_]\w*)\s*\)') +SYMBOL_LINE_RE = re.compile(r'^\s*([A-Za-z_]\w*)\s*=\s*(0x[0-9A-Fa-f]+)\s*;') + def _addr(v): """Accept a YAML int (0x.. parsed) or a hex string ('0x..' / bare hex) -> int.""" @@ -50,6 +78,59 @@ def group_members(g): yield b, a, nm +def _src_paths(binary): + """The .c files that make up a binary's source (main is the top-level src/*.c).""" + if binary == "main": + return sorted((ROOT / "src").glob("*.c")) + return sorted((ROOT / "src" / binary).glob("*.c")) + + +def _curated_syms(binary): + """curated name -> addr, from the symbol files that can name this binary's symbols.""" + out = {} + for cand in (f"symbols.{binary}.txt", "symbols.resident.txt", "symbols.us.txt"): + p = ROOT / "config" / cand + if not p.exists(): + continue + for line in p.read_text().splitlines(): + m = SYMBOL_LINE_RE.match(line) + if m: + out.setdefault(m.group(1), int(m.group(2), 16)) + return out + + +_STUB_CACHE = {} + + +def stub_addrs(binary): + """C2b's oracle: the set of vram addresses this binary still ships as INCLUDE_ASM. + + Resolves ANY C identifier, not just func_ — a curated stub name (listCdBuffer) is a stub. + An unresolvable stub symbol is a COVERAGE FAILURE, not a shrug: it means this oracle cannot + answer "is this member banked?" for that address, so we fail loud rather than guess 'banked'.""" + if binary in _STUB_CACHE: + return _STUB_CACHE[binary] + paths = _src_paths(binary) + if not paths: + sys.exit(f"dedup-check: binary {binary!r} has no source under src/ — cannot verify its bank claims") + syms = _curated_syms(binary) + addrs, unresolved = set(), [] + for p in paths: + for m in INCLUDE_ASM_RE.finditer(p.read_text()): + nm = m.group(1) + if re.fullmatch(r'func_[0-9A-Fa-f]{8}', nm): + addrs.add(int(nm[5:], 16)) + elif nm in syms: + addrs.add(syms[nm]) + else: + unresolved.append(f"{p.relative_to(ROOT)}: {nm}") + if unresolved: + sys.exit(f"dedup-check: {len(unresolved)} INCLUDE_ASM symbol(s) in {binary} resolve to no " + f"address — the bank oracle is blind to them:\n " + "\n ".join(unresolved[:10])) + _STUB_CACHE[binary] = addrs + return addrs + + def _load_sig_index(rel): """addr(int) -> sig row for .run/sig..jsonl, or None if the file is absent.""" p = ROOT / rel @@ -64,10 +145,9 @@ def _load_sig_index(rel): return idx -def check(groups, binary_filter=None): - """Validate every group's byte-honesty. Returns the failure count (0 = all honest). - A member whose binary has no signature file yet (e.g. an overlay before sig_image runs) is - UNVALIDATED (warn), never a hard failure; a present-but-mismatched hash is a hard FAIL.""" +def check(groups, binary_filter=None, allow_unsigned=False): + """Validate every group's C1 (equivalence, vs the original bytes) and C2 (bank, vs the build + invariant). Returns the failure count (0 = every claim honest AND fully covered).""" sigcache = {} def sig_for(binary): @@ -76,7 +156,17 @@ def check(groups, binary_filter=None): sigcache[binary] = _load_sig_index(cfg["sig"]) if cfg else None return sigcache[binary] - failures = warnings = validated = 0 + src_cache = {} + + def source_text(rel): + if rel not in src_cache: + p = ROOT / rel + src_cache[rel] = p.read_text() if p.exists() else None + return src_cache[rel] + + failures = unvalidated = validated = 0 + members_seen = members_c1 = 0 + for g in groups: gid = g.get("id", "?") members = list(group_members(g)) @@ -87,31 +177,67 @@ def check(groups, binary_filter=None): if len(members) < 2: print(f"[FAIL] {gid}: a share needs >=2 members (got {len(members)})"); failures += 1; continue src = g.get("source") - if not src or not (ROOT / src).exists(): + txt = source_text(src) if src else None + if txt is None: print(f"[FAIL] {gid}: shared source {src!r} missing"); failures += 1; continue + + # ---- C2a: the shared body this group names must actually EXIST in `source` --------------- + fn = g.get("func") + if not fn: + print(f"[FAIL] {gid}: no `func` (the group must name the shared body it claims)") + failures += 1; continue + if not re.search(rf'\b{re.escape(fn)}\b', txt): + print(f"[FAIL] {gid}: `func: {fn}` does NOT occur in {src} — the group claims a shared " + f"body that was never written (the share was registered but never propagated)") + failures += 1; continue + tier, want = g["tier"], g.get("hash") if not want: print(f"[FAIL] {gid}: no recorded hash"); failures += 1; continue + ok = True for (b, vram, name) in members: - vhex = f"0x{vram:08x}" + members_seen += 1 + + # ---- C2b: DERIVED from the build invariant — a member still stubbed is NOT banked ---- + if vram in stub_addrs(b): + print(f"[FAIL] {gid}: {b}:0x{vram:08x} ({name}) is claimed MATCHED but is still an " + f"INCLUDE_ASM stub — the registry is claiming work that was never done") + failures += 1; ok = False; continue + + # ---- C1: equivalence, against the ORIGINAL bytes ------------------------------------ idx = sig_for(b) if idx is None: - print(f"[WARN] {gid}: {b} has no sig yet — cannot validate {name} @ {vhex}") - warnings += 1; ok = False; continue + unvalidated += 1; ok = False + continue row = idx.get(vram) if row is None: - print(f"[FAIL] {gid}: {b}:{vhex} ({name}) not found in {b} signature"); failures += 1; ok = False; continue + print(f"[FAIL] {gid}: {b}:0x{vram:08x} ({name}) not found in {b} signature") + failures += 1; ok = False; continue got = row.get(tier) if got != want: - print(f"[FAIL] {gid}: {b}:{vhex} ({name}) {tier}={got} != recorded {want} — SHARE DRIFTED") - failures += 1; ok = False + print(f"[FAIL] {gid}: {b}:0x{vram:08x} ({name}) {tier}={got} != recorded {want} — SHARE DRIFTED") + failures += 1; ok = False; continue + members_c1 += 1 + if ok: validated += 1 uniq = sorted({b for b, _, _ in members}) disp = ",".join(uniq) if len(uniq) <= 6 else f"{len(uniq)} binaries" print(f"[ OK ] {gid}: {len(members)} members [{disp}] share {tier} {want[:12]}… (source {src})") - print(f"dedup-check: {validated} validated, {warnings} unvalidated (sig absent), {failures} failed") + + # ---- COVERAGE (R32): never print a green this run did not earn ------------------------------ + print(f"dedup-check: {validated} validated, {failures} failed " + f"| C1 coverage {members_c1}/{members_seen} members signed") + if unvalidated: + msg = (f"dedup-check: {unvalidated} member(s) UNVALIDATED — their binary has no " + f".run/sig..jsonl, so the equivalence claim could not be checked at all. " + f"Run `make sig-overlays` (or pass --allow-unsigned to accept an unverified registry).") + if allow_unsigned: + print(f"[WARN] {msg}") + else: + print(f"[FAIL] {msg}") + failures += unvalidated return failures @@ -119,7 +245,9 @@ def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--dedup", default="config/dedup.us.yaml") ap.add_argument("--binary", default=None, help="validate only groups with a member in this binary") - ap.add_argument("--check", action="store_true", help="validate byte-honesty (default mode)") + ap.add_argument("--check", action="store_true", help="validate (default mode)") + ap.add_argument("--allow-unsigned", action="store_true", + help="downgrade 'member has no sig file' from FAIL to WARN (fresh clone / pre-sig-refresh)") ap.add_argument("--apply", action="store_true", help="reserved (shares are source-level; validates only)") a = ap.parse_args() @@ -132,8 +260,8 @@ def main(): groups = data.get("groups") or [] if a.apply: print("dedup_integrate --apply: shares are SOURCE-LEVEL (authored in src/shared/, instantiated " - "per site); no build-time action — validating byte-honesty instead.") - if check(groups, a.binary): + "per site); no build-time action — validating instead.") + if check(groups, a.binary, a.allow_unsigned): sys.exit(1)