From 867f09221cf15dbfdba04606b7a08e11fdd6ede7 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:58:17 -0600 Subject: [PATCH] =?UTF-8?q?feat(oracle):=20main=20gets=20its=20independent?= =?UTF-8?q?=20second=20oracle=20=E2=80=94=20contract=20=C2=A71.3=20closed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The roadmap's completion contract requires both audit oracles green before any 100% claim on main, and main had none: audit-corpus covered overlays and resident only, and R34 is explicit that the byte gate is a perfect CORRECTNESS oracle and a NULL COVERAGE oracle — green whether a function was sliced right or invented, because the .s pieces paste back either way. sig_image gains multi-range signing, closing all three blockers docs/second-oracle.md scoped: * the 0x800 PS-X EXE header -> --vram-base 0x8000F800 puts file offset 0 at vram, so the header falls below the first range * interleaved data + linked islands -> --segments derives 28 game-code ranges from the splat yaml's SEGMENT rows * one text range -> the signer loops ranges, bootstrapping INSIDE each, which is what stops the linear partition running through a data island and minting functions out of it (the detector manufacturing the class it detects) INDEPENDENCE IS PRESERVED, NOT WORKED AROUND. Ranges come from segment TYPES, never from splat's function boundaries; entries are still found by byte-derived jal-closure. Seeding from splat's symbols would make every phantom look real — the trap the design doc names. .run/sig.main.jsonl (the splat-SEEDED atlas sig) is a different file and corpus.ORACLE_SIG keeps the audit off it. RESULT: 986 functions signed. main audit = 0 PHANTOM, 0 TRUNCATED, 1 PAD-TAIL. Fleet audit-corpus = 0 + 0, unchanged for resident and overlays. NEW AUDIT CLASS, from the first real finding. func_80062144: splat .s 65 ins, oracle 64 — the extra line is a nop one line BELOW endlabel. That is an alignment pad the matching side already emits from C (§295; two S77 wave agents did it on func_8005E13C and func_8005D538), not a mis-slice. Lumping it with TRUNCATED would make the oracle's first finding look like a defect and bury the class that is one. COVERAGE ASSERTED both ways before trusting it (R32): all 30 game-code stubs fall inside a range, and 0 of 199 addr-parseable LINKED stubs do. --- Makefile | 17 ++++++++ docs/second-oracle.md | 39 ++++++++++++++++++- tools/corpus.py | 91 +++++++++++++++++++++++++++++++++++++++---- tools/sig_image.py | 79 ++++++++++++++++++++++++++++++++++++- 4 files changed, 215 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 5bd3329c8..07ed66b7b 100644 --- a/Makefile +++ b/Makefile @@ -393,6 +393,23 @@ sig-main: $(VENV_PY) tools/sig_image.py --image $(main_EXE) --vram-base $(main_VRAM_BASE) --seeds .run/seeds.main.txt --name main echo "sig-main: signed $$(wc -l < .run/seeds.main.txt) main stubs (splat-true lengths) -> .run/sig.main.jsonl" +# sig-main-oracle (P31 S77) — MAIN'S INDEPENDENT SECOND ORACLE (roadmap contract §1.3). +# Distinct from `sig-main` above, which is splat-SEEDED on purpose. This one signs the ORIGINAL EXE +# bytes with NO splat symbols: `--vram-base 0x8000F800` puts file offset 0 at vram (so the 0x800 +# PS-X EXE header simply falls below the first range), and `--segments` derives the game-code ranges +# from the splat yaml's SEGMENT TYPES — coarse structure, never splat's FUNCTION boundaries, which +# is the thing the oracle must stay free to disagree with. Entries inside each range are found by +# byte-derived jal-closure, because seeding from splat's symbols would make every phantom look real +# (docs/second-oracle.md names that trap). LINKED PsyQ blocks are excluded: real library objects, +# outside the game-code denominator, and auditing them here would report ~960 phantoms that are +# artefacts of comparing two oracles that never measured the same thing. +sig-main-oracle: + @$(VENV_PY) -c "import sys;sys.path.insert(0,'tools');import progress;progress.set_binary('main');print(','.join(sorted(progress.LINKED_SEGS)))" > .run/main_linked_segs.txt + $(VENV_PY) tools/sig_image.py --image $(main_EXE) --vram-base 0x8000F800 \ + --segments config/splat.us.exe.yaml --exclude-subsegs "$$(cat .run/main_linked_segs.txt)" \ + --bootstrap --out .run/sig.main.oracle.jsonl + @$(VENV_PY) -c "import sys;sys.path.insert(0,'tools');import corpus;r=corpus.audit('main');print('sig-main-oracle: main is now INDEPENDENT — %d in-domain stubs, %d PHANTOM, %d TRUNCATED, %d PAD-TAIL'%(r['stubs'],len(r['phantom']),len(r['truncated']),len(r['pad_tail'])))" + # sig-modules (P30 S44): sign every module-class binary at ITS OWN vram (from modules.mk) with its # own TEXT_LO (the §154 module-id-word law: code starts past the header; bootstrap from offset 0 # yields 0 functions). Same derived-jobs shape as sig-overlays (R33). Empty MODULE_BINARIES = no-op. diff --git a/docs/second-oracle.md b/docs/second-oracle.md index 119577275..5c6cde544 100644 --- a/docs/second-oracle.md +++ b/docs/second-oracle.md @@ -12,7 +12,7 @@ |---|---|---|---| | 138 overlays | `sig_image` (byte-derived) | ✅ | validated 58,524/58,621 vs spimdisasm | | resident | `sig_image` (**Phase-27 T10**) | ✅ | `make sig-resident` — 144 fns, all 21 stubs present, 0 phantom/truncated | -| main | — | ❌ **deferred** | `sig_image` cannot yet sign the EXE (below) | +| main | `sig_image` multi-range (**P31 S77**) | ✅ | `make sig-main-oracle` — 28 game-code ranges, 986 fns, **0 phantom / 0 truncated / 1 pad-tail** | `corpus.sig_is_independent()` gates the check to exactly this domain: a boundary cross-check applied where the two oracles were never measuring the same thing is noise, not thoroughness (the audit's own @@ -25,7 +25,42 @@ sig — run `make sig-resident` before `make audit-corpus` (a stale Ghidra sig t "measuring Ghidra's limits" artefact). `h_exact` is raw-byte SHA1, so `weighted_metrics` is unaffected by the swap (fleet % unchanged to the decimal). -## Main EXE — why sig_image can't sign it yet (the deferral) +## Main EXE — DELIVERED (P31 S77). The deferral below is kept as the design record. + +`make sig-main-oracle` signs the ORIGINAL EXE bytes and `corpus.sig_is_independent("main")` is now +True whenever `.run/sig.main.oracle.jsonl` exists. All three structural blockers are closed: + +1. **The 0x800 header** — `--vram-base 0x8000F800` maps file offset 0 to vram, so the header simply + falls below the first code range. No `--skip` needed. +2. **Interleaved data + linked islands** — new `--segments ` + (`sig_image.code_ranges_from_splat`) derives **28 game-code ranges** from the yaml's SEGMENT rows. + It reads `[file_off, type, name]` and NOTHING else: segment *types* are coarse structure, not + splat's *function* boundaries, so the two oracles stay independent exactly where it matters. + `--exclude-subsegs` drops the LINKED PsyQ blocks. +3. **One text range** — the signer now loops ranges, bootstrapping entries INSIDE each. Per-range + discovery is what stops the linear partition running through a data island and minting functions + out of it, i.e. the detector manufacturing the phantom class it exists to detect. + +**The trap was avoided, not worked around.** Entries are still found by byte-derived jal-closure; +nothing is seeded from splat's symbols. `.run/sig.main.jsonl` (the splat-SEEDED atlas sig, P31 T3) +is a DIFFERENT file and the audit never reads it — `corpus.ORACLE_SIG` keeps them apart. + +**Domain (R14).** The oracle signs game code only, so `audit("main")` filters the LINKED stubs out. +Auditing them against it would report ~960 phantoms that are artefacts of comparing two oracles that +never measured the same thing — the same 914-vs-193 mistake this file already warns about. + +**First finding, and it is a real one.** `func_80062144`: splat's `.s` says 65 instructions, the +byte-derived boundary says 64. The extra line is a `nop` at `0x80062244`, emitted one line BELOW +`endlabel`. That is a **PAD-TAIL**, now its own audit class — a known alignment artefact the matching +side handles by emitting the pad from C (cookbook §295; two S77 wave agents did exactly that on +`func_8005E13C` and `func_8005D538`), not a splat mis-slice. Reporting it as TRUNCATED would have +made the oracle's first real finding look like a defect and buried the class that is one. + +**Fleet status after wiring:** `make audit-corpus` → **0 PHANTOM + 0 TRUNCATED**, +1 PAD-TAIL. + +--- + +## (Historical) Main EXE — why sig_image could not sign it `tools/sig_image.py` assumes a flat blob whose file offset 0 IS its vram base, one contiguous code region, and one `[lo,hi)` text range. The main EXE breaks all three — structurally, not with a flag: diff --git a/tools/corpus.py b/tools/corpus.py index 55c0136d3..e51e73068 100644 --- a/tools/corpus.py +++ b/tools/corpus.py @@ -322,6 +322,29 @@ def is_o0(src_path): # -------------------------------------------------------------------------------------------- # the invariant # -------------------------------------------------------------------------------------------- +# The INDEPENDENT boundary oracle, where that is a DIFFERENT file from the working sig (P31 S77). +# For main the working sig is deliberately splat-SEEDED (the atlas needs the boundaries a match must +# hit, P31 T3) — auditing against it would be a mirror, not an oracle. +ORACLE_SIG = {"main": ".run/sig.main.oracle.jsonl"} + + +def oracle_sig(binary): + """addr -> sig row from the INDEPENDENT signer (falls back to sig() where they are one file).""" + rel = ORACLE_SIG.get(binary) + if not rel: + return sig(binary) + full = os.path.join(REPO, rel) + if not os.path.exists(full): + raise CorpusError(f"{binary}: no independent sig at {rel} — run `make sig-main-oracle`") + out = {} + for line in open(full): + line = line.strip() + if line: + r = json.loads(line) + out[int(r["addr"], 16)] = r + return out + + @functools.lru_cache(maxsize=None) def sig(binary): """addr -> sig row (.run/sig..jsonl). Signs the ORIGINAL bytes: immutable w.r.t. src/.""" @@ -378,7 +401,17 @@ def sig_is_independent(binary): valid IFF .run/sig.resident.jsonl is the sig_image sig; run `make sig-resident` first (a stale Ghidra sig there would resurrect the 'measuring Ghidra's limits' artefact). MAIN stays excluded — sig_image cannot yet sign the EXE (0x800 header offset, interleaved data islands, one text range); - that second oracle is scoped-and-deferred in docs/second-oracle.md.""" + MAIN (P31 S77): covered too, IFF the byte-derived oracle sig exists. sig_image gained + multi-range signing, so the EXE's three structural blockers are gone — the 0x800 header via + `--vram-base 0x8000F800`, the interleaved data/linked islands via 28 game-code ranges derived + from the splat yaml's SEGMENT TYPES. Types are coarse structure; they are NOT splat's FUNCTION + boundaries, which is exactly what this oracle must stay free to disagree with. Entries inside + each range are still found by byte-derived jal-closure, because seeding from splat's symbols + would make every phantom look real (docs/second-oracle.md's named trap). + + Returns False until `make sig-main-oracle` has run — an honest deferral, never a fake green.""" + if binary == "main": + return os.path.exists(os.path.join(REPO, ORACLE_SIG["main"])) return binary.startswith(("ov_", "md_")) or binary == "resident" @@ -392,6 +425,32 @@ def s_ins_count(asm_path): if _INS.search(ln) and not _DATA_DIRECTIVE.search(ln)) + +def pad_tail(asm_path): + """Instruction lines AFTER `endlabel` in a splat .s, and whether they are all zero words. + + A splat slice can carry trailing pad past the function's own end — `func_80062144` ends + `jr $ra` + delay slot at 0x80062240 and the .s then emits a `nop` at 0x80062244, one line below + `endlabel`. `s_ins_count` counts it (65) while the byte-derived oracle ends the function at 64. + That is a PAD TAIL, not a mis-slice, and the two are worth telling apart: a mis-slice is a splat + defect, a pad tail is a known alignment artefact the MATCHING side already handles by emitting + the pad from C (cookbook §295; two S77 wave agents did exactly that on func_8005E13C and + func_8005D538). Reporting them in the same bucket would make the oracle's first real finding + look like a defect and bury the class that is one. (P31 S77)""" + p_ = asm_path if os.path.isabs(asm_path) else os.path.join(REPO, asm_path) + seen_end, n, allzero = False, 0, True + for ln in open(p_, errors="replace"): + if ln.strip().startswith("endlabel"): + seen_end = True + continue + if not (seen_end and _INS.search(ln) and not _DATA_DIRECTIVE.search(ln)): + continue + n += 1 + m = re.search(r"/\*\s*\S+\s+\S+\s+([0-9A-Fa-f]{8})\s*\*/", ln) + if not m or int(m.group(1), 16) != 0: + allzero = False + return n, allzero + def audit(binary): """Cross-check splat's function boundaries against the sig's independent ones. @@ -402,8 +461,18 @@ def audit(binary): so the image stays byte-identical either way. Only a second, independent oracle can see them -- which is the entire reason this function exists. See sig_is_independent(): the verdict is only meaningful where the sig genuinely is one.""" - st, sg = stubs(binary), sig(binary) - phantom, truncated = [], [] + st = stubs(binary) + sg = oracle_sig(binary) if sig_is_independent(binary) else sig(binary) + if binary == "main" and sig_is_independent(binary): + # DOMAIN (R14). The oracle signs GAME CODE only — the LINKED PsyQ blocks are excluded from + # its ranges on purpose, so auditing their stubs against it would report ~960 phantoms that + # are artefacts of comparing two oracles that never measured the same thing. That is the + # 914-vs-193 mistake this module's sig_is_independent docstring exists to prevent. + import progress as _pr + _pr.set_binary("main") + _lk = set(_pr.LINKED_SEGS) + st = {a: v for a, v in st.items() if v.asm_dir.split("/")[-1] not in _lk} + phantom, truncated, pad = [], [], [] if sig_is_independent(binary): for a, s in sorted(st.items()): row = sg.get(a) @@ -414,10 +483,14 @@ def audit(binary): if os.path.exists(p): n = s_ins_count(s.asm_path) if n != row["nins"]: - truncated.append((s, n, row["nins"])) + npad, zero = pad_tail(s.asm_path) + if zero and n - row["nins"] == npad and npad > 0: + pad.append((s, n, row["nins"], npad)) + else: + truncated.append((s, n, row["nins"])) return {"binary": binary, "stubs": len(st), "matched": len(matched(binary)), "independent": sig_is_independent(binary), - "phantom": phantom, "truncated": truncated} + "phantom": phantom, "truncated": truncated, "pad_tail": pad} def main(): @@ -436,7 +509,7 @@ def main(): import dup_report bins = sorted(dup_report.BINARIES) if (not args or args[0] == "--all") else args - tot_p = tot_t = 0 + tot_p = tot_t = tot_pad = 0 for b in bins: try: st, mt = stubs(b), matched(b) @@ -451,6 +524,7 @@ def main(): r = audit(b) tot_p += len(r["phantom"]) tot_t += len(r["truncated"]) + tot_pad += len(r["pad_tail"]) flag = "" if r["phantom"]: flag += f" PHANTOM={len(r['phantom'])}" @@ -460,7 +534,10 @@ def main(): print(line + (f" regions={dict(sorted(regions.items()))}" if len(bins) == 1 else "")) if do_audit: - print(f"\ncorpus audit: {tot_p} PHANTOM + {tot_t} TRUNCATED = {tot_p + tot_t} unmatchable slices") + print(f"\ncorpus audit: {tot_p} PHANTOM + {tot_t} TRUNCATED = {tot_p + tot_t} unmatchable slices" + + (f" (+{tot_pad} PAD-TAIL: trailing zero words past `endlabel` — an alignment " + f"artefact the matching side emits from C, cookbook §295, NOT a mis-slice)" + if tot_pad else "")) if tot_p or tot_t: print(" splat's boundaries disagree with sig_image's. sig_image is the independent oracle;\n" " a stub it does not recognise is a function NOBODY CAN EVER MATCH.") diff --git a/tools/sig_image.py b/tools/sig_image.py index c76461734..75780748c 100644 --- a/tools/sig_image.py +++ b/tools/sig_image.py @@ -222,6 +222,44 @@ def sign_image(data, vram_base, seeds_map, lo, hi, ends=None): return rows +def code_ranges_from_splat(yaml_path, vram_base, exclude=()): + """[(lo, hi)] game-code vram ranges from a splat config's SEGMENT rows. (P31 S77) + + THE INDEPENDENCE RULE THIS OBEYS. The main EXE is not one contiguous code region: it has a + 0x800 header, rodata/data islands, and linked PsyQ blocks interleaved between game code, so the + single `[--text-lo, --text-hi)` sweep this tool was built on cannot sign it (docs/second-oracle.md + names exactly these three blockers). The fix uses the splat yaml's SEGMENT rows — `[file_off, + type, name]` — and NOTHING else. Segment types are coarse structure (where code is at all); + they are NOT splat's function boundaries, which is the thing this oracle exists to disagree with. + + Seeding from splat's SYMBOLS would be the trap the design doc calls out: a phantom IS a + splat-invented address, so a symbol-seeded run makes every phantom look real and the oracle + becomes a mirror. Ranges keep the two oracles independent where it counts — inside a range, + entries are still discovered by byte-derived jal-closure and ends by `func_end`. + + `exclude` drops subsegments by name (the LINKED PsyQ blocks: real library objects, not + decompiled work, and progress.py already excludes them from the game-code denominator).""" + import re as _re + rows, txt = [], pathlib.Path(yaml_path).read_text() + for m in _re.finditer(r"^\s*-\s*\[\s*(0x[0-9A-Fa-f]+)\s*,\s*([A-Za-z_][\w]*)\s*(?:,\s*([\w.]+))?\s*\]", + txt, _re.M): + rows.append((int(m.group(1), 0), m.group(2), m.group(3) or "")) + if not rows: + raise SystemExit("sig_image: no segment rows parsed from %s" % yaml_path) + rows.sort() + out = [] + for i, (off, kind, name) in enumerate(rows): + nxt = rows[i + 1][0] if i + 1 < len(rows) else None + if kind != "c" or name in exclude or nxt is None: + continue + lo, hi = vram_base + off, vram_base + nxt + if out and out[-1][1] == lo: # merge adjacent kept ranges + out[-1] = (out[-1][0], hi) + else: + out.append((lo, hi)) + return [tuple(r) for r in out] + + def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--image", required=True, help="flat payload (0.4.dec / 1.1)") @@ -232,6 +270,12 @@ def main(): ap.add_argument("--text-lo", default=None, help="code region start vram (default: min seed)") ap.add_argument("--text-hi", default=None, help="code region end vram (default: image end)") ap.add_argument("--bootstrap", action="store_true", help="discover entries by jal-closure (no --seeds)") + ap.add_argument("--segments", default=None, + help="splat yaml: derive MULTIPLE game-code ranges from its segment rows " + "(types only, never function boundaries — see code_ranges_from_splat)") + ap.add_argument("--exclude-subsegs", default="", + help="comma-separated subseg names to skip (the LINKED PsyQ blocks)") + ap.add_argument("--ranges", default=None, help="explicit lo:hi[,lo:hi...] vram code ranges") a = ap.parse_args() data = pathlib.Path(a.image).read_bytes() @@ -256,13 +300,44 @@ def main(): hi = detect_code_end(data, vram_base, lo, hi) seeds_map = {s: "" for s in bootstrap_seeds(data, vram_base, lo, hi)} - rows = sign_image(data, vram_base, seeds_map, lo, hi, ends=seed_ends) + ranges = None + if a.ranges: + ranges = [tuple(int(x, 0) for x in r.split(":")) for r in a.ranges.split(",") if r.strip()] + elif a.segments: + ranges = code_ranges_from_splat(a.segments, vram_base, + exclude=set(x for x in a.exclude_subsegs.split(",") if x)) + if ranges: + # MULTI-RANGE: sign each island separately. A single sweep over the union would let + # bootstrap's linear partition run straight through a data or linked island and mint + # functions out of it — the phantom class this oracle exists to detect, manufactured by the + # detector. Per range: discover entries inside it, sign inside it. + rows, seen = [], set() + for rlo, rhi in ranges: + rlo, rhi = max(rlo, vram_base), min(rhi, img_end) + if rhi <= rlo: + continue + sm = dict(seeds_map) if seeds_map else {} + sm = {s: n for s, n in sm.items() if rlo <= s < rhi} + if not sm: + sm = {s: "" for s in bootstrap_seeds(data, vram_base, rlo, rhi)} + for r in sign_image(data, vram_base, sm, rlo, rhi, ends=seed_ends): + if r["addr"] in seen: + continue + seen.add(r["addr"]); rows.append(r) + print("sig_image: %d code range(s), %d functions" % (len(ranges), len(rows))) + else: + rows = sign_image(data, vram_base, seeds_map, lo, hi, ends=seed_ends) name = a.name or pathlib.Path(a.image).stem out = pathlib.Path(a.out) if a.out else pathlib.Path(".run") / f"sig.{name}.jsonl" out.parent.mkdir(parents=True, exist_ok=True) out.write_text("".join(json.dumps(r, separators=(",", ":")) + "\n" for r in rows)) - print(f"sig_image: {len(rows)} functions [{lo:#010x}..{hi:#010x}) -> {out}") + if ranges: + # the legacy [lo..hi) is meaningless in multi-range mode and printing it read as a + # one-range run over 0x6c bytes — a true number about the wrong scope. + print(f"sig_image: {len(rows)} functions across {len(ranges)} range(s) -> {out}") + else: + print(f"sig_image: {len(rows)} functions [{lo:#010x}..{hi:#010x}) -> {out}") if __name__ == "__main__":