diff --git a/Makefile b/Makefile index 5bd3329c8..07ed66b7b 100644 --- a/Makefile +++ b/Makefile @@ -393,6 +393,23 @@ sig-main: $(VENV_PY) tools/sig_image.py --image $(main_EXE) --vram-base $(main_VRAM_BASE) --seeds .run/seeds.main.txt --name main echo "sig-main: signed $$(wc -l < .run/seeds.main.txt) main stubs (splat-true lengths) -> .run/sig.main.jsonl" +# sig-main-oracle (P31 S77) — MAIN'S INDEPENDENT SECOND ORACLE (roadmap contract §1.3). +# Distinct from `sig-main` above, which is splat-SEEDED on purpose. This one signs the ORIGINAL EXE +# bytes with NO splat symbols: `--vram-base 0x8000F800` puts file offset 0 at vram (so the 0x800 +# PS-X EXE header simply falls below the first range), and `--segments` derives the game-code ranges +# from the splat yaml's SEGMENT TYPES — coarse structure, never splat's FUNCTION boundaries, which +# is the thing the oracle must stay free to disagree with. Entries inside each range are found by +# byte-derived jal-closure, because seeding from splat's symbols would make every phantom look real +# (docs/second-oracle.md names that trap). LINKED PsyQ blocks are excluded: real library objects, +# outside the game-code denominator, and auditing them here would report ~960 phantoms that are +# artefacts of comparing two oracles that never measured the same thing. +sig-main-oracle: + @$(VENV_PY) -c "import sys;sys.path.insert(0,'tools');import progress;progress.set_binary('main');print(','.join(sorted(progress.LINKED_SEGS)))" > .run/main_linked_segs.txt + $(VENV_PY) tools/sig_image.py --image $(main_EXE) --vram-base 0x8000F800 \ + --segments config/splat.us.exe.yaml --exclude-subsegs "$$(cat .run/main_linked_segs.txt)" \ + --bootstrap --out .run/sig.main.oracle.jsonl + @$(VENV_PY) -c "import sys;sys.path.insert(0,'tools');import corpus;r=corpus.audit('main');print('sig-main-oracle: main is now INDEPENDENT — %d in-domain stubs, %d PHANTOM, %d TRUNCATED, %d PAD-TAIL'%(r['stubs'],len(r['phantom']),len(r['truncated']),len(r['pad_tail'])))" + # sig-modules (P30 S44): sign every module-class binary at ITS OWN vram (from modules.mk) with its # own TEXT_LO (the §154 module-id-word law: code starts past the header; bootstrap from offset 0 # yields 0 functions). Same derived-jobs shape as sig-overlays (R33). Empty MODULE_BINARIES = no-op. diff --git a/docs/second-oracle.md b/docs/second-oracle.md index 119577275..5c6cde544 100644 --- a/docs/second-oracle.md +++ b/docs/second-oracle.md @@ -12,7 +12,7 @@ |---|---|---|---| | 138 overlays | `sig_image` (byte-derived) | ✅ | validated 58,524/58,621 vs spimdisasm | | resident | `sig_image` (**Phase-27 T10**) | ✅ | `make sig-resident` — 144 fns, all 21 stubs present, 0 phantom/truncated | -| main | — | ❌ **deferred** | `sig_image` cannot yet sign the EXE (below) | +| main | `sig_image` multi-range (**P31 S77**) | ✅ | `make sig-main-oracle` — 28 game-code ranges, 986 fns, **0 phantom / 0 truncated / 1 pad-tail** | `corpus.sig_is_independent()` gates the check to exactly this domain: a boundary cross-check applied where the two oracles were never measuring the same thing is noise, not thoroughness (the audit's own @@ -25,7 +25,42 @@ sig — run `make sig-resident` before `make audit-corpus` (a stale Ghidra sig t "measuring Ghidra's limits" artefact). `h_exact` is raw-byte SHA1, so `weighted_metrics` is unaffected by the swap (fleet % unchanged to the decimal). -## Main EXE — why sig_image can't sign it yet (the deferral) +## Main EXE — DELIVERED (P31 S77). The deferral below is kept as the design record. + +`make sig-main-oracle` signs the ORIGINAL EXE bytes and `corpus.sig_is_independent("main")` is now +True whenever `.run/sig.main.oracle.jsonl` exists. All three structural blockers are closed: + +1. **The 0x800 header** — `--vram-base 0x8000F800` maps file offset 0 to vram, so the header simply + falls below the first code range. No `--skip` needed. +2. **Interleaved data + linked islands** — new `--segments ` + (`sig_image.code_ranges_from_splat`) derives **28 game-code ranges** from the yaml's SEGMENT rows. + It reads `[file_off, type, name]` and NOTHING else: segment *types* are coarse structure, not + splat's *function* boundaries, so the two oracles stay independent exactly where it matters. + `--exclude-subsegs` drops the LINKED PsyQ blocks. +3. **One text range** — the signer now loops ranges, bootstrapping entries INSIDE each. Per-range + discovery is what stops the linear partition running through a data island and minting functions + out of it, i.e. the detector manufacturing the phantom class it exists to detect. + +**The trap was avoided, not worked around.** Entries are still found by byte-derived jal-closure; +nothing is seeded from splat's symbols. `.run/sig.main.jsonl` (the splat-SEEDED atlas sig, P31 T3) +is a DIFFERENT file and the audit never reads it — `corpus.ORACLE_SIG` keeps them apart. + +**Domain (R14).** The oracle signs game code only, so `audit("main")` filters the LINKED stubs out. +Auditing them against it would report ~960 phantoms that are artefacts of comparing two oracles that +never measured the same thing — the same 914-vs-193 mistake this file already warns about. + +**First finding, and it is a real one.** `func_80062144`: splat's `.s` says 65 instructions, the +byte-derived boundary says 64. The extra line is a `nop` at `0x80062244`, emitted one line BELOW +`endlabel`. That is a **PAD-TAIL**, now its own audit class — a known alignment artefact the matching +side handles by emitting the pad from C (cookbook §295; two S77 wave agents did exactly that on +`func_8005E13C` and `func_8005D538`), not a splat mis-slice. Reporting it as TRUNCATED would have +made the oracle's first real finding look like a defect and buried the class that is one. + +**Fleet status after wiring:** `make audit-corpus` → **0 PHANTOM + 0 TRUNCATED**, +1 PAD-TAIL. + +--- + +## (Historical) Main EXE — why sig_image could not sign it `tools/sig_image.py` assumes a flat blob whose file offset 0 IS its vram base, one contiguous code region, and one `[lo,hi)` text range. The main EXE breaks all three — structurally, not with a flag: diff --git a/tools/corpus.py b/tools/corpus.py index 55c0136d3..e51e73068 100644 --- a/tools/corpus.py +++ b/tools/corpus.py @@ -322,6 +322,29 @@ def is_o0(src_path): # -------------------------------------------------------------------------------------------- # the invariant # -------------------------------------------------------------------------------------------- +# The INDEPENDENT boundary oracle, where that is a DIFFERENT file from the working sig (P31 S77). +# For main the working sig is deliberately splat-SEEDED (the atlas needs the boundaries a match must +# hit, P31 T3) — auditing against it would be a mirror, not an oracle. +ORACLE_SIG = {"main": ".run/sig.main.oracle.jsonl"} + + +def oracle_sig(binary): + """addr -> sig row from the INDEPENDENT signer (falls back to sig() where they are one file).""" + rel = ORACLE_SIG.get(binary) + if not rel: + return sig(binary) + full = os.path.join(REPO, rel) + if not os.path.exists(full): + raise CorpusError(f"{binary}: no independent sig at {rel} — run `make sig-main-oracle`") + out = {} + for line in open(full): + line = line.strip() + if line: + r = json.loads(line) + out[int(r["addr"], 16)] = r + return out + + @functools.lru_cache(maxsize=None) def sig(binary): """addr -> sig row (.run/sig..jsonl). Signs the ORIGINAL bytes: immutable w.r.t. src/.""" @@ -378,7 +401,17 @@ def sig_is_independent(binary): valid IFF .run/sig.resident.jsonl is the sig_image sig; run `make sig-resident` first (a stale Ghidra sig there would resurrect the 'measuring Ghidra's limits' artefact). MAIN stays excluded — sig_image cannot yet sign the EXE (0x800 header offset, interleaved data islands, one text range); - that second oracle is scoped-and-deferred in docs/second-oracle.md.""" + MAIN (P31 S77): covered too, IFF the byte-derived oracle sig exists. sig_image gained + multi-range signing, so the EXE's three structural blockers are gone — the 0x800 header via + `--vram-base 0x8000F800`, the interleaved data/linked islands via 28 game-code ranges derived + from the splat yaml's SEGMENT TYPES. Types are coarse structure; they are NOT splat's FUNCTION + boundaries, which is exactly what this oracle must stay free to disagree with. Entries inside + each range are still found by byte-derived jal-closure, because seeding from splat's symbols + would make every phantom look real (docs/second-oracle.md's named trap). + + Returns False until `make sig-main-oracle` has run — an honest deferral, never a fake green.""" + if binary == "main": + return os.path.exists(os.path.join(REPO, ORACLE_SIG["main"])) return binary.startswith(("ov_", "md_")) or binary == "resident" @@ -392,6 +425,32 @@ def s_ins_count(asm_path): if _INS.search(ln) and not _DATA_DIRECTIVE.search(ln)) + +def pad_tail(asm_path): + """Instruction lines AFTER `endlabel` in a splat .s, and whether they are all zero words. + + A splat slice can carry trailing pad past the function's own end — `func_80062144` ends + `jr $ra` + delay slot at 0x80062240 and the .s then emits a `nop` at 0x80062244, one line below + `endlabel`. `s_ins_count` counts it (65) while the byte-derived oracle ends the function at 64. + That is a PAD TAIL, not a mis-slice, and the two are worth telling apart: a mis-slice is a splat + defect, a pad tail is a known alignment artefact the MATCHING side already handles by emitting + the pad from C (cookbook §295; two S77 wave agents did exactly that on func_8005E13C and + func_8005D538). Reporting them in the same bucket would make the oracle's first real finding + look like a defect and bury the class that is one. (P31 S77)""" + p_ = asm_path if os.path.isabs(asm_path) else os.path.join(REPO, asm_path) + seen_end, n, allzero = False, 0, True + for ln in open(p_, errors="replace"): + if ln.strip().startswith("endlabel"): + seen_end = True + continue + if not (seen_end and _INS.search(ln) and not _DATA_DIRECTIVE.search(ln)): + continue + n += 1 + m = re.search(r"/\*\s*\S+\s+\S+\s+([0-9A-Fa-f]{8})\s*\*/", ln) + if not m or int(m.group(1), 16) != 0: + allzero = False + return n, allzero + def audit(binary): """Cross-check splat's function boundaries against the sig's independent ones. @@ -402,8 +461,18 @@ def audit(binary): so the image stays byte-identical either way. Only a second, independent oracle can see them -- which is the entire reason this function exists. See sig_is_independent(): the verdict is only meaningful where the sig genuinely is one.""" - st, sg = stubs(binary), sig(binary) - phantom, truncated = [], [] + st = stubs(binary) + sg = oracle_sig(binary) if sig_is_independent(binary) else sig(binary) + if binary == "main" and sig_is_independent(binary): + # DOMAIN (R14). The oracle signs GAME CODE only — the LINKED PsyQ blocks are excluded from + # its ranges on purpose, so auditing their stubs against it would report ~960 phantoms that + # are artefacts of comparing two oracles that never measured the same thing. That is the + # 914-vs-193 mistake this module's sig_is_independent docstring exists to prevent. + import progress as _pr + _pr.set_binary("main") + _lk = set(_pr.LINKED_SEGS) + st = {a: v for a, v in st.items() if v.asm_dir.split("/")[-1] not in _lk} + phantom, truncated, pad = [], [], [] if sig_is_independent(binary): for a, s in sorted(st.items()): row = sg.get(a) @@ -414,10 +483,14 @@ def audit(binary): if os.path.exists(p): n = s_ins_count(s.asm_path) if n != row["nins"]: - truncated.append((s, n, row["nins"])) + npad, zero = pad_tail(s.asm_path) + if zero and n - row["nins"] == npad and npad > 0: + pad.append((s, n, row["nins"], npad)) + else: + truncated.append((s, n, row["nins"])) return {"binary": binary, "stubs": len(st), "matched": len(matched(binary)), "independent": sig_is_independent(binary), - "phantom": phantom, "truncated": truncated} + "phantom": phantom, "truncated": truncated, "pad_tail": pad} def main(): @@ -436,7 +509,7 @@ def main(): import dup_report bins = sorted(dup_report.BINARIES) if (not args or args[0] == "--all") else args - tot_p = tot_t = 0 + tot_p = tot_t = tot_pad = 0 for b in bins: try: st, mt = stubs(b), matched(b) @@ -451,6 +524,7 @@ def main(): r = audit(b) tot_p += len(r["phantom"]) tot_t += len(r["truncated"]) + tot_pad += len(r["pad_tail"]) flag = "" if r["phantom"]: flag += f" PHANTOM={len(r['phantom'])}" @@ -460,7 +534,10 @@ def main(): print(line + (f" regions={dict(sorted(regions.items()))}" if len(bins) == 1 else "")) if do_audit: - print(f"\ncorpus audit: {tot_p} PHANTOM + {tot_t} TRUNCATED = {tot_p + tot_t} unmatchable slices") + print(f"\ncorpus audit: {tot_p} PHANTOM + {tot_t} TRUNCATED = {tot_p + tot_t} unmatchable slices" + + (f" (+{tot_pad} PAD-TAIL: trailing zero words past `endlabel` — an alignment " + f"artefact the matching side emits from C, cookbook §295, NOT a mis-slice)" + if tot_pad else "")) if tot_p or tot_t: print(" splat's boundaries disagree with sig_image's. sig_image is the independent oracle;\n" " a stub it does not recognise is a function NOBODY CAN EVER MATCH.") diff --git a/tools/sig_image.py b/tools/sig_image.py index c76461734..75780748c 100644 --- a/tools/sig_image.py +++ b/tools/sig_image.py @@ -222,6 +222,44 @@ def sign_image(data, vram_base, seeds_map, lo, hi, ends=None): return rows +def code_ranges_from_splat(yaml_path, vram_base, exclude=()): + """[(lo, hi)] game-code vram ranges from a splat config's SEGMENT rows. (P31 S77) + + THE INDEPENDENCE RULE THIS OBEYS. The main EXE is not one contiguous code region: it has a + 0x800 header, rodata/data islands, and linked PsyQ blocks interleaved between game code, so the + single `[--text-lo, --text-hi)` sweep this tool was built on cannot sign it (docs/second-oracle.md + names exactly these three blockers). The fix uses the splat yaml's SEGMENT rows — `[file_off, + type, name]` — and NOTHING else. Segment types are coarse structure (where code is at all); + they are NOT splat's function boundaries, which is the thing this oracle exists to disagree with. + + Seeding from splat's SYMBOLS would be the trap the design doc calls out: a phantom IS a + splat-invented address, so a symbol-seeded run makes every phantom look real and the oracle + becomes a mirror. Ranges keep the two oracles independent where it counts — inside a range, + entries are still discovered by byte-derived jal-closure and ends by `func_end`. + + `exclude` drops subsegments by name (the LINKED PsyQ blocks: real library objects, not + decompiled work, and progress.py already excludes them from the game-code denominator).""" + import re as _re + rows, txt = [], pathlib.Path(yaml_path).read_text() + for m in _re.finditer(r"^\s*-\s*\[\s*(0x[0-9A-Fa-f]+)\s*,\s*([A-Za-z_][\w]*)\s*(?:,\s*([\w.]+))?\s*\]", + txt, _re.M): + rows.append((int(m.group(1), 0), m.group(2), m.group(3) or "")) + if not rows: + raise SystemExit("sig_image: no segment rows parsed from %s" % yaml_path) + rows.sort() + out = [] + for i, (off, kind, name) in enumerate(rows): + nxt = rows[i + 1][0] if i + 1 < len(rows) else None + if kind != "c" or name in exclude or nxt is None: + continue + lo, hi = vram_base + off, vram_base + nxt + if out and out[-1][1] == lo: # merge adjacent kept ranges + out[-1] = (out[-1][0], hi) + else: + out.append((lo, hi)) + return [tuple(r) for r in out] + + def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--image", required=True, help="flat payload (0.4.dec / 1.1)") @@ -232,6 +270,12 @@ def main(): ap.add_argument("--text-lo", default=None, help="code region start vram (default: min seed)") ap.add_argument("--text-hi", default=None, help="code region end vram (default: image end)") ap.add_argument("--bootstrap", action="store_true", help="discover entries by jal-closure (no --seeds)") + ap.add_argument("--segments", default=None, + help="splat yaml: derive MULTIPLE game-code ranges from its segment rows " + "(types only, never function boundaries — see code_ranges_from_splat)") + ap.add_argument("--exclude-subsegs", default="", + help="comma-separated subseg names to skip (the LINKED PsyQ blocks)") + ap.add_argument("--ranges", default=None, help="explicit lo:hi[,lo:hi...] vram code ranges") a = ap.parse_args() data = pathlib.Path(a.image).read_bytes() @@ -256,13 +300,44 @@ def main(): hi = detect_code_end(data, vram_base, lo, hi) seeds_map = {s: "" for s in bootstrap_seeds(data, vram_base, lo, hi)} - rows = sign_image(data, vram_base, seeds_map, lo, hi, ends=seed_ends) + ranges = None + if a.ranges: + ranges = [tuple(int(x, 0) for x in r.split(":")) for r in a.ranges.split(",") if r.strip()] + elif a.segments: + ranges = code_ranges_from_splat(a.segments, vram_base, + exclude=set(x for x in a.exclude_subsegs.split(",") if x)) + if ranges: + # MULTI-RANGE: sign each island separately. A single sweep over the union would let + # bootstrap's linear partition run straight through a data or linked island and mint + # functions out of it — the phantom class this oracle exists to detect, manufactured by the + # detector. Per range: discover entries inside it, sign inside it. + rows, seen = [], set() + for rlo, rhi in ranges: + rlo, rhi = max(rlo, vram_base), min(rhi, img_end) + if rhi <= rlo: + continue + sm = dict(seeds_map) if seeds_map else {} + sm = {s: n for s, n in sm.items() if rlo <= s < rhi} + if not sm: + sm = {s: "" for s in bootstrap_seeds(data, vram_base, rlo, rhi)} + for r in sign_image(data, vram_base, sm, rlo, rhi, ends=seed_ends): + if r["addr"] in seen: + continue + seen.add(r["addr"]); rows.append(r) + print("sig_image: %d code range(s), %d functions" % (len(ranges), len(rows))) + else: + rows = sign_image(data, vram_base, seeds_map, lo, hi, ends=seed_ends) name = a.name or pathlib.Path(a.image).stem out = pathlib.Path(a.out) if a.out else pathlib.Path(".run") / f"sig.{name}.jsonl" out.parent.mkdir(parents=True, exist_ok=True) out.write_text("".join(json.dumps(r, separators=(",", ":")) + "\n" for r in rows)) - print(f"sig_image: {len(rows)} functions [{lo:#010x}..{hi:#010x}) -> {out}") + if ranges: + # the legacy [lo..hi) is meaningless in multi-range mode and printing it read as a + # one-range run over 0x6c bytes — a true number about the wrong scope. + print(f"sig_image: {len(rows)} functions across {len(ranges)} range(s) -> {out}") + else: + print(f"sig_image: {len(rows)} functions [{lo:#010x}..{hi:#010x}) -> {out}") if __name__ == "__main__":