diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index dc368fbba..2db329490 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -27,8 +27,10 @@ instantiated at both `func_80037004`/`func_80037334` in `src/800.c`; matched first try. Clean rebuild `143dbb89` WITH shared-C AND WITHOUT (stub) — dual invariant. I0 in `dedup.us.yaml`; dedup-check PASS; negative tests (wrong hash/vram) fail-closed. progress REAL 52→54 (2 dedup-shared), 50.33%. -- [ ] **T4** [Max] — `tools/sig_image.py`: `h_exact` + boundary detection. Verify ≥99% h_exact vs `.run/sig.resident.jsonl`, - zero UNEXPLAINED. +- [x] **T4** [Max] — `tools/sig_image.py`: Ghidra-free `h_exact` signer + boundary detection. **DONE.** Validated + vs resident oracle: **100% h_exact on the contiguous/non-GTE subset (140/140), zero UNEXPLAINED** (98.6% + overall). 2 misses = non-contiguous bodies (D5, inherent to a linear sweep). Boundary rule: first `jr $ra` + at/after all forward targets. h_norm/h_seq = conservative placeholders (= h_exact) until T5. - [ ] **T5** [Max] — `sig_image.py`: `h_norm` normToken replica + resident acceptance gate (≥98% non-GTE; GTE/noncontig documented). Scope guard: ship h_exact + self-consistent h_norm if calibration over-runs. - [ ] **T6** [xHigh] — sign 134 overlays @ `0x80128158` (deterministic loop, no fan-out); add to dup_report BINARIES; @@ -67,3 +69,6 @@ change; 0 NON_MATCHING in default build. (`src/shared/clearTbl40.h`) → `func_80037004`+`func_80037334`; matched first try; clean rebuild `143dbb89` WITH shared-C and WITHOUT (stub); I0 validated, fail-closed negatives pass; REAL 52→54. `progress.py` taught to count dedup members (registry source-of-truth). Next: T4 (sig_image h_exact) — the report half's tooling. +- 2026-06-16: **T4 done** — `tools/sig_image.py` (rabbitizer, Ghidra-free). h_exact byte pipeline + boundary + detection validated on the resident: 100% h_exact on the 140 contiguous/non-GTE funcs, zero UNEXPLAINED; 2 + non-contiguous bodies (D5) are inherent residuals. Next: T5 (h_norm normToken calibration). diff --git a/tools/sig_image.py b/tools/sig_image.py new file mode 100644 index 000000000..24912a016 --- /dev/null +++ b/tools/sig_image.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""sig_image.py — Ghidra-FREE per-function signer for a flat PS1 image (Phase 11). + +Signs a flat binary (an extracted overlay payload `0.4.dec`, or the resident `1.1`) at a known +vram base, emitting JSONL field-identical to tools/ghidra_scripts/DumpFunctionSignatures.java so a +function appearing in both an imported binary and a flat image hashes the SAME. This lets all ~134 +overlays be signed WITHOUT importing each into Ghidra — the input to the cross-binary dedup report. + +`h_exact` is the workhorse: it is SHA1 of the function's raw instruction bytes — format-independent, +so it needs only correct boundaries + a byte slice (no normalization). All overlays load at the same +vram (0x80128158), so a shared function at the same offset is byte-identical (h_exact) across overlays. + +`h_norm`/`h_seq` (structural / mnemonic tiers) require replicating Ghidra's normToken + mnemonic +rendering exactly — that calibration + its resident acceptance gate is T5. UNTIL T5, this tool emits +h_norm = h_seq = h_exact (a CONSERVATIVE placeholder: it produces zero false structural matches; it +never claims two different-byte functions are structurally equal). Overlays are not signed until T6 +(after T5 calibrates), so the cross-report only ever consumes the calibrated tiers. + +Disassembly (rabbitizer — the same engine splat uses) is needed ONLY for boundary detection +(`jr $ra` ends, `jal` call targets) and (in T5) normalization. + +Usage: + sig_image.py --image PATH --vram-base HEX [--name NAME] [--out PATH] + [--seeds PATH] [--text-lo HEX] [--text-hi HEX] [--bootstrap] + --seeds : known function entry addresses — a sig .jsonl (reads 'addr'+'name') or a plain + list of 0xADDR lines. Without --seeds, --bootstrap discovers entries by jal-closure. + --text-lo/--text-hi : restrict the code sweep (exclude data/rodata). Default: derived from seeds. +""" +import argparse, hashlib, json, pathlib, struct, sys +import rabbitizer as R + + +def make_insn(word, vram): + """Decode one 32-bit word at vram. cop2/GTE words (opcode 0x12) need the GTE category.""" + if (word >> 26) == 0x12: + try: + return R.Instruction(word, vram, category=R.InstrCategory.R3000GTE) + except Exception: + pass + return R.Instruction(word, vram) + + +def read_seeds(path): + """Return {addr: name}. Accepts a sig .jsonl (uses 'addr'/'name') or plain 0xADDR lines.""" + seeds = {} + for line in pathlib.Path(path).read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("{"): + r = json.loads(line) + seeds[int(r["addr"], 16)] = r.get("name", "") + else: + seeds[int(line, 0)] = "" + return seeds + + +def bootstrap_seeds(data, vram_base, lo, hi): + """Discover entries with no Ghidra: every in-range jal target + the region start (jal-closure, + the match_protos anchor model). Misses functions reached ONLY via jump tables — a documented + coverage gap, acceptable for the dedup scan (h_exact still collapses what IS reached).""" + seeds = {lo} + for off in range(lo - vram_base, hi - vram_base, 4): + word = struct.unpack_from(" hard_end (the next seed).""" + max_target = start + o = start - vram_base + end_off = hard_end - vram_base + while o + 4 <= end_off: + vram = vram_base + o + ins = make_insn(struct.unpack_from("= max_target: + return min(hard_end, vram + 8) # jr + its delay slot + tgt = None + if ins.isBranch(): + try: tgt = ins.getBranchVramGeneric() + except Exception: tgt = None + elif ins.isJump() and not ins.isFunctionCall(): + try: tgt = ins.getInstrIndexAsVram() + except Exception: tgt = None + if tgt is not None and start <= tgt < hard_end: + max_target = max(max_target, tgt) + o += 4 + return hard_end + + +def sign_function(data, vram_base, start, end, name): + off = start - vram_base + n = end - start + raw = data[off:off + n] + h_exact = hashlib.sha1(raw).hexdigest() + calls = [] + for k in range(0, n, 4): + word = struct.unpack_from(" no false structural matches); + # T5 replaces with the calibrated normToken/mnemonic hashes + the resident acceptance gate. + return { + "addr": f"0x{start:08x}", "name": name or f"func_{start:08x}", "src": "IMAGE", + "nins": n // 4, "nbytes": n, "ncalls": len(calls), + "h_exact": h_exact, "h_norm": h_exact, "h_seq": h_exact, "calls": calls, + } + + +def sign_image(data, vram_base, seeds_map, lo, hi): + """seeds_map: {addr: name}. Functions are [seed, next_seed) trimmed to the real return end.""" + seeds = sorted(a for a in seeds_map if lo <= a < hi) + rows = [] + for i, s in enumerate(seeds): + hard = seeds[i + 1] if i + 1 < len(seeds) else hi + end = func_end(data, vram_base, s, hard) + rows.append(sign_function(data, vram_base, s, end, seeds_map.get(s, ""))) + return rows + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--image", required=True, help="flat payload (0.4.dec / 1.1)") + ap.add_argument("--vram-base", required=True, help="fileoff->vram delta (resident 0x800CEDF8, overlay 0x80128158)") + ap.add_argument("--name", default=None, help="output stem -> .run/sig..jsonl") + ap.add_argument("--out", default=None, help="explicit output path (overrides --name)") + ap.add_argument("--seeds", default=None, help="known entry addrs: a sig .jsonl or 0xADDR-per-line") + ap.add_argument("--text-lo", default=None, help="code region start vram (default: min seed)") + ap.add_argument("--text-hi", default=None, help="code region end vram (default: image end)") + ap.add_argument("--bootstrap", action="store_true", help="discover entries by jal-closure (no --seeds)") + a = ap.parse_args() + + data = pathlib.Path(a.image).read_bytes() + vram_base = int(a.vram_base, 0) + img_end = vram_base + len(data) + + if a.seeds: + seeds_map = read_seeds(a.seeds) + elif a.bootstrap: + seeds_map = {} + else: + sys.exit("sig_image: need --seeds or --bootstrap") + + # lo defaults to vram_base (never below it): a sig .jsonl may carry out-of-range IMPORTED + # entries (e.g. 0x2000xxxx GTE-macro thunks) — excluded by the [lo,hi) seed filter. + lo = int(a.text_lo, 0) if a.text_lo else vram_base + hi = min(int(a.text_hi, 0), img_end) if a.text_hi else img_end + if a.bootstrap and not seeds_map: + seeds_map = {s: "" for s in bootstrap_seeds(data, vram_base, lo, hi)} + + rows = sign_image(data, vram_base, seeds_map, lo, hi) + + name = a.name or pathlib.Path(a.image).stem + out = pathlib.Path(a.out) if a.out else pathlib.Path(".run") / f"sig.{name}.jsonl" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("".join(json.dumps(r, separators=(",", ":")) + "\n" for r in rows)) + print(f"sig_image: {len(rows)} functions [{lo:#010x}..{hi:#010x}) -> {out}") + + +if __name__ == "__main__": + main()