From 9092d15dc81dbd0832cebac2bf6b72a1dac25f71 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Fri, 17 Jul 2026 21:51:25 -0700 Subject: [PATCH 01/37] aot_overlay_spike: export-table dispatch seeds + coverage recall report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extract_generic: supplement prologue seeds with the overlay's OWN export/dispatch table (the header pointers) as dispatch entries — the game's authoritative entry-point manifest, on the disc, play-free. compile_overlays classifies mid-fn targets DISPATCH_INTERIOR aliases. Honest note: no-op for Tomba 2 (skipped=23) — the recompiler's control-flow walk already reaches those entries; kept because it helps engines where the walk cannot. coverage_report.py (new): measures play-free static-extraction RECALL against a played vault (most complete needed-set) and/or a live-session captures json, and persists a durable gap manifest (misses grouped by region). Tomba 2: 60.0% entry-level recall vs vault (1113/1856), 743 misses dominated by MAIN.EXE frameless/indirect functions + the kernel region. Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/aot_overlay_spike/coverage_report.py | 141 +++++++++++++++++++++ tools/aot_overlay_spike/extract_generic.py | 24 +++- 2 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 tools/aot_overlay_spike/coverage_report.py diff --git a/tools/aot_overlay_spike/coverage_report.py b/tools/aot_overlay_spike/coverage_report.py new file mode 100644 index 00000000..04ce8a02 --- /dev/null +++ b/tools/aot_overlay_spike/coverage_report.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""AOT static-coverage recall report + gap manifest. + +Answers: "of every overlay function the game ACTUALLY runs, what fraction did our +play-free static extractor discover?" — and persists the ones it MISSED so we +have a durable reference and a scoreboard for the discovery tooling. + +Two ground-truth "needed" sources: + * a played/coverage VAULT cache dir (.ranges from a full playthrough) — the most + complete needed-set we have; the primary recall benchmark. + * a runtime CAPTURES json (executed_pcs from a live session) — what a specific + drive exercised; a live spot-check. + +"Covered" = the entries our static shard cache (.ranges) defines. Recall is +reported at two strengths: + * entry-level : same function ENTRY address discovered (did we find it?) + * entry+code_crc : same entry AND identical bytes (is our shard the real fn?) + +Usage: + coverage_report.py --static --vault + [--captures ] + --out-md --out-json [--game ] +Cache dirs are scanned recursively for *.ranges; a captures json is the v2 schema. +""" +import argparse, os, sys, json, re, glob +from collections import defaultdict + +MASK = 0x1FFFFFFF + +def norm(a): return a & MASK + +def region_of(a): + """Page-region label for grouping (aligns to the overlay window bases).""" + return norm(a) & 0xFFFFF000 + +def parse_ranges_dir(d): + """-> dict entry_masked -> code_crc (last wins); also the raw (entry,crc) set.""" + ent = {} + for rf in glob.glob(os.path.join(d, '**', '*.ranges'), recursive=True): + for line in open(rf, errors='ignore'): + m = re.match(r'F\s+([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+)', line) + if m: + ent[norm(int(m.group(1), 16))] = int(m.group(2), 16) + return ent + +def parse_captures_executed(path): + """-> set of FUNCTION-ENTRY addrs (masked) the runtime dispatched to. + Use dispatch/function entries only — executed_pcs is per-basic-block PCs + (many per function), which is not comparable to the vault's function-entry + granularity and would pollute the recall denominator.""" + ent = set() + for cap in json.load(open(path)): + for key in ('dispatch_entry_pcs', 'function_entry_pcs'): + for a in cap.get(key, []): + ent.add(norm(int(a, 16))) + return ent + +def recall(needed_entries, static_ent, needed_crc=None): + """needed_entries: set; static_ent: {entry->crc}. Returns dict of metrics.""" + covered = set(static_ent) + hit = needed_entries & covered + miss = needed_entries - covered + out = {'needed': len(needed_entries), 'covered_here': len(hit), + 'missed': len(miss), 'recall_entry': (len(hit)/len(needed_entries) if needed_entries else 0.0), + 'misses': sorted(miss)} + if needed_crc is not None: + crc_hit = sum(1 for e in hit if static_ent.get(e) == needed_crc.get(e)) + out['covered_entry_crc'] = crc_hit + out['recall_entry_crc'] = (crc_hit/len(needed_entries) if needed_entries else 0.0) + return out + +def group_by_region(addrs): + g = defaultdict(list) + for a in addrs: g[region_of(a)].append(a) + return {r: sorted(v) for r, v in sorted(g.items())} + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--static', required=True, help='static shard cache dir (.ranges)') + ap.add_argument('--vault', help='played/coverage vault cache dir (.ranges)') + ap.add_argument('--captures', help='runtime overlay_captures.json (executed_pcs)') + ap.add_argument('--game', default='UNKNOWN') + ap.add_argument('--out-md', required=True) + ap.add_argument('--out-json', required=True) + a = ap.parse_args() + + static_ent = parse_ranges_dir(a.static) + report = {'game': a.game, 'static_entries': len(static_ent), 'sources': {}} + + md = [] + md.append(f'# AOT static-coverage recall — {a.game}\n') + md.append('_How much of what the game actually runs did the play-free static ' + 'extractor discover? Misses are the gap the runtime autocompile ' + '(or better discovery) must still cover._\n') + md.append(f'- Static shard cache: `{a.static}`') + md.append(f'- Static-covered overlay functions: **{len(static_ent)}**\n') + + if a.vault: + vault_ent = parse_ranges_dir(a.vault) + r = recall(set(vault_ent), static_ent, vault_ent) + report['sources']['vault'] = {k: v for k, v in r.items() if k != 'misses'} + report['vault_misses'] = [f'0x{x|0x80000000:08X}' for x in r['misses']] + md.append('## vs played vault (most complete needed-set)\n') + md.append(f'- Functions the full playthrough ran: **{r["needed"]}**') + md.append(f'- Discovered by static: **{r["covered_here"]}** ' + f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') + md.append(f'- Byte-identical (entry+code_crc): **{r.get("covered_entry_crc",0)}** ' + f'(**{r.get("recall_entry_crc",0)*100:.1f}%**) ' + f'_(cg-version differences lower this vs entry-level)_') + md.append(f'- **MISSED: {r["missed"]}** functions the game needs but static did not find\n') + md.append('### Misses grouped by overlay region\n') + for reg, addrs in group_by_region(r['misses']).items(): + md.append(f'- region `0x{reg|0x80000000:08X}`: {len(addrs)} misses') + md.append(' ```') + for i in range(0, len(addrs), 8): + md.append(' ' + ' '.join(f'{x|0x80000000:08X}' for x in addrs[i:i+8])) + md.append(' ```') + md.append('') + + if a.captures and os.path.exists(a.captures): + live = parse_captures_executed(a.captures) + r = recall(live, static_ent) + report['sources']['live_session'] = {k: v for k, v in r.items() if k != 'misses'} + report['live_misses'] = [f'0x{x|0x80000000:08X}' for x in r['misses']] + md.append('## vs live session captures (this drive)\n') + md.append(f'- Functions this session exercised: **{r["needed"]}**') + md.append(f'- Discovered by static: **{r["covered_here"]}** ' + f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') + md.append(f'- **MISSED live: {r["missed"]}**\n') + + os.makedirs(os.path.dirname(a.out_md), exist_ok=True) + open(a.out_md, 'w', encoding='utf-8').write('\n'.join(md)) + json.dump(report, open(a.out_json, 'w'), indent=1) + print(f'wrote {a.out_md}') + print(f'wrote {a.out_json}') + for src, m in report['sources'].items(): + print(f' {src}: recall_entry={m.get("recall_entry",0)*100:.1f}% ' + f'({m.get("covered_here")}/{m.get("needed")}), missed={m.get("missed")}') + +if __name__ == '__main__': + main() diff --git a/tools/aot_overlay_spike/extract_generic.py b/tools/aot_overlay_spike/extract_generic.py index 6c6f018b..e88e503b 100644 --- a/tools/aot_overlay_spike/extract_generic.py +++ b/tools/aot_overlay_spike/extract_generic.py @@ -139,10 +139,17 @@ def full_discovery_seeds(data, recompiler, tmp): a=int(m.group(1),16); seeds.add(a if a>=0x80000000 else (0x80000000|a)) return sorted(seeds) or None -def rec(load_addr, data, seeds): +def rec(load_addr, data, seeds, dispatch_extra=None): + # function_entry_pcs = prologue-scanned function starts (walk roots). + # dispatch_entry_pcs = those PLUS any extra dispatch targets (e.g. the + # overlay's own export table). compile_overlays promotes clean starts to + # entries and mid-function dispatch targets to DISPATCH_INTERIOR aliases + # (impossible_entry_start guards garbage), so passing the game's authoritative + # dispatch table here recovers indirect-only entries a prologue scan misses. + disp = sorted(set(seeds) | set(dispatch_extra or ())) return {"schema":"psxrecomp overlay capture v2","load_addr":f"0x{load_addr:08X}", "size":len(data),"bytes_b64":base64.b64encode(data).decode(), - "executed_pcs":[],"dispatch_entry_pcs":[f"0x{a:08X}" for a in seeds], + "executed_pcs":[],"dispatch_entry_pcs":[f"0x{a:08X}" for a in disp], "function_entry_pcs":[f"0x{a:08X}" for a in seeds], "seeds":[f"0x{a:08X}" for a in seeds]} @@ -212,6 +219,13 @@ def main(): continue base,score=rb seeds=prologues(data,base) # seed via full-file prologue scan + # SUPPLEMENT the prologue seeds with the overlay's OWN export/dispatch + # table (the header pointers) — the game's authoritative list of live + # entry points, sitting on the disc (play-free). These are exactly the + # indirect-dispatch-only / interior entries a prologue scan can't see; + # compile_overlays emits them as DISPATCH_INTERIOR aliases as needed. + hdr_entries=[p for p in ptrs + if p and (p&3)==0 and base <= p < base+len(data)] # The runtime keys shards by a PAGE-ALIGNED region_start (the first # dirty page of the overlay's contiguous run), and compile_overlays # takes the capture's load_addr verbatim as that key (phys = load_addr @@ -222,11 +236,13 @@ def main(): page_base = base & ~0xFFF fill = base - page_base region = b'\x00'*fill + data - records.append(rec(page_base, region, seeds)) + records.append(rec(page_base, region, seeds, dispatch_extra=hdr_entries)) nh+=1 + extra=len(set(hdr_entries)-set(seeds)) print(f" [header-table] {p}: {len(data)}B file@0x{base:08X} " f"region@0x{page_base:08X}(+{fill}) " - f"(jal-fit score={score}), prologue seeds={len(seeds)}") + f"(jal-fit score={score}), prologue seeds={len(seeds)} " + f"+{extra} export-table dispatch entries") json.dump(records, open(a.out,'w')) print(f"producers: {np} PS-X EXE (full-discovery), {nh} header-table; {len(records)} regions -> {a.out}") From 43cd37624abc2bd35fdcff73f8f9c2f788f804c6 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Fri, 17 Jul 2026 22:58:09 -0700 Subject: [PATCH 02/37] aot: discover frameless static call targets --- docs/AOT_OVERLAY_PLAN.md | 10 +- recompiler/src/function_analysis.cpp | 169 +++++++++++++++--- recompiler/src/main_psx.cpp | 17 +- recompiler/tests/reachable_discovery_test.cpp | 58 +++++- .../tests/test_aot_overlay_discovery.py | 70 ++++++++ tools/aot_overlay_spike/README.md | 18 +- tools/aot_overlay_spike/coverage_report.py | 11 +- tools/compile_overlays.py | 117 ++++++++++-- 8 files changed, 414 insertions(+), 56 deletions(-) create mode 100644 recompiler/tests/test_aot_overlay_discovery.py diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index 5caa4848..c71e6c10 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -1,10 +1,18 @@ # AOT Overlay Sharding — Spike Findings & Plan (Tomba-first) -Status: SPIKE / DESIGN (2026-07-17). Not yet executed. +Status: ENHANCEMENT SPIKE (2026-07-17). Foundation executed and live-proven. Author: investigation via 4 parallel agents + adversarial source verification. Scope: can we move overlay sharding from *runtime discovery* to *build-time (AOT)*, starting with Tomba (SCUS-94236)? +Current Tomba 2 checkpoint: generic play-free extraction emits 23 clean regions; +all 23 compile under `cg5_d0a05489`. Direct-call frameless-leaf discovery raised +full-playthrough-vault entry recall from 1113/1856 (60.0%) to 1187/1856 (64.0%), +and entry+code_crc recall from 1012 to 1084. All 74 newly covered vault entries are +in MAIN.EXE. The remaining manifest has 669 misses: 268 MAIN.EXE, 356 overlay-window, +and 45 kernel/low-memory entries. Runtime static-only smoke loaded the new tag with +`stale_blocked=0`; misses continue to fail safely to the interpreter via CRC guards. + --- ## ⭐ SESSION-2 BREAKTHROUGH (2026-07-17, VERIFIED FROM DISC + CORPUS) diff --git a/recompiler/src/function_analysis.cpp b/recompiler/src/function_analysis.cpp index 33d7584f..3ac817c0 100644 --- a/recompiler/src/function_analysis.cpp +++ b/recompiler/src/function_analysis.cpp @@ -392,6 +392,19 @@ uint32_t FunctionAnalyzer::find_function_start(uint32_t return_addr) { // from the previous function's trailing gap. return skip_leading_padding_nops(exe_, search_addr + 8, return_addr); } + + // Packed Psy-Q BIOS thunks tail-jump through A0/B0/C0 and therefore do + // not contain `jr $ra`. Treat their JR delay slot as a real boundary + // while scanning backward, otherwise the following frameless routine + // is swallowed into the whole preceding thunk run. + if (search_addr >= exe_.header.load_address + 4u) { + uint32_t thunk_jr = 0; + if (is_bios_dispatch_thunk(search_addr - 4u, thunk_jr) && + thunk_jr == search_addr) { + return skip_leading_padding_nops(exe_, search_addr + 8u, + return_addr); + } + } } // Couldn't find clear start, assume max search distance @@ -506,6 +519,96 @@ static uint32_t exact_jump_target(uint32_t pc, uint32_t instr) { return ((pc + 4u) & 0xF0000000u) | ((instr & 0x03FFFFFFu) << 2); } +static bool exact_instruction_writes_gpr(uint32_t instr, uint32_t reg) { + if (reg == 0u) return false; + + uint32_t opcode = (instr >> 26) & 0x3Fu; + uint32_t rs = (instr >> 21) & 0x1Fu; + uint32_t rt = (instr >> 16) & 0x1Fu; + uint32_t rd = (instr >> 11) & 0x1Fu; + uint32_t funct = instr & 0x3Fu; + + if (opcode == 0x00u) { + // SPECIAL instructions with a GPR destination use rd. JR, stores to + // HI/LO, multiply/divide, syscall, and break have no GPR result. + switch (funct) { + case 0x08u: case 0x0Cu: case 0x0Du: + case 0x11u: case 0x13u: + case 0x18u: case 0x19u: case 0x1Au: case 0x1Bu: + return false; + default: + return rd == reg; + } + } + if (opcode == 0x03u) return reg == 31u; // JAL + if (opcode == 0x01u && (rt == 0x10u || rt == 0x11u)) return reg == 31u; + + // Immediate ALU ops, LUI, and loads write rt. Branches and stores do not. + if ((opcode >= 0x08u && opcode <= 0x0Fu) || + (opcode >= 0x20u && opcode <= 0x26u) || + opcode == 0x30u || opcode == 0x32u) { + return rt == reg; + } + // MFC/CFC from COP0/COP2 write rt; MTC/CTC do not. + if ((opcode == 0x10u || opcode == 0x12u) && (rs == 0u || rs == 2u)) { + return rt == reg; + } + return false; +} + +// Resolve the common absolute indirect-transfer idiom within one basic block: +// +// lui rN, hi(target) +// addiu/ori rN, rN, lo(target) +// jr/jalr rN +// +// The backward scan stops at control flow or any intervening write to rN, so a +// stale constant from another path cannot be mistaken for the transfer target. +static bool exact_resolve_constant_transfer(const PS1Executable& exe, + uint32_t entry, + uint32_t transfer_pc, + uint32_t target_reg, + uint32_t& target_out) { + bool have_low = false; + bool low_is_addiu = false; + uint16_t low = 0; + + for (uint32_t count = 1; count <= 16u; count++) { + uint32_t bytes = count * 4u; + if (transfer_pc < entry + bytes) break; + uint32_t pc = transfer_pc - bytes; + auto word_opt = exe.read_word(pc); + if (!word_opt.has_value()) break; + uint32_t instr = *word_opt; + uint32_t opcode = (instr >> 26) & 0x3Fu; + uint32_t rs = (instr >> 21) & 0x1Fu; + uint32_t rt = (instr >> 16) & 0x1Fu; + + if (!have_low && (opcode == 0x09u || opcode == 0x0Du) && + rs == target_reg && rt == target_reg) { + have_low = true; + low_is_addiu = opcode == 0x09u; + low = static_cast(instr & 0xFFFFu); + continue; + } + if (opcode == 0x0Fu && rt == target_reg) { + uint32_t upper = (instr & 0xFFFFu) << 16; + if (!have_low) { + target_out = upper; + } else if (low_is_addiu) { + target_out = upper + static_cast( + static_cast(static_cast(low))); + } else { + target_out = upper | low; + } + return true; + } + if (exact_instruction_writes_gpr(instr, target_reg)) break; + if (FunctionAnalyzer::is_branch_or_jump(instr)) break; + } + return false; +} + static ExactCf exact_classify_cf(uint32_t pc, uint32_t instr) { ExactCf cf; uint32_t opcode = (instr >> 26) & 0x3Fu; @@ -557,22 +660,11 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries(const std::vector auto callable_direct_jal_target = [&](uint32_t addr) { auto word_opt = exe_.read_word(addr); - if (!word_opt.has_value() || !exact_is_valid_mips_word(*word_opt)) return false; - if (addr == exe_.header.load_address) return true; - - if (addr >= exe_.header.load_address + 8u) { - auto prev8_opt = exe_.read_word(addr - 8u); - if (prev8_opt.has_value() && exact_is_jr_ra_word(*prev8_opt)) return true; - } - - if (exact_is_addiu_sp_neg(*word_opt)) { - if (addr < exe_.header.load_address + 4u) return true; - auto prev_opt = exe_.read_word(addr - 4u); - if (!prev_opt.has_value()) return true; - return exact_classify_cf(addr - 4u, *prev_opt).kind == ExactCfKind::Normal; - } - - return false; + // A JAL encountered by the reachable CFG walk is direct call evidence. + // Requiring a stack prologue here discarded legitimate frameless leaf + // functions and tiny library wrappers. The target still has to decode + // as a valid PS1 instruction and lie inside the verified image bound. + return word_opt.has_value() && exact_is_valid_mips_word(*word_opt); }; auto find_jump_table_targets = [&](uint32_t entry, uint32_t hard_cap, @@ -703,6 +795,15 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries(const std::vector case ExactCfKind::Jalr: if (in_function(delay)) wr.visited.insert(delay); if (in_function(pc + 8u)) work.push(pc + 8u); + { + uint32_t target = 0; + uint32_t target_reg = (instr >> 21) & 0x1Fu; + if (exact_resolve_constant_transfer(exe_, entry, pc, + target_reg, target) && + in_exe(target)) { + wr.direct_jal_targets.insert(target); + } + } break; case ExactCfKind::JrRa: if (in_function(delay)) wr.visited.insert(delay); @@ -712,9 +813,23 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries(const std::vector if (in_function(delay)) wr.visited.insert(delay); { uint32_t jr_rs = (instr >> 21) & 0x1Fu; - for (uint32_t target : find_jump_table_targets(entry, hard_cap, pc, jr_rs)) { - wr.jump_table_targets.insert(target); - if (in_function(target)) work.push(target); + uint32_t target = 0; + if (exact_resolve_constant_transfer(exe_, entry, pc, jr_rs, + target)) { + if (in_function(target)) { + wr.jump_table_targets.insert(target); + work.push(target); + } else if (in_exe(target)) { + // An absolute JR out of the current function is a + // statically proven tail-call entry. + wr.direct_jal_targets.insert(target); + } + } else { + for (uint32_t jt : find_jump_table_targets(entry, hard_cap, + pc, jr_rs)) { + wr.jump_table_targets.insert(jt); + if (in_function(jt)) work.push(jt); + } } } break; @@ -1286,13 +1401,21 @@ FunctionAnalysisResult FunctionAnalyzer::analyze() { break; case ExactCfKind::JrOther: { if (in_fn(delay)) visited.insert(delay); - std::vector jt; uint32_t jr_rs = (instr >> 21) & 0x1Fu; - if (resolve_jt_p3(entry, hard_cap, pc, jr_rs, jt)) { + uint32_t constant_target = 0; + if (exact_resolve_constant_transfer(exe_, entry, pc, jr_rs, + constant_target)) { resolved_jt = true; - for (uint32_t t : jt) if (in_fn(t)) work.push(t); + if (in_fn(constant_target)) work.push(constant_target); + else reached_exit = true; } else { - unresolved_indirect = true; // computed jump we can't bound + std::vector jt; + if (resolve_jt_p3(entry, hard_cap, pc, jr_rs, jt)) { + resolved_jt = true; + for (uint32_t t : jt) if (in_fn(t)) work.push(t); + } else { + unresolved_indirect = true; // computed jump we can't bound + } } break; } diff --git a/recompiler/src/main_psx.cpp b/recompiler/src/main_psx.cpp index 340a9287..c2459eca 100644 --- a/recompiler/src/main_psx.cpp +++ b/recompiler/src/main_psx.cpp @@ -420,7 +420,9 @@ int main(int argc, char** argv) { * 0xCF0): the static recompiler's install-slot hooks tail-dispatch into * exactly these PCs, so they are real execution roots even though no * prologue / preceding jr $ra exists. They are trusted walk roots and - * exempt from the overlay-mode boundary re-check below. + * exempt from the overlay-mode boundary re-check below. Lines of the + * form `call_root 0xXXXXXXXX` carry the same mechanical trust for a + * statically proven direct or constant-register call/tail-call target. * * Seeds are accepted within the loaded image's own bounds — overlay mode * wraps arbitrary regions (kernel RAM at 0x80000000, overlays at @@ -429,6 +431,7 @@ int main(int argc, char** argv) { std::vector file_seeds; std::vector interior_seeds; std::set trusted_root_seeds; + std::set trusted_call_root_seeds; if (extra_funcs_path) { std::ifstream ef(extra_funcs_path); if (ef.is_open()) { @@ -439,6 +442,7 @@ int main(int argc, char** argv) { if (line.empty() || line[0] == '#') continue; bool interior = false; bool trusted_root = false; + bool trusted_call_root = false; const char* p = line.c_str(); if (line.rfind("interior", 0) == 0) { interior = true; @@ -446,6 +450,9 @@ int main(int argc, char** argv) { } else if (line.rfind("dispatch_root", 0) == 0) { trusted_root = true; p += 13; + } else if (line.rfind("call_root", 0) == 0) { + trusted_call_root = true; + p += 9; } uint32_t addr = (uint32_t)std::strtoul(p, nullptr, 16); if (addr >= seed_lo && addr < seed_hi) { @@ -453,15 +460,17 @@ int main(int argc, char** argv) { interior_seeds.push_back(addr); } else { if (trusted_root) trusted_root_seeds.insert(addr); + if (trusted_call_root) trusted_call_root_seeds.insert(addr); file_seeds.push_back(addr); exact_entries.push_back(addr); } } } fmt::print("Loaded {} extra function addresses ({} interior, " - "{} dispatch-root) from {}\n", + "{} dispatch-root, {} call-root) from {}\n", file_seeds.size() + interior_seeds.size(), interior_seeds.size(), trusted_root_seeds.size(), + trusted_call_root_seeds.size(), extra_funcs_path); } else { fmt::print("WARNING: Cannot open extra-funcs file: {}\n", extra_funcs_path); @@ -507,6 +516,10 @@ int main(int argc, char** argv) { fmt::print(" seed 0x{:08X} accepted as dispatch root " "(install-slot class, no boundary evidence)\n", a); roots.insert(a); + } else if (trusted_call_root_seeds.count(a)) { + fmt::print(" seed 0x{:08X} accepted as static call root " + "(direct/constant-register target)\n", a); + roots.insert(a); } else if (callable_boundary(a)) { roots.insert(a); } else { diff --git a/recompiler/tests/reachable_discovery_test.cpp b/recompiler/tests/reachable_discovery_test.cpp index 64429fd2..5ea74b2b 100644 --- a/recompiler/tests/reachable_discovery_test.cpp +++ b/recompiler/tests/reachable_discovery_test.cpp @@ -60,16 +60,18 @@ int main() { auto image = make_exe_buffer(0x2000); const size_t text = 2048; - // Root: direct call to a callable target, unresolved jalr, direct call - // beyond the eventual bound, then return. + // Root: direct call to a framed target, direct call to a frameless leaf, + // unresolved jalr, direct call beyond the eventual bound, then return. put32(image, text + 0x00, jal(kLoad + 0x100)); put32(image, text + 0x04, 0x00000000u); - put32(image, text + 0x08, 0x0320F809u); // jalr $ra,$t9 (unresolved) + put32(image, text + 0x08, jal(kLoad + 0x300)); put32(image, text + 0x0C, 0x00000000u); - put32(image, text + 0x10, jal(kLoad + 0x1100)); + put32(image, text + 0x10, 0x0320F809u); // jalr $ra,$t9 (unresolved) put32(image, text + 0x14, 0x00000000u); - put32(image, text + 0x18, 0x03E00008u); + put32(image, text + 0x18, jal(kLoad + 0x1100)); put32(image, text + 0x1C, 0x00000000u); + put32(image, text + 0x20, 0x03E00008u); + put32(image, text + 0x24, 0x00000000u); // Callable direct target. put32(image, text + 0x100, 0x27BDFFF0u); @@ -81,6 +83,12 @@ int main() { put32(image, text + 0x204, 0x03E00008u); put32(image, text + 0x208, 0x27BD0010u); + // A direct JAL is sufficient callable-boundary evidence even when the + // target is a tiny frameless leaf. + put32(image, text + 0x300, 0x24020001u); // addiu $v0,$zero,1 + put32(image, text + 0x304, 0x03E00008u); + put32(image, text + 0x308, 0x00000000u); + // Callable target outside the verified static bound. put32(image, text + 0x1100, 0x27BDFFF0u); put32(image, text + 0x1104, 0x03E00008u); @@ -95,13 +103,49 @@ int main() { PSXRecomp::FunctionAnalyzer analyzer(exe); const auto reachable = starts(analyzer.analyze_exact_entries({kLoad})); - CHECK(reachable == std::set({kLoad, kLoad + 0x100}), - "reachable analysis follows direct callable JAL only"); + CHECK(reachable == std::set({kLoad, kLoad + 0x100, + kLoad + 0x300}), + "reachable analysis follows framed and frameless direct JAL targets"); CHECK(!reachable.count(kLoad + 0x200), "unseen indirect callback fails closed"); CHECK(!reachable.count(kLoad + 0x1100), "direct target beyond verified bound fails closed"); + // A constant-register JR out of the current function is a statically + // proven tail call. Resolve the common lui/addiu/jr sequence without an + // execution-derived seed. + auto indirect_image = make_exe_buffer(0x1000, kLoad + 0x500); + put32(indirect_image, text + 0x400, 0x24020001u); + put32(indirect_image, text + 0x404, 0x03E00008u); + put32(indirect_image, text + 0x408, 0x00000000u); + put32(indirect_image, text + 0x500, 0x3C088001u); // lui $t0,0x8001 + put32(indirect_image, text + 0x504, 0x25080400u); // addiu $t0,$t0,0x400 + put32(indirect_image, text + 0x508, 0x01000008u); // jr $t0 + put32(indirect_image, text + 0x50C, 0x00000000u); + auto indirect_exe = parse(indirect_image); + PSXRecomp::FunctionAnalyzer indirect_analyzer(indirect_exe); + const auto indirect = starts( + indirect_analyzer.analyze_exact_entries({kLoad + 0x500})); + CHECK(indirect.count(kLoad + 0x400) && indirect.count(kLoad + 0x500), + "constant-register JR discovers its frameless tail-call target"); + + // A packed BIOS thunk is also a backward-scan boundary. The frameless + // leaf after its delay slot must not be swallowed into the thunk run. + auto thunk_image = make_exe_buffer(0x1000); + put32(thunk_image, text + 0x600, 0x240A00B0u); + put32(thunk_image, text + 0x604, 0x01400008u); + put32(thunk_image, text + 0x608, 0x24090008u); + put32(thunk_image, text + 0x60C, 0x00000000u); + put32(thunk_image, text + 0x610, 0x24020001u); + put32(thunk_image, text + 0x614, 0x03E00008u); + put32(thunk_image, text + 0x618, 0x00000000u); + auto thunk_exe = parse(thunk_image); + PSXRecomp::FunctionAnalyzer thunk_analyzer(thunk_exe); + const auto thunk_starts = starts(thunk_analyzer.analyze()); + CHECK(thunk_starts.count(kLoad + 0x600) && + thunk_starts.count(kLoad + 0x610), + "packed BIOS thunk bounds the following frameless leaf"); + PSXRecomp::FunctionAnalyzer seeded_analyzer(exe); const auto seeded = starts( seeded_analyzer.analyze_exact_entries({kLoad, kLoad + 0x200})); diff --git a/recompiler/tests/test_aot_overlay_discovery.py b/recompiler/tests/test_aot_overlay_discovery.py new file mode 100644 index 00000000..fb3d253a --- /dev/null +++ b/recompiler/tests/test_aot_overlay_discovery.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Focused regression checks for play-free overlay call-target discovery.""" +import importlib.util +import pathlib +import struct +import sys + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "compile_overlays", ROOT / "tools" / "compile_overlays.py") +MOD = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MOD) + +LOAD = 0x80010000 + + +def put(data, offset, word): + struct.pack_into("> 2) & 0x03FFFFFF) + + +def main(): + data = bytearray(0x200) + + # A framed root directly calls a frameless leaf. JAL is the boundary + # proof; the leaf deliberately has neither a prologue nor a preceding + # return. + put(data, 0x00, 0x27BDFFF0) + put(data, 0x04, jal(LOAD + 0x40)) + put(data, 0x08, 0x00000000) + put(data, 0x0C, 0x03E00008) + put(data, 0x10, 0x27BD0010) + put(data, 0x40, 0x24020001) + put(data, 0x44, 0x03E00008) + put(data, 0x48, 0x00000000) + + cap = { + "schema": "psxrecomp overlay capture v2", + "load_addr": f"0x{LOAD:08X}", + "size": len(data), + "executed_pcs": [], + "dispatch_entry_pcs": [f"0x{LOAD:08X}"], + "function_entry_pcs": [f"0x{LOAD:08X}"], + "seeds": [f"0x{LOAD:08X}"], + } + seeds, audit = MOD.classify_overlay_seeds( + cap, bytes(data), LOAD, len(data), 0, {}) + assert audit["included_reasons"][LOAD + 0x40] == "DIRECT_JAL_TARGET" + assert f"call_root 0x{LOAD + 0x40:08X}" in seeds + + # A constant-register JR tail-calls backward to a frameless leaf. + put(data, 0x100, 0x27BDFFF0) + put(data, 0x104, 0x3C088001) # lui $t0,0x8001 + put(data, 0x108, 0x25080040) # addiu $t0,$t0,0x40 + put(data, 0x10C, 0x01000008) # jr $t0 + put(data, 0x110, 0x27BD0010) + walk = MOD._walk_overlay_function( + bytes(data), LOAD, len(data), LOAD + 0x100, LOAD + len(data)) + assert walk["static_indirect_targets"] == {LOAD + 0x40} + + print("ALL PASS") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index efb5db0e..6a81a92f 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -28,10 +28,12 @@ discovery quality, not to be run as-is in a build. - **Tomba 1:** 90% of played functions reproduced play-free + ~1964 unplayed extra; live-validated in-game (Dwarf Village went native, no runtime compile). -- **Tomba 2:** 661 functions byte-identical (entry+code_crc) to the shipped/played - vault — provably real — from the MAIN.EXE producer. The BIN header-table producer - now recovers a correct, self-consistent common base (jal-fit, below) — 21 A*.BIN - overlays converge on region 0x80108000 (+3996), 4 weak-signal files safely skipped. +- **Tomba 2:** the clean play-free cache now covers 1187/1856 played entries + (**64.0%**) and byte-matches 1084 (**58.4% entry+code_crc**). Direct-call + provenance for frameless leaves added 74 MAIN.EXE hits over the 60.0% baseline. + The BIN header-table producer recovers a correct, self-consistent common base + (jal-fit, below): 21 A*.BIN overlays converge on region 0x80108000 (+3996), with + 4 weak-signal files safely skipped. ## Correctness guarantee (why partial coverage is safe) @@ -82,8 +84,12 @@ misses, production autocompile (tcc/gcc) self-heals on first visit. ## Next to raise coverage (future session) -1. Frameless-leaf + indirect-call-table seed discovery (helps every game; would - lift MAIN.EXE past ~62% and Tomba 1 past 90%). +1. ~~Direct-call frameless-leaf discovery~~ — DONE. Reachable `jal` targets are + emitted as trusted `call_root` seeds; same-basic-block constant-register + `jalr`/tail-`jr` targets are resolved with clobber checks. Tomba 2 vault recall + rose 60.0% -> 64.0% (+74 MAIN.EXE entries), with 23/23 shards compiling cleanly. + Continue with statically proven function-pointer/jump-table consumers; the + first corpus sweep correctly rejected apparent constants clobbered by `lw`. 2. ~~Header-table base recovery~~ — DONE (jal-fit, above). 3. Recover the weak-signal skipped overlays (Tomba 2 A09/A0J/GAME/OPN): likely compressed/archived or too few intra-overlay calls; may need a decompressor or diff --git a/tools/aot_overlay_spike/coverage_report.py b/tools/aot_overlay_spike/coverage_report.py index 04ce8a02..6a25b878 100644 --- a/tools/aot_overlay_spike/coverage_report.py +++ b/tools/aot_overlay_spike/coverage_report.py @@ -128,9 +128,14 @@ def main(): f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') md.append(f'- **MISSED live: {r["missed"]}**\n') - os.makedirs(os.path.dirname(a.out_md), exist_ok=True) - open(a.out_md, 'w', encoding='utf-8').write('\n'.join(md)) - json.dump(report, open(a.out_json, 'w'), indent=1) + for output_path in (a.out_md, a.out_json): + output_dir = os.path.dirname(os.path.abspath(output_path)) + os.makedirs(output_dir, exist_ok=True) + with open(a.out_md, 'w', encoding='utf-8', newline='\n') as out_md: + out_md.write('\n'.join(md).rstrip('\n') + '\n') + with open(a.out_json, 'w', encoding='utf-8', newline='\n') as out_json: + json.dump(report, out_json, indent=1) + out_json.write('\n') print(f'wrote {a.out_md}') print(f'wrote {a.out_json}') for src, m in report['sources'].items(): diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index ca446a2a..a39f0000 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -281,6 +281,7 @@ def make_psxexe(load_addr: int, entry_pc: int, data: bytes) -> bytes: INCLUDE_REASONS = { 'DISPATCH_ENTRY', 'DIRECT_JAL_TARGET', + 'STATIC_INDIRECT_TARGET', 'FUNCTION_POINTER_TARGET', 'TOML_DECLARED_ENTRY', # Dispatch-proven PC with no callable boundary (mid-function code reached @@ -504,6 +505,64 @@ def _callable_direct_jal_target(data: bytes, load_addr: int, addr: int) -> bool: return _callable_legacy_seed(data, load_addr, addr) +def _instruction_writes_gpr(word: int, reg: int) -> bool: + if not reg: + return False + op = (word >> 26) & 0x3F + rs = (word >> 21) & 0x1F + rt = (word >> 16) & 0x1F + rd = (word >> 11) & 0x1F + fn = word & 0x3F + if op == 0: + if fn in (0x08, 0x0C, 0x0D, 0x11, 0x13, + 0x18, 0x19, 0x1A, 0x1B): + return False + return rd == reg + if op == 0x03: + return reg == 31 + if op == 0x01 and rt in (0x10, 0x11): + return reg == 31 + if op in range(0x08, 0x10) or op in range(0x20, 0x27) \ + or op in (0x30, 0x32): + return rt == reg + if op in (0x10, 0x12) and rs in (0, 2): + return rt == reg + return False + + +def _resolve_constant_transfer(data: bytes, load_addr: int, + entry: int, transfer_pc: int, + target_reg: int): + """Resolve a same-basic-block lui + addiu/ori constant feeding jr/jalr.""" + low = None + low_is_addiu = False + for count in range(1, 17): + pc = transfer_pc - count * 4 + if pc < entry: + break + word = _word_at(data, load_addr, pc) + if word is None: + break + op = (word >> 26) & 0x3F + rs = (word >> 21) & 0x1F + rt = (word >> 16) & 0x1F + if low is None and op in (0x09, 0x0D) \ + and rs == target_reg and rt == target_reg: + low = word & 0xFFFF + low_is_addiu = op == 0x09 + continue + if op == 0x0F and rt == target_reg: + upper = (word & 0xFFFF) << 16 + if low is None: + return upper + if low_is_addiu and low & 0x8000: + low -= 0x10000 + return (upper + low) & 0xFFFFFFFF if low_is_addiu else upper | low + if _instruction_writes_gpr(word, target_reg) or _is_control_flow(word): + break + return None + + def _walk_overlay_function(data: bytes, load_addr: int, size: int, entry: int, hard_cap: int) -> dict: lo = load_addr @@ -515,6 +574,7 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, work = deque([entry]) visited = set() direct_jals = set() + static_indirect_targets = set() branch_targets = set() jump_table_targets = set() @@ -554,7 +614,7 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, work.append(pc + 8) branch_targets.add(pc + 8) if (lo <= target < hi and (target & 3) == 0 and - _callable_direct_jal_target(data, load_addr, target)): + _is_valid_mips_word(_word_at(data, load_addr, target))): direct_jals.add(target) elif kind == 'jalr': if in_function(delay): @@ -562,16 +622,29 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, if in_function(pc + 8): work.append(pc + 8) branch_targets.add(pc + 8) + target_reg = (word >> 21) & 0x1F + target = _resolve_constant_transfer(data, load_addr, entry, pc, + target_reg) + if target is not None and lo <= target < hi and (target & 3) == 0: + static_indirect_targets.add(target) elif kind == 'jr': if in_function(delay): visited.add(delay) jr_rs = (word >> 21) & 0x1F - for jt in _find_jump_table_targets(data, load_addr, size, - entry, hard_cap, pc, jr_rs): - jump_table_targets.add(jt) - branch_targets.add(jt) - if in_function(jt): - work.append(jt) + target = _resolve_constant_transfer(data, load_addr, entry, pc, jr_rs) + if target is not None and lo <= target < hi and (target & 3) == 0: + if in_function(target): + branch_targets.add(target) + work.append(target) + else: + static_indirect_targets.add(target) + else: + for jt in _find_jump_table_targets(data, load_addr, size, + entry, hard_cap, pc, jr_rs): + jump_table_targets.add(jt) + branch_targets.add(jt) + if in_function(jt): + work.append(jt) elif kind == 'jr_ra': if in_function(delay): visited.add(delay) @@ -579,6 +652,7 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, return { 'visited': visited, 'direct_jals': direct_jals, + 'static_indirect_targets': static_indirect_targets, 'branch_targets': branch_targets, 'jump_table_targets': jump_table_targets, } @@ -679,14 +753,19 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, not _callable_legacy_seed(data, load_addr, addr)): included.setdefault(addr, 'DISPATCH_INTERIOR') return - elif (reason != 'TOML_DECLARED_ENTRY' and addr in jump_table_targets and + elif (reason not in ('DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET', + 'TOML_DECLARED_ENTRY') and + addr in jump_table_targets and not _callable_legacy_seed(data, load_addr, addr)): excluded[addr] = 'BRANCH_TARGET_ONLY' return if reason in FATAL_SEED_REASONS: raise RuntimeError(f'BUG: refusing to include 0x{addr:08X} as {reason}') old = included.get(addr) - if old is None or old in ('FUNCTION_POINTER_TARGET', 'DISPATCH_INTERIOR'): + if reason in ('DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET'): + # Static call-edge evidence outranks heuristic capture metadata. + included[addr] = reason + elif old is None or old in ('FUNCTION_POINTER_TARGET', 'DISPATCH_INTERIOR'): included[addr] = reason for addr in dispatch_entry_pcs: @@ -725,11 +804,18 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, all_branch_targets.update(walk['branch_targets']) all_branch_targets.update(walk['jump_table_targets']) for target in sorted(walk['direct_jals']): - if target not in known: - include(target, 'DIRECT_JAL_TARGET') - if target in included: - known.add(target) - pending.append(target) + # Always re-include: a target may already be present as a + # heuristic FUNCTION_POINTER_TARGET. Reachable JAL evidence + # upgrades it to a trusted call root. + include(target, 'DIRECT_JAL_TARGET') + if target in included and target not in known: + known.add(target) + pending.append(target) + for target in sorted(walk['static_indirect_targets']): + include(target, 'STATIC_INDIRECT_TARGET') + if target in included and target not in known: + known.add(target) + pending.append(target) if not kernel_window: break @@ -810,6 +896,8 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, return f'interior 0x{addr:08X}' if r == 'DISPATCH_ROOT': return f'dispatch_root 0x{addr:08X}' + if r in ('DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET'): + return f'call_root 0x{addr:08X}' return f'0x{addr:08X}' seeds = [seed_line(addr) for addr in sorted(included)] return seeds, audit @@ -822,6 +910,7 @@ def print_seed_audit(audit: dict) -> None: print(f'dispatch_entry_pcs: {len(audit["dispatch_entry_pcs"])}') print(f'function_entry_pcs: {len(audit["function_entry_pcs"])}') print(f'direct_jal_targets_included: {audit["counts"].get("DIRECT_JAL_TARGET", 0)}') + print(f'static_indirect_targets_included: {audit["counts"].get("STATIC_INDIRECT_TARGET", 0)}') print(f'function_pointer_targets_included: {audit["counts"].get("FUNCTION_POINTER_TARGET", 0)}') print(f'toml_entries_included: {audit["counts"].get("TOML_DECLARED_ENTRY", 0)}') print(f'dispatch_interior_included: {audit["counts"].get("DISPATCH_INTERIOR", 0)}') From 2834f0468d7b9b411b7bb2692ea80ce66f8072c5 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Fri, 17 Jul 2026 23:29:38 -0700 Subject: [PATCH 03/37] aot: recover call-sparse overlay bases --- docs/AOT_OVERLAY_PLAN.md | 13 +- recompiler/tests/test_aot_base_recovery.py | 42 +++++++ tools/aot_overlay_spike/README.md | 20 ++-- tools/aot_overlay_spike/extract_generic.py | 132 ++++++++++++++------- 4 files changed, 146 insertions(+), 61 deletions(-) create mode 100644 recompiler/tests/test_aot_base_recovery.py diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index c71e6c10..e5182101 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -5,12 +5,13 @@ Author: investigation via 4 parallel agents + adversarial source verification. Scope: can we move overlay sharding from *runtime discovery* to *build-time (AOT)*, starting with Tomba (SCUS-94236)? -Current Tomba 2 checkpoint: generic play-free extraction emits 23 clean regions; -all 23 compile under `cg5_d0a05489`. Direct-call frameless-leaf discovery raised -full-playthrough-vault entry recall from 1113/1856 (60.0%) to 1187/1856 (64.0%), -and entry+code_crc recall from 1012 to 1084. All 74 newly covered vault entries are -in MAIN.EXE. The remaining manifest has 669 misses: 268 MAIN.EXE, 356 overlay-window, -and 45 kernel/low-memory entries. Runtime static-only smoke loaded the new tag with +Current Tomba 2 checkpoint: generic play-free extraction emits 27 clean regions; +all 27 compile under `cg5_d0a05489`. Direct-call frameless-leaf discovery plus +call-only/cross-file base recovery raised full-playthrough-vault entry recall from +1113/1856 (60.0%) to 1219/1856 (65.7%), and entry+code_crc recall from 1012 to +1102. The gains are 74 MAIN.EXE entries plus 32 overlay-window entries. The +remaining manifest has 637 misses: 268 MAIN.EXE, 324 overlay-window, and 45 +kernel/low-memory entries. Runtime static-only smoke loaded the matching tag with `stale_blocked=0`; misses continue to fail safely to the interpreter via CRC guards. --- diff --git a/recompiler/tests/test_aot_base_recovery.py b/recompiler/tests/test_aot_base_recovery.py new file mode 100644 index 00000000..5c95e9a3 --- /dev/null +++ b/recompiler/tests/test_aot_base_recovery.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Focused regression checks for generic AOT overlay base selection.""" +import collections +import importlib.util +import pathlib +import struct +import sys + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "extract_generic", ROOT / "tools" / "aot_overlay_spike" / + "extract_generic.py") +MOD = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MOD) + + +def main(): + true_base = 0x80106228 + nearby_false = 0x80106220 + + # A near-tie without independent evidence fails closed. If a call-dense + # sibling has already proved the exact true base, that one unique match is + # enough to resolve the call-sparse file. + votes = collections.Counter({nearby_false: 13, true_base: 12}) + assert MOD._select_base_vote(votes) is None + assert MOD._select_base_vote(votes, {true_base}) == (true_base, 12) + assert MOD._select_base_vote(votes, {true_base, nearby_false}) is None + + # Only JAL is callable-boundary evidence. A nearby unconditional J branch + # must not participate in base recovery. + data = bytearray(12) + struct.pack_into("> 2) & 0x03FFFFFF)) + struct.pack_into("> 2) & 0x03FFFFFF)) + assert MOD.jal_targets(data) == {0x80105678} + + print("ALL PASS") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 6a81a92f..296aded4 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -28,12 +28,12 @@ discovery quality, not to be run as-is in a build. - **Tomba 1:** 90% of played functions reproduced play-free + ~1964 unplayed extra; live-validated in-game (Dwarf Village went native, no runtime compile). -- **Tomba 2:** the clean play-free cache now covers 1187/1856 played entries - (**64.0%**) and byte-matches 1084 (**58.4% entry+code_crc**). Direct-call - provenance for frameless leaves added 74 MAIN.EXE hits over the 60.0% baseline. - The BIN header-table producer recovers a correct, self-consistent common base - (jal-fit, below): 21 A*.BIN overlays converge on region 0x80108000 (+3996), with - 4 weak-signal files safely skipped. +- **Tomba 2:** the clean play-free cache now covers 1219/1856 played entries + (**65.7%**) and byte-matches 1102 (**59.4% entry+code_crc**). Direct-call + provenance for frameless leaves added 74 MAIN.EXE hits over the 60.0% baseline; + recovering the four call-sparse files added another 32 overlay-window hits. + All 22 A*.BIN overlays converge on 0x80108F9C (region 0x80108000 +3996), + DEMO/GAME converge on 0x80106228, and OPN resolves to 0x8018A000. ## Correctness guarantee (why partial coverage is safe) @@ -91,8 +91,10 @@ misses, production autocompile (tcc/gcc) self-heals on first visit. Continue with statically proven function-pointer/jump-table consumers; the first corpus sweep correctly rejected apparent constants clobbered by `lw`. 2. ~~Header-table base recovery~~ — DONE (jal-fit, above). -3. Recover the weak-signal skipped overlays (Tomba 2 A09/A0J/GAME/OPN): likely - compressed/archived or too few intra-overlay calls; may need a decompressor or - a lower-confidence seed source before jal-fit can lock a base. +3. ~~Recover weak-signal A09/A0J/GAME/OPN~~ — DONE. They are ordinary uncompressed + MIPS, not archives. Counting only callable `jal` targets sharply resolves + A09/A0J/OPN; GAME's 8-byte near-tie resolves through the exact base independently + proved by DEMO. The played vault byte-proves GAME at 0x80106228. Ambiguous files + without one unique trusted match still fail closed. 4. Extract the kernel `0x80000000` region once from the BIOS (game-independent). 5. Generalize into a real `tools/` producer (no hardcoded paths; disc + game.toml in). diff --git a/tools/aot_overlay_spike/extract_generic.py b/tools/aot_overlay_spike/extract_generic.py index e88e503b..870b15a9 100644 --- a/tools/aot_overlay_spike/extract_generic.py +++ b/tools/aot_overlay_spike/extract_generic.py @@ -55,25 +55,53 @@ def prologue_offsets(data): and (struct.unpack_from('>26) in (0x02,0x03): # j / jal + if (w>>26) == 0x03: # jal only ts.add(0x80000000 | ((w & 0x03FFFFFF) << 2)) return ts -def recover_base(data, ptrs): +def _select_base_vote(hist, trusted_bases=()): + """Select a sharp vote peak, or an exact independently trusted near-tie. + + A trusted match is useful when two position-fixed files share a load base: + one call-dense file proves it sharply, while a small sibling has the same + exact base among its candidates but too few calls to clear the ratio test. + Require one unique trusted match with at least four votes; otherwise fail + closed. + """ + if not hist: + return None + top=hist.most_common(2) + base,score=top[0] + second=top[1][1] if len(top)>1 else 0 + if score >= 4 and score >= second*1.5: + return base, score + trusted=[(base, hist[base]) for base in set(trusted_bases) + if hist.get(base, 0) >= 4] + return trusted[0] if len(trusted) == 1 else None + +def recover_base(data, ptrs, trusted_bases=()): """Recover the verbatim load base of a position-fixed header-table overlay. The export-table pointers point at mid-function DISPATCH entries, not prologues, so matching them against 0x27BD is useless (0 hits at the true base — measured). Instead use jal-target self-consistency: jal targets are base-independent, and at the TRUE base the maximum number of intra-overlay - jal targets land on a real 0x27BD prologue. This peaks uniquely and sharply - on the correct base (~2x margin over runner-up on Tomba 1's 22 overlays). + jal call targets land on a real 0x27BD prologue. This peaks uniquely and + sharply on the correct base (~2x margin over runner-up on Tomba 1's 22 + overlays). Small call-sparse siblings may additionally match one exact base + independently proved by another file; ambiguous/no-consensus cases still + fail closed. Returns (base, score) or None when the signal is too weak to trust (safe: a skipped overlay is coverage loss, never wrong execution — the runtime's @@ -100,12 +128,7 @@ def recover_base(data, ptrs): for o in P[lo_i:hi_i]: hist[t-o]+=1 if not hist: return None - top=hist.most_common(2) - base,score=top[0] - second=top[1][1] if len(top)>1 else 0 - if score < 4 or score < second*1.5: # demand a clear, unique-ish peak - return None - return base, score + return _select_base_vote(hist, trusted_bases) def is_psx_exe(data): return len(data) >= 0x20 and data[:8] == b'PS-X EXE' @@ -177,7 +200,7 @@ def main(): os.makedirs(tmp, exist_ok=True) # The boot EXE (game.toml `exe`) is the STATIC-recompiled base, NOT an overlay. exe_base=os.path.basename(str(game.get('exe',''))).upper() - records=[]; np=nh=0; seen_crc=set() + records=[]; np=nh=0; seen_crc=set(); header_files=[] for p,l,s in sorted(files): if s<8 or s>2_000_000: continue if exe_base and os.path.basename(p).upper()==exe_base: @@ -209,40 +232,57 @@ def main(): else: cnt=is_header_table(data) if not cnt: continue - # jal-target self-consistency base recovery (robust; supersedes the - # old prologue-density delta-sweep, which matched export pointers - # against prologues they never point at -> wrong base by 2-16KB). ptrs=struct.unpack_from(f'<{cnt}I',data,4) - rb=recover_base(data, ptrs) - if rb is None: - print(f" [header-table] {p}: base signal too weak, SKIPPED (safe)") - continue - base,score=rb - seeds=prologues(data,base) # seed via full-file prologue scan - # SUPPLEMENT the prologue seeds with the overlay's OWN export/dispatch - # table (the header pointers) — the game's authoritative list of live - # entry points, sitting on the disc (play-free). These are exactly the - # indirect-dispatch-only / interior entries a prologue scan can't see; - # compile_overlays emits them as DISPATCH_INTERIOR aliases as needed. - hdr_entries=[p for p in ptrs - if p and (p&3)==0 and base <= p < base+len(data)] - # The runtime keys shards by a PAGE-ALIGNED region_start (the first - # dirty page of the overlay's contiguous run), and compile_overlays - # takes the capture's load_addr verbatim as that key (phys = load_addr - # & 0x1FFFFFFF, no re-align). So we must present a page-aligned base - # with a FILL prefix bridging page_base -> file base, exactly as the - # played-vault layout for these position-fixed overlays. Seeds are - # absolute so they are unaffected; region bytes stay byte-identical. - page_base = base & ~0xFFF - fill = base - page_base - region = b'\x00'*fill + data - records.append(rec(page_base, region, seeds, dispatch_extra=hdr_entries)) - nh+=1 - extra=len(set(hdr_entries)-set(seeds)) - print(f" [header-table] {p}: {len(data)}B file@0x{base:08X} " - f"region@0x{page_base:08X}(+{fill}) " - f"(jal-fit score={score}), prologue seeds={len(seeds)} " - f"+{extra} export-table dispatch entries") + header_files.append((p, data, ptrs)) + + # Resolve call-dense files first. A small sibling that does not clear the + # peak-margin test may then reuse one exact base independently proved by a + # strong file, but only when that trusted candidate is unique. + resolved=[]; weak=[] + for p,data,ptrs in header_files: + rb=recover_base(data, ptrs) + if rb is None: + weak.append((p,data,ptrs)) + else: + resolved.append((p,data,ptrs,rb,'jal-fit')) + trusted_bases={rb[0] for _,_,_,rb,_ in resolved} + for p,data,ptrs in weak: + rb=recover_base(data, ptrs, trusted_bases) + if rb is None: + print(f" [header-table] {p}: base signal too weak, SKIPPED (safe)") + else: + resolved.append((p,data,ptrs,rb,'jal-fit+cross-file')) + + for p,data,ptrs,rb,fit_method in sorted(resolved): + # jal-call-target self-consistency base recovery (robust; supersedes + # the old prologue-density delta-sweep, which matched export pointers + # against prologues they never point at -> wrong base by 2-16KB). + base,score=rb + seeds=prologues(data,base) # seed via full-file prologue scan + # SUPPLEMENT the prologue seeds with the overlay's OWN export/dispatch + # table (the header pointers) — the game's authoritative list of live + # entry points, sitting on the disc (play-free). These are exactly the + # indirect-dispatch-only / interior entries a prologue scan can't see; + # compile_overlays emits them as DISPATCH_INTERIOR aliases as needed. + hdr_entries=[p for p in ptrs + if p and (p&3)==0 and base <= p < base+len(data)] + # The runtime keys shards by a PAGE-ALIGNED region_start (the first + # dirty page of the overlay's contiguous run), and compile_overlays + # takes the capture's load_addr verbatim as that key (phys = load_addr + # & 0x1FFFFFFF, no re-align). So we must present a page-aligned base + # with a FILL prefix bridging page_base -> file base, exactly as the + # played-vault layout for these position-fixed overlays. Seeds are + # absolute so they are unaffected; region bytes stay byte-identical. + page_base = base & ~0xFFF + fill = base - page_base + region = b'\x00'*fill + data + records.append(rec(page_base, region, seeds, dispatch_extra=hdr_entries)) + nh+=1 + extra=len(set(hdr_entries)-set(seeds)) + print(f" [header-table] {p}: {len(data)}B file@0x{base:08X} " + f"region@0x{page_base:08X}(+{fill}) " + f"({fit_method} score={score}), prologue seeds={len(seeds)} " + f"+{extra} export-table dispatch entries") json.dump(records, open(a.out,'w')) print(f"producers: {np} PS-X EXE (full-discovery), {nh} header-table; {len(records)} regions -> {a.out}") From a544809edf796e08eb5d603b2ebff1ad43e03646 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Fri, 17 Jul 2026 23:56:58 -0700 Subject: [PATCH 04/37] aot: persist overlay capture history atomically --- .gitignore | 1 + docs/AOT_OVERLAY_PLAN.md | 18 ++ recompiler/src/config_loader.cpp | 19 ++ recompiler/src/config_loader.h | 13 + recompiler/tests/recompiler_patch_test.cpp | 32 +++ runtime/include/overlay_capture.h | 9 + runtime/src/main.cpp | 21 ++ runtime/src/overlay_capture.c | 315 +++++++++++++++++++-- runtime/tests/test_capture_history.py | 64 +++++ tools/aot_overlay_spike/README.md | 18 ++ tools/coverage_vault.py | 77 ++++- 11 files changed, 554 insertions(+), 33 deletions(-) create mode 100644 runtime/tests/test_capture_history.py diff --git a/.gitignore b/.gitignore index b67a0585..cdd34f56 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ sdl2-msvc/ # Diagnostic dumps & smoke logs (transient) *.log *.jsonl +/.aot_capture_history/ # Build-generated codegen-hash header (hash_codegen.cmake): folds an emitter # content hash into the overlay cache path; regenerated each build, never committed. diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index e5182101..3eb87c84 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -116,6 +116,24 @@ executed_pcs[], dispatch_entry_pcs[], function_entry_pcs[], seeds[] (`g_dirty_ram_pc_table`, `g_dirty_ram_exec_pc_table`) — **execution-verified**, not disassembly guesses. +#### Durable capture history (opt-in) + +`overlay_captures.json` remains the canonical latest snapshot consumed by the live +compiler, but it is published through a same-directory temporary file plus atomic +replace so readers never observe a half-written manifest. Games may additionally +set `overlay_capture_history = true` to append each changed coherent snapshot as +an independent JSONL record to `overlay_captures.addendum.jsonl` beside the +executable. A hard kill can damage only the last line; the next append quarantines +that tail, and `tools/coverage_vault.py merge --addendum ...` ignores malformed +lines while additively merging every valid record. + +Development configs may also set a project-relative +`overlay_capture_persist_dir`. This creates one atomically published, immutable +JSON snapshot per changed capture. Absolute paths and `..` components are rejected. +Production configs normally omit this key and retain only the naive executable- +local addendum. Both history modes are off by default and contain game-derived +bytes, so their outputs remain private/gitignored artifacts. + ### 1.3 Compile (offline, `tools/compile_overlays.py`) 1. Python does function-boundary discovery from the seeds (`classify_overlay_seeds`, `_walk_overlay_function`) with a strict callable-prologue gate; interior/ diff --git a/recompiler/src/config_loader.cpp b/recompiler/src/config_loader.cpp index bfa105a9..77493ed0 100644 --- a/recompiler/src/config_loader.cpp +++ b/recompiler/src/config_loader.cpp @@ -234,6 +234,25 @@ static RuntimeConfig parse_runtime_block(const toml::value& cfg, const fs::path& if (runtime.contains("overlay_cache")) { rt.overlay_cache = toml::find(runtime, "overlay_cache"); } + if (runtime.contains("overlay_capture_history")) { + rt.overlay_capture_history = + toml::find(runtime, "overlay_capture_history"); + } + if (runtime.contains("overlay_capture_persist_dir")) { + rt.overlay_capture_persist_dir = + toml::find(runtime, "overlay_capture_persist_dir"); + const std::filesystem::path persist(rt.overlay_capture_persist_dir); + if (persist.is_absolute()) { + throw std::runtime_error( + "runtime.overlay_capture_persist_dir must be project-relative"); + } + for (const auto& component : persist) { + if (component == "..") { + throw std::runtime_error( + "runtime.overlay_capture_persist_dir must stay inside the project"); + } + } + } if (runtime.contains("turbo_loads")) { rt.turbo_loads = toml::find(runtime, "turbo_loads"); } diff --git a/recompiler/src/config_loader.h b/recompiler/src/config_loader.h index d90e1017..cdb4e114 100644 --- a/recompiler/src/config_loader.h +++ b/recompiler/src/config_loader.h @@ -138,6 +138,19 @@ struct RuntimeConfig { // overlay bytes to overlay_captures.json for offline compilation. bool overlay_cache = false; + // overlay_capture_history: opt-in durable capture history. The runtime + // keeps overlay_captures.json as an atomic latest snapshot for the live + // compiler and additionally appends every changed coherent snapshot to + // overlay_captures.addendum.jsonl beside the executable. A malformed tail + // from a hard kill cannot destroy earlier records. + bool overlay_capture_history = false; + + // overlay_capture_persist_dir: optional DEV-only, project-relative safe + // directory for one immutable JSON file per changed snapshot. Absolute + // paths and `..` components are rejected. Production normally leaves this + // unset and retains only the addendum beside the executable. + std::string overlay_capture_persist_dir; + // turbo_loads: OPT-IN per game. While the game is loading (CD data // stream active, XA/FMV excluded, post-BIOS-handoff only) the frontend // skips wall-clock pacing so the guest runs at host speed — compressing diff --git a/recompiler/tests/recompiler_patch_test.cpp b/recompiler/tests/recompiler_patch_test.cpp index e1546e89..acfe0f64 100644 --- a/recompiler/tests/recompiler_patch_test.cpp +++ b/recompiler/tests/recompiler_patch_test.cpp @@ -166,6 +166,37 @@ replacement = "0x24020001" "instruction-aligned", "parser rejects unaligned sites"); } +void capture_history_config_tests(const fs::path& root) { + const auto valid = write_config(root, "capture-history", R"toml( +[runtime] +overlay_cache = true +overlay_capture_history = true +overlay_capture_persist_dir = ".aot_capture_history/TEST-00000" +)toml"); + const auto cfg = PSXRecompV4::load_game_config(valid); + check(cfg.runtime.overlay_capture_history, + "parser enables durable overlay capture history"); + check(cfg.runtime.overlay_capture_persist_dir == + ".aot_capture_history/TEST-00000", + "parser preserves project-relative capture history directory"); + + const auto escaping = write_config(root, "capture-history-escape", R"toml( +[runtime] +overlay_capture_persist_dir = "../outside" +)toml"); + check_throws([&] { (void)PSXRecompV4::load_game_config(escaping); }, + "must stay inside the project", + "parser rejects escaping capture history directory"); + + const auto absolute = write_config(root, "capture-history-absolute", R"toml( +[runtime] +overlay_capture_persist_dir = "C:/outside" +)toml"); + check_throws([&] { (void)PSXRecompV4::load_game_config(absolute); }, + "must be project-relative", + "parser rejects absolute capture history directory"); +} + void codegen_tests() { constexpr uint32_t original = 0x24020002u; // addiu v0, zero, 2 constexpr uint32_t replacement = 0x24020001u; // addiu v0, zero, 1 @@ -224,6 +255,7 @@ int main() { try { parser_tests(root); + capture_history_config_tests(root); codegen_tests(); } catch (const std::exception& e) { fmt::print(stderr, "FAIL unexpected exception: {}\n", e.what()); diff --git a/runtime/include/overlay_capture.h b/runtime/include/overlay_capture.h index 240184c7..f2fa580c 100644 --- a/runtime/include/overlay_capture.h +++ b/runtime/include/overlay_capture.h @@ -32,6 +32,13 @@ void overlay_capture_init(const char *out_dir); * overlay_capture_on_dma() and overlay_capture_write_json() are no-ops. */ void overlay_capture_set_enabled(int enabled); +/* Opt-in durable history. When enabled, every changed coherent latest snapshot + * is appended to overlay_captures.addendum.jsonl beside the executable. If + * persist_dir is non-empty, the same snapshot is also copied atomically to one + * new immutable JSON file there. game_id is used only in safe filenames. */ +void overlay_capture_configure_history(int enabled, const char *persist_dir, + const char *game_id); + /* Call from dma.c execute_ch3_cdrom() after every forward CH3 transfer with * load_start < 0x1C0000 and fntrace_is_game_started(). * load_addr is the physical RAM destination, size is byte count, @@ -54,6 +61,8 @@ int overlay_capture_count(void); * when [runtime] overlay_autocompile_cmd is configured. */ void overlay_autocapture_set_enabled(int on); void overlay_autocapture_tick(void); +/* Join any in-flight atomic writer before the final shutdown snapshot. */ +void overlay_autocapture_shutdown(void); void overlay_autocapture_get_status(int *enabled, uint32_t *triggers, uint64_t *last_delta); diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index 68642a3e..3e1eb0bb 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -978,6 +978,7 @@ static void shutdown_runtime(void) { /* (sljit removed 2026-07-15: overlay_compile_worker_stop joined the * off-thread JIT worker here; the worker no longer exists.) */ memcard_flush_all(); + overlay_autocapture_shutdown(); overlay_capture_write_json(); if (sdl_audio_device) { SDL_ClearQueuedAudio(sdl_audio_device); @@ -3241,6 +3242,26 @@ int main(int argc, char** argv) { std::string cache_dir = (exe_dir / "cache").string(); overlay_capture_set_out_dir(exe_dir.string().c_str()); overlay_capture_set_enabled(1); + std::string capture_persist_dir; + if (gc.runtime.overlay_capture_history && + !gc.runtime.overlay_capture_persist_dir.empty()) { + std::filesystem::path persist = + gc.project_root / gc.runtime.overlay_capture_persist_dir; + std::error_code persist_ec; + std::filesystem::create_directories(persist, persist_ec); + if (persist_ec) { + std::fprintf(stderr, + "psxrecomp: cannot create overlay capture history %s: %s\n", + persist.string().c_str(), persist_ec.message().c_str()); + } else { + capture_persist_dir = persist.string(); + } + } + overlay_capture_configure_history( + gc.runtime.overlay_capture_history ? 1 : 0, + capture_persist_dir.empty() ? nullptr : + capture_persist_dir.c_str(), + game_id.c_str()); overlay_loader_init(cache_dir.c_str(), game_id.c_str()); for (uint32_t addr : gc.runtime.overlay_native_block) { overlay_loader_native_block_add(addr); diff --git a/runtime/src/overlay_capture.c b/runtime/src/overlay_capture.c index 1cb51d10..516be488 100644 --- a/runtime/src/overlay_capture.c +++ b/runtime/src/overlay_capture.c @@ -6,7 +6,16 @@ #include #include #include +#include +#include #include +#ifdef _WIN32 +#include +#include +#include +#else +#include +#endif /* ---- Capture set -------------------------------------------------------- * Tracks which physical addresses have received CD DMA loads since game @@ -34,6 +43,68 @@ static int s_count = 0; static char s_out_dir[512]; static int s_active = 0; static int s_enabled = 0; /* config gate; off unless overlay cache enabled */ +static int s_history_enabled = 0; +static char s_history_addendum[600]; +static char s_history_persist_dir[600]; +static char s_history_game_id[64]; +static char s_history_session[64]; +static uint32_t s_history_sequence; +static uint64_t s_history_last_sig; +static int s_history_written; +static SDL_atomic_t s_autocap_write_state; /* 0 idle, 1 writing, 2 complete */ + +static int capture_process_id(void) +{ +#ifdef _WIN32 + return _getpid(); +#else + return (int)getpid(); +#endif +} + +static void sanitize_filename(char *dst, size_t dst_size, const char *src) +{ + size_t out = 0; + if (!dst_size) return; + while (src && *src && out + 1 < dst_size) { + unsigned char c = (unsigned char)*src++; + dst[out++] = (char)(isalnum(c) || c == '-' || c == '_' ? c : '_'); + } + dst[out] = '\0'; +} + +static int sync_file(FILE *f) +{ + if (fflush(f) != 0) return 0; +#ifdef _WIN32 + return _commit(_fileno(f)) == 0; +#else + return fsync(fileno(f)) == 0; +#endif +} + +static int atomic_replace_file(const char *tmp_path, const char *path) +{ +#ifdef _WIN32 + return MoveFileExA(tmp_path, path, + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) != 0; +#else + return rename(tmp_path, path) == 0; +#endif +} + +static uint64_t capture_file_sig(const char *path) +{ + FILE *f = fopen(path, "rb"); + uint64_t h = 1469598103934665603ull; + unsigned char buf[65536]; + size_t n, i; + if (!f) return 0; + while ((n = fread(buf, 1, sizeof(buf), f)) > 0) + for (i = 0; i < n; i++) { h ^= buf[i]; h *= 1099511628211ull; } + fclose(f); + return h; +} /* ---- Base64 encoder ----------------------------------------------------- */ @@ -72,6 +143,44 @@ void overlay_capture_set_enabled(int enabled) s_enabled = enabled ? 1 : 0; } +void overlay_capture_configure_history(int enabled, const char *persist_dir, + const char *game_id) +{ + time_t now; + struct tm tm_now; + char stamp[32]; + s_history_enabled = enabled ? 1 : 0; + s_history_persist_dir[0] = '\0'; + s_history_sequence = 0; + s_history_last_sig = 0; + s_history_written = 0; + sanitize_filename(s_history_game_id, sizeof(s_history_game_id), + game_id && game_id[0] ? game_id : "UNKNOWN"); + if (persist_dir && persist_dir[0]) { + snprintf(s_history_persist_dir, sizeof(s_history_persist_dir), "%s", + persist_dir); + } + snprintf(s_history_addendum, sizeof(s_history_addendum), + "%s/overlay_captures.addendum.jsonl", s_out_dir); + + now = time(NULL); +#ifdef _WIN32 + gmtime_s(&tm_now, &now); +#else + gmtime_r(&now, &tm_now); +#endif + strftime(stamp, sizeof(stamp), "%Y%m%dT%H%M%SZ", &tm_now); + snprintf(s_history_session, sizeof(s_history_session), "%s-p%d", + stamp, capture_process_id()); + if (s_history_enabled) { + fprintf(stdout, + "psxrecomp: durable overlay capture history enabled (%s%s%s)\n", + s_history_addendum, + s_history_persist_dir[0] ? "; immutable dir=" : "", + s_history_persist_dir[0] ? s_history_persist_dir : ""); + } +} + void overlay_capture_on_dma(uint32_t load_addr, uint32_t size, const uint8_t *bytes) { @@ -237,17 +346,17 @@ static void write_json_window(FILE *f, uint32_t win_lo_page, } } -static int write_json_snapshot(const char *path, uint32_t bw, - const uint32_t *bitmap, - const DirtyRamPcEntry *pc_table, - const DirtyRamPcEntry *exec_pc_table, - const uint8_t *ram_base) +static int write_json_snapshot_direct(const char *path, uint32_t bw, + const uint32_t *bitmap, + const DirtyRamPcEntry *pc_table, + const DirtyRamPcEntry *exec_pc_table, + const uint8_t *ram_base) { FILE *f; uint32_t page_sz; int first_region; page_sz = 4096u; - f = fopen(path, "w"); + f = fopen(path, "wb"); if (!f) return 0; fprintf(f, "[\n"); @@ -271,10 +380,152 @@ static int write_json_snapshot(const char *path, uint32_t bw, &first_region, bitmap, pc_table, exec_pc_table, ram_base); fprintf(f, "\n]\n"); - fclose(f); + { + int ok = !ferror(f) && sync_file(f); + if (fclose(f) != 0) ok = 0; + return ok; + } +} + +/* The live compiler needs one canonical latest snapshot, but it must never see + * a half-written JSON file. Format and fsync a same-directory temporary, then + * atomically replace the old snapshot. A hard kill leaves either the old valid + * file or the new valid file (plus a harmless .tmp), never a truncated latest. */ +static int write_json_snapshot_atomic(const char *path, uint32_t bw, + const uint32_t *bitmap, + const DirtyRamPcEntry *pc_table, + const DirtyRamPcEntry *exec_pc_table, + const uint8_t *ram_base) +{ + char tmp_path[700]; + snprintf(tmp_path, sizeof(tmp_path), "%s.tmp-p%d", path, + capture_process_id()); + if (!write_json_snapshot_direct(tmp_path, bw, bitmap, pc_table, + exec_pc_table, ram_base)) { + remove(tmp_path); + return 0; + } + if (!atomic_replace_file(tmp_path, path)) { + remove(tmp_path); + return 0; + } return 1; } +static int copy_file_atomic(const char *src_path, const char *dst_path) +{ + char tmp_path[700]; + unsigned char buf[65536]; + FILE *src, *dst; + size_t n; + int ok = 1; + snprintf(tmp_path, sizeof(tmp_path), "%s.tmp-p%d", dst_path, + capture_process_id()); + src = fopen(src_path, "rb"); + if (!src) return 0; + dst = fopen(tmp_path, "wb"); + if (!dst) { fclose(src); return 0; } + while ((n = fread(buf, 1, sizeof(buf), src)) > 0) { + if (fwrite(buf, 1, n, dst) != n) { ok = 0; break; } + } + if (ferror(src)) ok = 0; + fclose(src); + if (ok) ok = sync_file(dst); + if (fclose(dst) != 0) ok = 0; + if (!ok || !atomic_replace_file(tmp_path, dst_path)) { + remove(tmp_path); + return 0; + } + return 1; +} + +/* Append the canonical JSON array as one minified JSONL record. Whitespace is + * stripped only outside strings, so bytes_b64 remains byte-for-byte intact. + * A crash can damage only the final line; every earlier launch remains a valid + * independently parseable record. */ +static int append_history_record(const char *snapshot_path, const char *reason, + uint64_t sig, uint32_t sequence) +{ + FILE *src = fopen(snapshot_path, "rb"); + FILE *dst, *tail; + int c, in_string = 0, escape = 0, ok = 1; + int needs_separator = 0; + if (!src) return 0; + tail = fopen(s_history_addendum, "rb"); + if (tail) { + if (fseek(tail, -1, SEEK_END) == 0) { + c = fgetc(tail); + needs_separator = c != '\n'; + } + fclose(tail); + } + dst = fopen(s_history_addendum, "ab"); + if (!dst) { fclose(src); return 0; } + /* Quarantine an incomplete final record from a previous hard kill. */ + if (needs_separator && fputc('\n', dst) == EOF) ok = 0; + if (ok && fprintf(dst, + "{\"schema\":\"psxrecomp overlay capture addendum v1\"," + "\"game\":\"%s\",\"session\":\"%s\",\"sequence\":%u," + "\"reason\":\"%s\",\"fnv64\":\"%016llX\",\"captures\":", + s_history_game_id, s_history_session, sequence, + reason ? reason : "snapshot", (unsigned long long)sig) < 0) ok = 0; + while (ok && (c = fgetc(src)) != EOF) { + if (in_string) { + if (fputc(c, dst) == EOF) { ok = 0; break; } + if (escape) escape = 0; + else if (c == '\\') escape = 1; + else if (c == '"') in_string = 0; + } else { + if (c == '"') { + in_string = 1; + if (fputc(c, dst) == EOF) { ok = 0; break; } + } else if (!isspace((unsigned char)c)) { + if (fputc(c, dst) == EOF) { ok = 0; break; } + } + } + } + if (ferror(src) || in_string) ok = 0; + fclose(src); + if (ok && fprintf(dst, "}\n") < 0) ok = 0; + if (ok) ok = sync_file(dst); + if (fclose(dst) != 0) ok = 0; + return ok; +} + +static void persist_history_snapshot(const char *snapshot_path, + const char *reason, uint64_t sig) +{ + char persist_path[800]; + uint32_t sequence; + int addendum_saved; + int persist_saved = s_history_persist_dir[0] ? 0 : 1; + if (!s_history_enabled || !sig) return; + if (s_history_written && sig == s_history_last_sig) return; + sequence = ++s_history_sequence; + + if (s_history_persist_dir[0]) { + snprintf(persist_path, sizeof(persist_path), + "%s/%s_%s_%04u_%016llX.json", + s_history_persist_dir, s_history_game_id, s_history_session, + sequence, (unsigned long long)sig); + if (copy_file_atomic(snapshot_path, persist_path)) persist_saved = 1; + else fprintf(stderr, + "psxrecomp: failed to persist overlay capture snapshot %s\n", + persist_path); + } + addendum_saved = append_history_record(snapshot_path, reason, sig, sequence); + if (!addendum_saved) + fprintf(stderr, "psxrecomp: failed to append overlay capture history %s\n", + s_history_addendum); + /* Retry an unchanged snapshot if either configured durable sink failed. + * Repeated addendum records or immutable files are harmless: vault merges + * are content-keyed and idempotent. */ + if (addendum_saved && persist_saved) { + s_history_last_sig = sig; + s_history_written = 1; + } +} + void overlay_capture_write_json(void) { extern uint8_t *memory_get_ram_ptr(void); @@ -282,12 +533,18 @@ void overlay_capture_write_json(void) uint32_t bw = dirty_ram_get_bitmap_word_count(); uint32_t *bitmap; if (!s_active) return; + /* Manual dumps do not race the periodic worker. Shutdown joins it first. */ + if (SDL_AtomicGet(&s_autocap_write_state) != 0) return; bitmap = (uint32_t *)malloc((size_t)bw * sizeof(uint32_t)); if (!bitmap) return; for (uint32_t i = 0; i < bw; i++) bitmap[i] = dirty_ram_get_bitmap_word(i); snprintf(path, sizeof(path), "%s/overlay_captures.json", s_out_dir); - write_json_snapshot(path, bw, bitmap, g_dirty_ram_pc_table, - g_dirty_ram_exec_pc_table, memory_get_ram_ptr()); + if (write_json_snapshot_atomic(path, bw, bitmap, g_dirty_ram_pc_table, + g_dirty_ram_exec_pc_table, + memory_get_ram_ptr())) { + uint64_t sig = capture_file_sig(path); + persist_history_snapshot(path, "shutdown-or-manual", sig); + } free(bitmap); } @@ -355,9 +612,9 @@ typedef struct { uint32_t *bitmap; uint32_t bitmap_words; uint64_t manifest_sig; + int write_ok; char path[600]; } AutocapWriteJob; -static SDL_atomic_t s_autocap_write_state; /* 0 idle, 1 writing, 2 complete */ static SDL_Thread *s_autocap_write_thread; static AutocapWriteJob *s_autocap_write_job; static int s_autocap_seeded_existing; @@ -401,17 +658,8 @@ void overlay_autocapture_get_futility(uint32_t *backoff, uint32_t *futile) { static uint64_t autocap_manifest_sig(void) { char path[600]; - FILE *f; - uint64_t h = 1469598103934665603ull; - unsigned char buf[65536]; - size_t n, i; snprintf(path, sizeof(path), "%s/overlay_captures.json", s_out_dir); - f = fopen(path, "rb"); - if (!f) return 0; - while ((n = fread(buf, 1, sizeof(buf), f)) > 0) - for (i = 0; i < n; i++) { h ^= buf[i]; h *= 1099511628211ull; } - fclose(f); - return h; + return capture_file_sig(path); } static void autocap_write_job_free(AutocapWriteJob *job) @@ -425,9 +673,14 @@ static int autocap_write_thread_main(void *opaque) { AutocapWriteJob *job = (AutocapWriteJob *)opaque; SDL_SetThreadPriority(SDL_THREAD_PRIORITY_LOW); - if (write_json_snapshot(job->path, job->bitmap_words, job->bitmap, - job->pc_table, job->exec_pc_table, job->ram)) - job->manifest_sig = autocap_manifest_sig(); + job->write_ok = write_json_snapshot_atomic( + job->path, job->bitmap_words, job->bitmap, + job->pc_table, job->exec_pc_table, job->ram); + if (job->write_ok) { + job->manifest_sig = capture_file_sig(job->path); + persist_history_snapshot(job->path, "autocap", + job->manifest_sig); + } SDL_AtomicSet(&s_autocap_write_state, 2); return 0; } @@ -531,6 +784,22 @@ void overlay_autocapture_tick(void) autocap_write_start(); } +void overlay_autocapture_shutdown(void) +{ + int state = SDL_AtomicGet(&s_autocap_write_state); + if (state == 0) return; + if (s_autocap_write_thread) { + SDL_WaitThread(s_autocap_write_thread, NULL); + s_autocap_write_thread = NULL; + } + /* The worker atomically published and durably archived the coherent + * snapshot before setting state=2. No compiler request is needed while + * shutting down; just release its copied inputs before the final snapshot. */ + autocap_write_job_free(s_autocap_write_job); + s_autocap_write_job = NULL; + SDL_AtomicSet(&s_autocap_write_state, 0); +} + uint32_t overlay_capture_get_region_crc(uint32_t region_start, uint32_t region_size) { diff --git a/runtime/tests/test_capture_history.py b/runtime/tests/test_capture_history.py new file mode 100644 index 00000000..4d43578e --- /dev/null +++ b/runtime/tests/test_capture_history.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Regression checks for append-only overlay capture history ingestion.""" +import importlib.util +import json +import pathlib +import tempfile + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "coverage_vault", ROOT / "tools" / "coverage_vault.py") +MOD = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MOD) + + +def region(addr, payload, pcs): + return { + "schema": "psxrecomp overlay capture v2", + "load_addr": addr, + "size": 4, + "bytes_b64": payload, + "executed_pcs": pcs, + "dispatch_entry_pcs": pcs, + "function_entry_pcs": [], + "seeds": pcs, + } + + +def main(): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + addendum = root / "overlay_captures.addendum.jsonl" + vault = root / "vault" / "overlay_captures.json" + first = region("0x80100000", "AAAAAA==", ["0x80100000"]) + second = region("0x80100000", "AQAAAA==", ["0x80100004"]) + records = [ + {"schema": "psxrecomp overlay capture addendum v1", + "captures": [first]}, + {"schema": "psxrecomp overlay capture addendum v1", + "captures": [first, second]}, + ] + with addendum.open("w", encoding="utf-8", newline="\n") as out: + out.write(json.dumps(records[0], separators=(",", ":")) + "\n") + # A later launch first quarantines a hard-killed partial record. + out.write('{"schema":"torn-tail"\n') + out.write(json.dumps({"schema": "unrelated", "captures": [second]}, + separators=(",", ":")) + "\n") + out.write(json.dumps(records[1], separators=(",", ":")) + "\n") + + new_variants, new_pcs = MOD.merge_addendum(str(vault), str(addendum)) + assert new_variants == 2 + assert new_pcs == 2 + merged = json.loads(vault.read_text(encoding="utf-8")) + assert len(merged) == 2 + assert not list(vault.parent.glob("*.tmp-*")) + + # Re-ingesting the same durable history is additive and idempotent. + assert MOD.merge_addendum(str(vault), str(addendum)) == (0, 0) + print("ALL PASS") + + +if __name__ == "__main__": + main() diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 296aded4..65aa210c 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -43,6 +43,24 @@ A mis-positioned / data-as-code shard either fails audit at compile time or neve fires at runtime → coverage loss, never incorrect execution. Whatever static misses, production autocompile (tcc/gcc) self-heals on first visit. +## Durable runtime capture history + +The canonical `overlay_captures.json` is atomically replaced for live consumers. +With `[runtime] overlay_capture_history = true`, each changed coherent snapshot is +also appended as one independent record to +`overlay_captures.addendum.jsonl` beside the executable. A torn final line does +not invalidate earlier or later records. Merge it into the private additive vault +with: + +```sh +python tools/coverage_vault.py merge --vault \ + --addendum /overlay_captures.addendum.jsonl +``` + +Dev configs can additionally set the project-relative +`overlay_capture_persist_dir` to retain immutable per-snapshot JSON files. Both +settings are opt-in; capture artifacts contain game bytes and must stay ignored. + ## Multi-game sweep findings (2026-07-17) `extract_generic.py` runs the improved extraction from a game.toml. Results: diff --git a/tools/coverage_vault.py b/tools/coverage_vault.py index 929a8c04..22c9bda6 100644 --- a/tools/coverage_vault.py +++ b/tools/coverage_vault.py @@ -19,7 +19,8 @@ It contains game-derived bytes, so the vault dir must stay gitignored / private and is committed. Usage: - coverage_vault.py merge --vault DIR [--captures captures.json] [--cache CACHE_DIR] + coverage_vault.py merge --vault DIR [--captures captures.json] + [--addendum captures.addendum.jsonl] [--cache CACHE_DIR] coverage_vault.py stats --vault DIR """ import argparse, json, os, shutil, hashlib, sys @@ -35,16 +36,58 @@ def _load_list(path): if not os.path.exists(path): return [] try: - v = json.load(open(path, encoding="utf-8")) - return v if isinstance(v, list) else [] + with open(path, encoding="utf-8") as source: + value = json.load(source) + return value if isinstance(value, list) else [] except Exception as e: print(" warn: could not read %s (%s); starting fresh" % (path, e)) return [] -def merge_captures(vault_json, src_json): - if not src_json or not os.path.exists(src_json): - return 0, 0 - src = _load_list(src_json) +def _load_addendum(path): + """Load every valid append-only history record, ignoring a torn tail. + + Each line is an independent snapshot wrapper. A hard kill may truncate only + the final line; earlier launches remain usable and a later runtime append + quarantines the bad tail with a newline. + """ + regions = [] + if not path or not os.path.exists(path): + return regions + with open(path, encoding="utf-8", errors="replace") as history: + for lineno, line in enumerate(history, 1): + if not line.strip(): + continue + try: + record = json.loads(line) + except Exception as e: + print(" warn: ignoring invalid addendum line %d in %s (%s)" % + (lineno, path, e)) + continue + if not isinstance(record, dict) or record.get("schema") != ( + "psxrecomp overlay capture addendum v1"): + print(" warn: ignoring unknown addendum record on line %d in %s" % + (lineno, path)) + continue + captures = record.get("captures", []) + if isinstance(captures, list): + regions.extend(r for r in captures if isinstance(r, dict)) + return regions + +def _write_list_atomic(path, value): + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + tmp = "%s.tmp-%d" % (path, os.getpid()) + try: + with open(tmp, "w", encoding="utf-8", newline="\n") as out: + json.dump(value, out, indent=1) + out.write("\n") + out.flush() + os.fsync(out.fileno()) + os.replace(tmp, path) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + +def merge_capture_regions(vault_json, src): if not src: return 0, 0 index = { _variant_key(r): r for r in _load_list(vault_json) } @@ -63,10 +106,17 @@ def merge_captures(vault_json, src_json): if fld == "executed_pcs": new_pcs += len(add - cur) tgt[fld] = sorted(cur | add) - os.makedirs(os.path.dirname(vault_json) or ".", exist_ok=True) - json.dump(list(index.values()), open(vault_json, "w", encoding="utf-8"), indent=1) + _write_list_atomic(vault_json, list(index.values())) return new_variants, new_pcs +def merge_captures(vault_json, src_json): + if not src_json or not os.path.exists(src_json): + return 0, 0 + return merge_capture_regions(vault_json, _load_list(src_json)) + +def merge_addendum(vault_json, addendum): + return merge_capture_regions(vault_json, _load_addendum(addendum)) + def merge_cache(vault_cache, src_cache): """Mirror compiled DLLs/.ranges into the vault, PRESERVING the relative layout (//cg_/file). The layout is load- @@ -111,6 +161,7 @@ def main(): ap.add_argument("cmd", choices=["merge", "stats"]) ap.add_argument("--vault", required=True, help="vault directory (kept gitignored/private)") ap.add_argument("--captures", help="source overlay_captures.json to merge in") + ap.add_argument("--addendum", help="append-only overlay capture history (.jsonl)") ap.add_argument("--cache", help="source cache dir (e.g. build/cache/) to merge in") a = ap.parse_args() if a.cmd == "stats": @@ -118,7 +169,13 @@ def main(): return 0 vj = os.path.join(a.vault, CAP_NAME) vc = os.path.join(a.vault, CACHE_SUB) - nv, np_ = merge_captures(vj, a.captures) if a.captures else (0, 0) + nv = np_ = 0 + if a.captures: + add_v, add_p = merge_captures(vj, a.captures) + nv += add_v; np_ += add_p + if a.addendum: + add_v, add_p = merge_addendum(vj, a.addendum) + nv += add_v; np_ += add_p nd = merge_cache(vc, a.cache) if a.cache else 0 print("coverage_vault: +%d new variant(s), +%d new PC(s), +%d new DLL(s) -> %s" % (nv, np_, nd, a.vault)) return 0 From bc24934048ff0e846941dc8000ce9e4bcb0b3b19 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 00:38:26 -0700 Subject: [PATCH 05/37] aot: recover raw and adjacent overlay producers --- docs/AOT_OVERLAY_PLAN.md | 7 ++ tools/aot_overlay_spike/README.md | 28 ++++++- tools/aot_overlay_spike/coverage_report.py | 34 ++++++++ tools/aot_overlay_spike/extract_generic.py | 96 ++++++++++++++++++++-- 4 files changed, 157 insertions(+), 8 deletions(-) diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index 3eb87c84..da1c080d 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -134,6 +134,13 @@ Production configs normally omit this key and retain only the naive executable- local addendum. Both history modes are off by default and contain game-derived bytes, so their outputs remain private/gitignored artifacts. +The recall scoreboard can roll a persisted live gap set forward after a verified +monotonic static expansion with +`coverage_report.py --prior-report --assume-static-superset`. The +explicit assertion is required because an old gap +manifest stores misses, not every previously covered entry; silently assuming +monotonicity would make a regressed static set look healthier than it is. + ### 1.3 Compile (offline, `tools/compile_overlays.py`) 1. Python does function-boundary discovery from the seeds (`classify_overlay_seeds`, `_walk_overlay_function`) with a strict callable-prologue gate; interior/ diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 65aa210c..e6e0d8fc 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -28,10 +28,12 @@ discovery quality, not to be run as-is in a build. - **Tomba 1:** 90% of played functions reproduced play-free + ~1964 unplayed extra; live-validated in-game (Dwarf Village went native, no runtime compile). -- **Tomba 2:** the clean play-free cache now covers 1219/1856 played entries - (**65.7%**) and byte-matches 1102 (**59.4% entry+code_crc**). Direct-call +- **Tomba 2:** the clean play-free cache now covers 1264/1856 played entries + (**68.1%**) and byte-matches 1144 (**61.6% entry+code_crc**). Direct-call provenance for frameless leaves added 74 MAIN.EXE hits over the 60.0% baseline; - recovering the four call-sparse files added another 32 overlay-window hits. + recovering the four call-sparse files added another 32 overlay-window hits; + strict raw-code recovery of CRD/SOP added 43 more played hits, and the first + audited adjacent-envelope shard added 2 more. All 22 A*.BIN overlays converge on 0x80108F9C (region 0x80108000 +3996), DEMO/GAME converge on 0x80106228, and OPN resolves to 0x8018A000. @@ -100,6 +102,26 @@ settings are opt-in; capture artifacts contain game bytes and must stay ignored. - Bugs fixed by the sweep: multi-.bin cue (data-track selection), base-EXE exclusion, game.toml UTF-8 BOM, normal-mode .ranges format (`F ` w/o crc). +## Raw-code and adjacent-envelope recovery (2026-07-18) + +Not every position-fixed code file has a pure `{count, ptr[]}` export header. +`recover_raw_base()` applies a stricter unbounded jal/prologue vote: score >= 8, +at least 2x the runner-up, and at least 25% of apparent prologues agreeing on one +RAM-fitting base. This recovers Tomba 2 `CRD.BIN` at `0x8018A000` (32:3) and +`SOP.BIN` at `0x80108F9C` (12:3). Both are byte-identical to multiple played-vault +images; their two shards compile with zero unsupported instructions or bad +targets. It also finds Tomba 1 `OPTSUB00.BIN` (exact played-vault match) and +`DSPSUB2.BIN` (99.96% live-byte match, consistent with runtime fixups), without +touching the Tomba 1 cache. + +Runtime dirty-page capture may coalesce directly adjacent files under the earlier +page key. The extractor therefore also emits exact two-producer concatenations. +For Tomba 2, `GAME.BIN` ends exactly where every A*/SOP file begins, producing +`0x80106000` envelope variants in addition to the standalone `0x80108000` shards. +The A01 composite—the live attract scene's measured hot variant—audits cleanly +(459 functions, zero unsupported/bad targets). Its performance recheck remains +pending because an unrelated eight-core build prevented a valid BelowNormal soak. + ## Next to raise coverage (future session) 1. ~~Direct-call frameless-leaf discovery~~ — DONE. Reachable `jal` targets are diff --git a/tools/aot_overlay_spike/coverage_report.py b/tools/aot_overlay_spike/coverage_report.py index 6a25b878..9daaec88 100644 --- a/tools/aot_overlay_spike/coverage_report.py +++ b/tools/aot_overlay_spike/coverage_report.py @@ -19,6 +19,7 @@ reported at two strengths: Usage: coverage_report.py --static --vault [--captures ] + [--prior-report --assume-static-superset] --out-md --out-json [--game ] Cache dirs are scanned recursively for *.ranges; a captures json is the v2 schema. """ @@ -79,11 +80,17 @@ def main(): ap.add_argument('--static', required=True, help='static shard cache dir (.ranges)') ap.add_argument('--vault', help='played/coverage vault cache dir (.ranges)') ap.add_argument('--captures', help='runtime overlay_captures.json (executed_pcs)') + ap.add_argument('--prior-report', help='roll forward a persisted live gap set') + ap.add_argument('--assume-static-superset', action='store_true', + help='assert current static entries retain every prior static entry') ap.add_argument('--game', default='UNKNOWN') ap.add_argument('--out-md', required=True) ap.add_argument('--out-json', required=True) a = ap.parse_args() + if a.prior_report and not a.assume_static_superset: + ap.error('--prior-report requires the explicit --assume-static-superset assertion') + static_ent = parse_ranges_dir(a.static) report = {'game': a.game, 'static_entries': len(static_ent), 'sources': {}} @@ -127,6 +134,33 @@ def main(): md.append(f'- Discovered by static: **{r["covered_here"]}** ' f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') md.append(f'- **MISSED live: {r["missed"]}**\n') + elif a.prior_report: + with open(a.prior_report, encoding='utf-8') as prior_in: + prior = json.load(prior_in) + prior_live = prior.get('sources', {}).get('live_session') + prior_misses = prior.get('live_misses') + if not isinstance(prior_live, dict) or not isinstance(prior_misses, list): + ap.error('--prior-report has no persisted live_session/live_misses') + if len(static_ent) < int(prior.get('static_entries', 0)): + ap.error('current static entry count is smaller than the prior report; ' + 'the asserted superset cannot hold') + needed = int(prior_live.get('needed', 0)) + old_miss = {norm(int(x, 16)) for x in prior_misses} + miss = old_miss - set(static_ent) + covered = needed - len(miss) + r = {'needed': needed, 'covered_here': covered, 'missed': len(miss), + 'recall_entry': (covered/needed if needed else 0.0)} + r['provenance'] = ('rolled forward from the persisted live gap manifest; ' + 'caller asserted the current static entry set is a superset') + report['sources']['live_session'] = r + report['live_misses'] = [f'0x{x|0x80000000:08X}' for x in sorted(miss)] + md.append('## vs persisted live-session gaps (monotonic roll-forward)\n') + md.append(f'- Functions the persisted session exercised: **{needed}**') + md.append(f'- Discovered by current static: **{covered}** ' + f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') + md.append(f'- **MISSED live: {len(miss)}**') + md.append('- Provenance: caller explicitly asserted that the current static ' + 'entry set retains every prior static entry.\n') for output_path in (a.out_md, a.out_json): output_dir = os.path.dirname(os.path.abspath(output_path)) diff --git a/tools/aot_overlay_spike/extract_generic.py b/tools/aot_overlay_spike/extract_generic.py index 870b15a9..ab6a5fc0 100644 --- a/tools/aot_overlay_spike/extract_generic.py +++ b/tools/aot_overlay_spike/extract_generic.py @@ -130,6 +130,40 @@ def recover_base(data, ptrs, trusted_bases=()): if not hist: return None return _select_base_vote(hist, trusted_bases) +def recover_raw_base(data, base_lo, base_hi=0x80200000): + """Recover a position-fixed raw-code blob without an export-pointer table. + + This uses the same base-independent jal-target/prologue self-consistency as + recover_base(), but has no pointer-containment window. Compensate with a + deliberately stricter proof: at least eight distinct call targets, a 2x + margin over the runner-up, and at least 25% of all apparent prologues must + agree on one RAM-fitting base. This finds call-dense raw siblings such as + Tomba 2 CRD.BIN/SOP.BIN while rejecting ordinary data files. A false positive + remains fail-safe at dispatch because every compiled function is code-CRC + guarded, but the strict gate also avoids wasting shards on data-as-code. + """ + import bisect + from collections import Counter + P=prologue_offsets(data) + T=jal_targets(data) + if not P or not T: return None + hist=Counter() + max_base=base_hi-len(data) + for t in T: + lo_i=bisect.bisect_left(P,t-max_base) + hi_i=bisect.bisect_right(P,t-base_lo) + for o in P[lo_i:hi_i]: + base=t-o + if (base&3)==0: + hist[base]+=1 + if not hist: return None + top=hist.most_common(2) + base,score=top[0] + second=top[1][1] if len(top)>1 else 0 + if score < 8 or score < second*2 or score*4 < len(P): + return None + return base,score,second + def is_psx_exe(data): return len(data) >= 0x20 and data[:8] == b'PS-X EXE' @@ -200,7 +234,8 @@ def main(): os.makedirs(tmp, exist_ok=True) # The boot EXE (game.toml `exe`) is the STATIC-recompiled base, NOT an overlay. exe_base=os.path.basename(str(game.get('exe',''))).upper() - records=[]; np=nh=0; seen_crc=set(); header_files=[] + records=[]; np=nh=nr=nc=0; seen_crc=set(); header_files=[]; raw_files=[] + positioned=[] for p,l,s in sorted(files): if s<8 or s>2_000_000: continue if exe_base and os.path.basename(p).upper()==exe_base: @@ -231,9 +266,11 @@ def main(): print(f" [PS-X EXE] {p}: {len(body)}B @0x{0x80000000|base:08X}, full-disc seeds={len(seeds_all)}") else: cnt=is_header_table(data) - if not cnt: continue - ptrs=struct.unpack_from(f'<{cnt}I',data,4) - header_files.append((p, data, ptrs)) + if cnt: + ptrs=struct.unpack_from(f'<{cnt}I',data,4) + header_files.append((p, data, ptrs)) + else: + raw_files.append((p, data)) # Resolve call-dense files first. A small sibling that does not clear the # peak-margin test may then reuse one exact base independently proved by a @@ -277,13 +314,62 @@ def main(): fill = base - page_base region = b'\x00'*fill + data records.append(rec(page_base, region, seeds, dispatch_extra=hdr_entries)) + positioned.append((p,data,base,seeds,hdr_entries)) nh+=1 extra=len(set(hdr_entries)-set(seeds)) print(f" [header-table] {p}: {len(data)}B file@0x{base:08X} " f"region@0x{page_base:08X}(+{fill}) " f"({fit_method} score={score}), prologue seeds={len(seeds)} " f"+{extra} export-table dispatch entries") + + # Some engines keep call-dense position-fixed code in raw siblings whose + # leading table mixes pointers with strings/offsets, so the strict + # {count,ptr[]} recognizer correctly refuses to call it an export table. + # Recover only raw blobs with a much stronger unbounded jal-fit proof. + raw_base_lo=0x80000000|floor_page + for p,data in raw_files: + rb=recover_raw_base(data, raw_base_lo) + if rb is None: continue + base,score,second=rb + seeds=prologues(data,base) + page_base=base&~0xFFF + fill=base-page_base + region=b'\x00'*fill+data + records.append(rec(page_base,region,seeds)) + positioned.append((p,data,base,seeds,())) + nr+=1 + print(f" [raw-code] {p}: {len(data)}B file@0x{base:08X} " + f"region@0x{page_base:08X}(+{fill}) " + f"(strict jal-fit score={score}, runner-up={second}), " + f"prologue seeds={len(seeds)}") + + # Runtime dirty-page capture coalesces directly adjacent producers into one + # region keyed by the earlier page. Emit each provable two-file composition + # as an additional variant so the later file's functions are registered + # under that live region key. The bytes are an exact disc concatenation and + # per-function code CRCs still guard every dispatch. Tomba 2's GAME.BIN ends + # exactly at the common A*/SOP base; standalone records remain necessary for + # sessions where either producer appears under its own page key. + composite_keys=set() + for lp,ldata,lbase,lseeds,ldispatch in positioned: + for rp,rdata,rbase,rseeds,rdispatch in positioned: + if lbase >= rbase or lbase+len(ldata) != rbase: + continue + page_base=lbase&~0xFFF + region=b'\x00'*(lbase-page_base)+ldata+rdata + key=(page_base,binascii.crc32(region)&0xFFFFFFFF) + if key in composite_keys: continue + composite_keys.add(key) + seeds=sorted(set(lseeds)|set(rseeds)) + dispatch=sorted(set(ldispatch)|set(rdispatch)) + records.append(rec(page_base,region,seeds,dispatch_extra=dispatch)) + nc+=1 + print(f" [composite] {lp} + {rp}: {len(region)}B " + f"region@0x{page_base:08X} (exact adjacent producers), " + f"seeds={len(seeds)}") json.dump(records, open(a.out,'w')) - print(f"producers: {np} PS-X EXE (full-discovery), {nh} header-table; {len(records)} regions -> {a.out}") + print(f"producers: {np} PS-X EXE (full-discovery), {nh} header-table, " + f"{nr} raw-code, {nc} adjacent composites; " + f"{len(records)} regions -> {a.out}") if __name__=='__main__': main() From 0874d7b914217dd1ff31ed7451af1edacf93c2a5 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 01:17:27 -0700 Subject: [PATCH 06/37] recompiler: retain padded frameless overlay entries --- recompiler/src/main_psx.cpp | 14 +++++ .../tests/test_aot_overlay_discovery.py | 52 +++++++++++++++++++ .../tests/test_overlay_guard_codegen.py | 12 +++-- .../tests/test_reachable_discovery_codegen.py | 13 +++-- 4 files changed, 84 insertions(+), 7 deletions(-) diff --git a/recompiler/src/main_psx.cpp b/recompiler/src/main_psx.cpp index c2459eca..68268caa 100644 --- a/recompiler/src/main_psx.cpp +++ b/recompiler/src/main_psx.cpp @@ -496,6 +496,20 @@ int main(int argc, char** argv) { if (!PSXRecomp::FunctionAnalyzer::is_valid_mips_word(w)) return false; if (a == exe_lo) return true; if (a >= exe_lo + 8 && read_w(a - 8) == 0x03E00008u) return true; + // Normal-mode discovery already skips alignment padding after a + // return when it infers a frameless function start. Preserve that + // evidence here. Requiring the next entry to be exactly jr-ra+8 + // demoted legitimate Psy-Q leaf routines separated by one to six + // padding NOPs to orphan interiors, so exact-entry analysis emitted + // no function at all. The delay slot (jr+4) may be non-NOP; only + // words after it and before the proposed entry must be padding. + bool padding_only = true; + for (uint32_t back = 12; back <= 32 && a >= exe_lo + back; + back += 4) { + if (read_w(a - back + 8) != 0u) padding_only = false; + if (!padding_only) break; + if (read_w(a - back) == 0x03E00008u) return true; + } int32_t frame = 0; return PSXRecomp::FunctionAnalyzer::is_prologue(w, frame) && !(a >= exe_lo + 4 && diff --git a/recompiler/tests/test_aot_overlay_discovery.py b/recompiler/tests/test_aot_overlay_discovery.py index fb3d253a..df679e72 100644 --- a/recompiler/tests/test_aot_overlay_discovery.py +++ b/recompiler/tests/test_aot_overlay_discovery.py @@ -1,9 +1,13 @@ #!/usr/bin/env python3 """Focused regression checks for play-free overlay call-target discovery.""" +import argparse import importlib.util +import os import pathlib import struct +import subprocess import sys +import tempfile ROOT = pathlib.Path(__file__).resolve().parents[2] @@ -24,7 +28,53 @@ def jal(target): return 0x0C000000 | ((target >> 2) & 0x03FFFFFF) +def make_psxexe(entry, data): + header = bytearray(2048) + header[0:8] = b"PS-X EXE" + put(header, 0x10, entry) + put(header, 0x18, LOAD) + put(header, 0x1C, len(data)) + return bytes(header) + data + + +def check_padded_return_boundary(recompiler): + # Exact-entry mode re-verifies every untrusted seed. Psy-Q aligns frameless + # leaves with NOPs after the prior function's JR delay slot; those NOPs must + # not erase the otherwise exact return-boundary evidence. + data = bytearray(0x80) + put(data, 0x00, 0x03E00008) # jr $ra + put(data, 0x04, 0x24020007) # non-NOP delay slot + # 0x08..0x10 are alignment NOPs. + put(data, 0x14, 0x24020001) # padded frameless leaf + put(data, 0x18, 0x03E00008) + put(data, 0x1C, 0x00000000) + + with tempfile.TemporaryDirectory() as tmp: + psx = os.path.join(tmp, "padded.psx") + seeds = os.path.join(tmp, "seeds.txt") + out = os.path.join(tmp, "out") + with open(psx, "wb") as f: + f.write(make_psxexe(LOAD, data)) + with open(seeds, "w", encoding="utf-8") as f: + f.write(f"0x{LOAD:08X}\n0x{LOAD + 0x14:08X}\n") + result = subprocess.run( + [recompiler, psx, "--seeds", seeds, "--out-dir", out, "--overlay"], + capture_output=True, text=True) + assert result.returncode == 0, result.stderr or result.stdout + full = next(pathlib.Path(out).glob("*_full.c")).read_text() + assert f"void func_{LOAD + 0x14:08X}" in full + ranges = next(pathlib.Path(out).glob("*_full.ranges")).read_text() + assert f"F {LOAD + 0x14:08X}" in ranges + + def main(): + default_recompiler = ROOT / "recompiler" / "build" / "psxrecomp-game.exe" + parser = argparse.ArgumentParser() + parser.add_argument("--recompiler", default=str(default_recompiler)) + args = parser.parse_args() + if not os.path.isfile(args.recompiler): + raise SystemExit(f"recompiler not found: {args.recompiler}") + data = bytearray(0x200) # A framed root directly calls a frameless leaf. JAL is the boundary @@ -63,6 +113,8 @@ def main(): bytes(data), LOAD, len(data), LOAD + 0x100, LOAD + len(data)) assert walk["static_indirect_targets"] == {LOAD + 0x40} + check_padded_return_boundary(args.recompiler) + print("ALL PASS") diff --git a/recompiler/tests/test_overlay_guard_codegen.py b/recompiler/tests/test_overlay_guard_codegen.py index b3c04ca2..3f66119b 100644 --- a/recompiler/tests/test_overlay_guard_codegen.py +++ b/recompiler/tests/test_overlay_guard_codegen.py @@ -69,11 +69,15 @@ def gen_c(recompiler, overlay, tmp): if r.returncode != 0: raise SystemExit(f"recompiler failed ({'overlay' if overlay else 'static'}):\n" f"{r.stderr or r.stdout}") - full = [f for f in os.listdir(out) if f.endswith("_full.c")] + full = [f for f in os.listdir(out) + if "_full" in f and f.endswith(".c") and "_dispatch" not in f] if not full: - raise SystemExit(f"no _full.c emitted in {out}") - with open(os.path.join(out, full[0])) as f: - return f.read() + raise SystemExit(f"no _full*.c emitted in {out}") + chunks = [] + for name in sorted(full): + with open(os.path.join(out, name)) as f: + chunks.append(f.read()) + return "\n".join(chunks) def main(): diff --git a/recompiler/tests/test_reachable_discovery_codegen.py b/recompiler/tests/test_reachable_discovery_codegen.py index e446a7cc..57bee453 100644 --- a/recompiler/tests/test_reachable_discovery_codegen.py +++ b/recompiler/tests/test_reachable_discovery_codegen.py @@ -6,6 +6,7 @@ Usage: python test_reachable_discovery_codegen.py \ """ import argparse import os +import pathlib import struct import subprocess import sys @@ -94,11 +95,17 @@ def main(): if result.returncode != 0: failures.append(f"reachable recompile failed:\n{output}") else: - full_path = os.path.join(tmp, "generated", "test.exe_full.c") + full_paths = sorted(path for path in + pathlib.Path(tmp, "generated").glob("test.exe_full*.c") + if "_dispatch" not in path.name) dispatch_path = os.path.join( tmp, "generated", "test.exe_dispatch.c") - with open(full_path, encoding="utf-8") as f: - full = f.read() + if not full_paths: + failures.append("reachable recompile emitted no _full*.c shards") + full = "" + else: + full = "\n".join(path.read_text(encoding="utf-8") + for path in full_paths) with open(dispatch_path, encoding="utf-8") as f: dispatch = f.read() From 88e471274f84e39de74d5b704bf6f91acd693927 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 01:18:48 -0700 Subject: [PATCH 07/37] aot: separate code-range recall from entry parity --- docs/AOT_OVERLAY_PLAN.md | 21 ++-- tools/aot_overlay_spike/README.md | 23 +++-- tools/aot_overlay_spike/coverage_report.py | 111 +++++++++++++++------ 3 files changed, 113 insertions(+), 42 deletions(-) diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index da1c080d..a7fe96b7 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -5,14 +5,19 @@ Author: investigation via 4 parallel agents + adversarial source verification. Scope: can we move overlay sharding from *runtime discovery* to *build-time (AOT)*, starting with Tomba (SCUS-94236)? -Current Tomba 2 checkpoint: generic play-free extraction emits 27 clean regions; -all 27 compile under `cg5_d0a05489`. Direct-call frameless-leaf discovery plus -call-only/cross-file base recovery raised full-playthrough-vault entry recall from -1113/1856 (60.0%) to 1219/1856 (65.7%), and entry+code_crc recall from 1012 to -1102. The gains are 74 MAIN.EXE entries plus 32 overlay-window entries. The -remaining manifest has 637 misses: 268 MAIN.EXE, 324 overlay-window, and 45 -kernel/low-memory entries. Runtime static-only smoke loaded the matching tag with -`stale_blocked=0`; misses continue to fail safely to the interpreter via CRC guards. +Current Tomba 2 checkpoint: generic play-free extraction emits 29 base/raw regions +plus exact adjacent-producer composites (52 capture records total). The 29 base/raw +shards and the live A01 composite audit with zero unsupported instructions or bad +targets under `cg5_d0a05489`. Direct/constant-register call discovery, raw/cross-file +base recovery, and padded-return frameless-boundary retention raised full-playthrough +vault exact-entry recall from 1113/1856 (60.0%) to 1270/1856 (68.4%); entry+code_crc +is 1150/1856 (62.0%). Exact entry parity is intentionally no longer treated as the +uncovered-code count: the runtime vault contains fine-grained fragments, including +consecutive instruction entries, while static AOT emits broad functions. Compiled +code-range recall is 1811/1856 (97.6%), leaving exactly 45 true gaps, all in the +game-independent kernel/low-memory region. Runtime static-only validation loaded +the richer MAIN shard at 60 fps with `stale_blocked=0`, no range/manifest overflow, +and byte-matching candidates; misses continue to fail safely to the interpreter. --- diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index e6e0d8fc..5a56f6b0 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -120,21 +120,32 @@ For Tomba 2, `GAME.BIN` ends exactly where every A*/SOP file begins, producing `0x80106000` envelope variants in addition to the standalone `0x80108000` shards. The A01 composite—the live attract scene's measured hot variant—audits cleanly (459 functions, zero unsupported/bad targets). Its performance recheck remains -pending because an unrelated eight-core build prevented a valid BelowNormal soak. +pending; the earlier attempt overlapped an unrelated build and was not a valid +BelowNormal measurement. + +`coverage_report.py` reports both exact manifest-entry recall and compiled +code-range recall. The distinction matters: runtime/autocompile vaults may contain +one-entry fragments at consecutive instructions, while static AOT emits one broad +function with the same PCs in its guarded `R` ranges. Exact-entry parity is retained +for continuity; code-range recall is the useful uncovered-code roadmap. At the +current Tomba 2 checkpoint these are 1270/1856 (68.4%) exact entries and 1811/1856 +(97.6%) code-range coverage. All 45 remaining code-range gaps are in low/kernel RAM. ## Next to raise coverage (future session) 1. ~~Direct-call frameless-leaf discovery~~ — DONE. Reachable `jal` targets are emitted as trusted `call_root` seeds; same-basic-block constant-register - `jalr`/tail-`jr` targets are resolved with clobber checks. Tomba 2 vault recall - rose 60.0% -> 64.0% (+74 MAIN.EXE entries), with 23/23 shards compiling cleanly. - Continue with statically proven function-pointer/jump-table consumers; the - first corpus sweep correctly rejected apparent constants clobbered by `lw`. + `jalr`/tail-`jr` targets are resolved with clobber checks. Exact-entry recall + first rose by 74 MAIN.EXE entries. Exact-entry boundary verification now also + retains normal-mode frameless leaves aligned by NOPs after a preceding return; + this added 13 clean MAIN definitions and closed every remaining MAIN code-range + gap. The generated-C audit remains zero unsupported / zero bad targets. 2. ~~Header-table base recovery~~ — DONE (jal-fit, above). 3. ~~Recover weak-signal A09/A0J/GAME/OPN~~ — DONE. They are ordinary uncompressed MIPS, not archives. Counting only callable `jal` targets sharply resolves A09/A0J/OPN; GAME's 8-byte near-tie resolves through the exact base independently proved by DEMO. The played vault byte-proves GAME at 0x80106228. Ambiguous files without one unique trusted match still fail closed. -4. Extract the kernel `0x80000000` region once from the BIOS (game-independent). +4. Extract the kernel `0x80000000` region once from the BIOS (game-independent): + this is now the complete 45-PC Tomba 2 code-range gap set. 5. Generalize into a real `tools/` producer (no hardcoded paths; disc + game.toml in). diff --git a/tools/aot_overlay_spike/coverage_report.py b/tools/aot_overlay_spike/coverage_report.py index 9daaec88..982f25e5 100644 --- a/tools/aot_overlay_spike/coverage_report.py +++ b/tools/aot_overlay_spike/coverage_report.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """AOT static-coverage recall report + gap manifest. -Answers: "of every overlay function the game ACTUALLY runs, what fraction did our -play-free static extractor discover?" — and persists the ones it MISSED so we -have a durable reference and a scoreboard for the discovery tooling. +Answers both "which played manifest entries did static extraction reproduce?" +and "which played entry PCs lie in compiled static code ranges?" The latter is +the effective code-coverage view when the played vault contains fine-grained +runtime fragments but static extraction emits broader functions. Two ground-truth "needed" sources: * a played/coverage VAULT cache dir (.ranges from a full playthrough) — the most @@ -11,9 +12,9 @@ Two ground-truth "needed" sources: * a runtime CAPTURES json (executed_pcs from a live session) — what a specific drive exercised; a live spot-check. -"Covered" = the entries our static shard cache (.ranges) defines. Recall is -reported at two strengths: +Recall is reported at three strengths: * entry-level : same function ENTRY address discovered (did we find it?) + * code-range : played entry PC lies in an R range compiled by static AOT * entry+code_crc : same entry AND identical bytes (is our shard the real fn?) Usage: @@ -23,7 +24,7 @@ Usage: --out-md --out-json [--game ] Cache dirs are scanned recursively for *.ranges; a captures json is the v2 schema. """ -import argparse, os, sys, json, re, glob +import argparse, bisect, os, sys, json, re, glob from collections import defaultdict MASK = 0x1FFFFFFF @@ -35,14 +36,26 @@ def region_of(a): return norm(a) & 0xFFFFF000 def parse_ranges_dir(d): - """-> dict entry_masked -> code_crc (last wins); also the raw (entry,crc) set.""" + """Return ({entry_masked: code_crc}, merged compiled-code intervals).""" ent = {} + intervals = [] for rf in glob.glob(os.path.join(d, '**', '*.ranges'), recursive=True): for line in open(rf, errors='ignore'): m = re.match(r'F\s+([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+)', line) if m: ent[norm(int(m.group(1), 16))] = int(m.group(2), 16) - return ent + continue + m = re.match(r'R\s+([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+)', line) + if m: + lo = norm(int(m.group(1), 16)) + intervals.append((lo, lo + int(m.group(2), 16))) + merged = [] + for lo, hi in sorted(intervals): + if not merged or lo > merged[-1][1]: + merged.append([lo, hi]) + elif hi > merged[-1][1]: + merged[-1][1] = hi + return ent, [(lo, hi) for lo, hi in merged] def parse_captures_executed(path): """-> set of FUNCTION-ENTRY addrs (masked) the runtime dispatched to. @@ -56,14 +69,24 @@ def parse_captures_executed(path): ent.add(norm(int(a, 16))) return ent -def recall(needed_entries, static_ent, needed_crc=None): +def recall(needed_entries, static_ent, static_ranges, needed_crc=None): """needed_entries: set; static_ent: {entry->crc}. Returns dict of metrics.""" covered = set(static_ent) hit = needed_entries & covered miss = needed_entries - covered + range_starts = [lo for lo, _ in static_ranges] + def in_static_range(entry): + i = bisect.bisect_right(range_starts, entry) - 1 + return i >= 0 and entry < static_ranges[i][1] + range_hit = {entry for entry in needed_entries if in_static_range(entry)} + range_miss = needed_entries - range_hit out = {'needed': len(needed_entries), 'covered_here': len(hit), 'missed': len(miss), 'recall_entry': (len(hit)/len(needed_entries) if needed_entries else 0.0), - 'misses': sorted(miss)} + 'misses': sorted(miss), + 'covered_by_code_range': len(range_hit), + 'missed_code_range': len(range_miss), + 'recall_code_range': (len(range_hit)/len(needed_entries) if needed_entries else 0.0), + 'range_misses': sorted(range_miss)} if needed_crc is not None: crc_hit = sum(1 for e in hit if static_ent.get(e) == needed_crc.get(e)) out['covered_entry_crc'] = crc_hit @@ -91,32 +114,43 @@ def main(): if a.prior_report and not a.assume_static_superset: ap.error('--prior-report requires the explicit --assume-static-superset assertion') - static_ent = parse_ranges_dir(a.static) + static_ent, static_ranges = parse_ranges_dir(a.static) report = {'game': a.game, 'static_entries': len(static_ent), 'sources': {}} md = [] md.append(f'# AOT static-coverage recall — {a.game}\n') - md.append('_How much of what the game actually runs did the play-free static ' - 'extractor discover? Misses are the gap the runtime autocompile ' - '(or better discovery) must still cover._\n') + md.append('_How much of the played reference set did the play-free static ' + 'extractor reproduce, and how much lies in compiled static code?_\n') md.append(f'- Static shard cache: `{a.static}`') - md.append(f'- Static-covered overlay functions: **{len(static_ent)}**\n') + md.append(f'- Static manifest entries: **{len(static_ent)}**\n') if a.vault: - vault_ent = parse_ranges_dir(a.vault) - r = recall(set(vault_ent), static_ent, vault_ent) - report['sources']['vault'] = {k: v for k, v in r.items() if k != 'misses'} + vault_ent, _ = parse_ranges_dir(a.vault) + r = recall(set(vault_ent), static_ent, static_ranges, vault_ent) + report['sources']['vault'] = { + k: v for k, v in r.items() if k not in ('misses', 'range_misses')} report['vault_misses'] = [f'0x{x|0x80000000:08X}' for x in r['misses']] + report['vault_range_misses'] = [ + f'0x{x|0x80000000:08X}' for x in r['range_misses']] md.append('## vs played vault (most complete needed-set)\n') - md.append(f'- Functions the full playthrough ran: **{r["needed"]}**') + md.append(f'- Manifest entries in the full-playthrough vault: **{r["needed"]}**') md.append(f'- Discovered by static: **{r["covered_here"]}** ' f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') + md.append(f'- Covered by compiled static code ranges: ' + f'**{r["covered_by_code_range"]}** ' + f'(**{r["recall_code_range"]*100:.1f}%** code-range recall)') + md.append(' - Code-range recall answers whether the played entry PC is in ' + 'byte-guarded native code. Exact-entry recall is stricter manifest ' + 'granularity; runtime fragment caches can contain one entry per ' + 'instruction, so it substantially understates broad static shards.') md.append(f'- Byte-identical (entry+code_crc): **{r.get("covered_entry_crc",0)}** ' f'(**{r.get("recall_entry_crc",0)*100:.1f}%**) ' f'_(cg-version differences lower this vs entry-level)_') - md.append(f'- **MISSED: {r["missed"]}** functions the game needs but static did not find\n') - md.append('### Misses grouped by overlay region\n') - for reg, addrs in group_by_region(r['misses']).items(): + md.append(f'- **MISSED exact entries: {r["missed"]}**') + md.append(f'- **TRUE CODE-RANGE GAPS: {r["missed_code_range"]}** played ' + 'entry PCs outside all compiled static ranges\n') + md.append('### Code-range gaps grouped by overlay region\n') + for reg, addrs in group_by_region(r['range_misses']).items(): md.append(f'- region `0x{reg|0x80000000:08X}`: {len(addrs)} misses') md.append(' ```') for i in range(0, len(addrs), 8): @@ -126,13 +160,19 @@ def main(): if a.captures and os.path.exists(a.captures): live = parse_captures_executed(a.captures) - r = recall(live, static_ent) - report['sources']['live_session'] = {k: v for k, v in r.items() if k != 'misses'} + r = recall(live, static_ent, static_ranges) + report['sources']['live_session'] = { + k: v for k, v in r.items() if k not in ('misses', 'range_misses')} report['live_misses'] = [f'0x{x|0x80000000:08X}' for x in r['misses']] + report['live_range_misses'] = [ + f'0x{x|0x80000000:08X}' for x in r['range_misses']] md.append('## vs live session captures (this drive)\n') - md.append(f'- Functions this session exercised: **{r["needed"]}**') + md.append(f'- Dispatch entries this session exercised: **{r["needed"]}**') md.append(f'- Discovered by static: **{r["covered_here"]}** ' f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') + md.append(f'- Covered by compiled static code ranges: ' + f'**{r["covered_by_code_range"]}** ' + f'(**{r["recall_code_range"]*100:.1f}%** code-range recall)') md.append(f'- **MISSED live: {r["missed"]}**\n') elif a.prior_report: with open(a.prior_report, encoding='utf-8') as prior_in: @@ -147,17 +187,30 @@ def main(): needed = int(prior_live.get('needed', 0)) old_miss = {norm(int(x, 16)) for x in prior_misses} miss = old_miss - set(static_ent) + range_starts = [lo for lo, _ in static_ranges] + def in_static_range(entry): + i = bisect.bisect_right(range_starts, entry) - 1 + return i >= 0 and entry < static_ranges[i][1] + range_miss = {entry for entry in old_miss if not in_static_range(entry)} covered = needed - len(miss) r = {'needed': needed, 'covered_here': covered, 'missed': len(miss), - 'recall_entry': (covered/needed if needed else 0.0)} + 'recall_entry': (covered/needed if needed else 0.0), + 'covered_by_code_range': needed - len(range_miss), + 'missed_code_range': len(range_miss), + 'recall_code_range': ((needed - len(range_miss))/needed if needed else 0.0)} r['provenance'] = ('rolled forward from the persisted live gap manifest; ' 'caller asserted the current static entry set is a superset') report['sources']['live_session'] = r report['live_misses'] = [f'0x{x|0x80000000:08X}' for x in sorted(miss)] + report['live_range_misses'] = [ + f'0x{x|0x80000000:08X}' for x in sorted(range_miss)] md.append('## vs persisted live-session gaps (monotonic roll-forward)\n') - md.append(f'- Functions the persisted session exercised: **{needed}**') + md.append(f'- Dispatch entries the persisted session exercised: **{needed}**') md.append(f'- Discovered by current static: **{covered}** ' f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') + md.append(f'- Covered by current static code ranges: ' + f'**{r["covered_by_code_range"]}** ' + f'(**{r["recall_code_range"]*100:.1f}%** code-range recall)') md.append(f'- **MISSED live: {len(miss)}**') md.append('- Provenance: caller explicitly asserted that the current static ' 'entry set retains every prior static entry.\n') @@ -174,7 +227,9 @@ def main(): print(f'wrote {a.out_json}') for src, m in report['sources'].items(): print(f' {src}: recall_entry={m.get("recall_entry",0)*100:.1f}% ' - f'({m.get("covered_here")}/{m.get("needed")}), missed={m.get("missed")}') + f'({m.get("covered_here")}/{m.get("needed")}), ' + f'recall_code_range={m.get("recall_code_range",0)*100:.1f}%, ' + f'missed={m.get("missed")}') if __name__ == '__main__': main() From 6c67f81a3358473aeaaaea35d05dae0ea97038f6 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 01:21:05 -0700 Subject: [PATCH 08/37] docs: record Tomba 2 static attract recovery --- docs/AOT_OVERLAY_PLAN.md | 5 ++++- tools/aot_overlay_spike/README.md | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index a7fe96b7..7f688abd 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -17,7 +17,10 @@ consecutive instruction entries, while static AOT emits broad functions. Compile code-range recall is 1811/1856 (97.6%), leaving exactly 45 true gaps, all in the game-independent kernel/low-memory region. Runtime static-only validation loaded the richer MAIN shard at 60 fps with `stale_blocked=0`, no range/manifest overflow, -and byte-matching candidates; misses continue to fail safely to the interpreter. +and byte-matching candidates. The clean static-only attract soak then loaded the +459-function A01 composite and ran the formerly ~14 fps lava demo at 65–67 fps at +BelowNormal priority, with correct rendering and no range/manifest overflow. Misses +continue to fail safely to the interpreter. --- diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 5a56f6b0..5d16070e 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -119,9 +119,10 @@ page key. The extractor therefore also emits exact two-producer concatenations. For Tomba 2, `GAME.BIN` ends exactly where every A*/SOP file begins, producing `0x80106000` envelope variants in addition to the standalone `0x80108000` shards. The A01 composite—the live attract scene's measured hot variant—audits cleanly -(459 functions, zero unsupported/bad targets). Its performance recheck remains -pending; the earlier attempt overlapped an unrelated build and was not a valid -BelowNormal measurement. +(459 functions, zero unsupported/bad targets). A clean static-only soak loaded +that exact `00106000_0DC32D96` shard and ran the formerly ~14 fps lava-area attract +demo at 65–67 fps while the runtime process remained BelowNormal. The scene rendered +correctly, with no range-index or lazy-manifest overflow. `coverage_report.py` reports both exact manifest-entry recall and compiled code-range recall. The distinction matters: runtime/autocompile vaults may contain From f53569dfeab9d4534148b21d7a211b04358e8239 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 01:32:49 -0700 Subject: [PATCH 09/37] aot: account for native BIOS coverage --- docs/AOT_OVERLAY_PLAN.md | 6 + tools/aot_overlay_spike/README.md | 12 +- tools/aot_overlay_spike/coverage_report.py | 147 +++++++++++++++++- .../aot_overlay_spike/test_coverage_report.py | 43 +++++ 4 files changed, 199 insertions(+), 9 deletions(-) create mode 100644 tools/aot_overlay_spike/test_coverage_report.py diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index 7f688abd..027b5b64 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -149,6 +149,12 @@ explicit assertion is required because an old gap manifest stores misses, not every previously covered entry; silently assuming monotonicity would make a regressed static set look healthier than it is. +The base BIOS recompiler is a separate static producer. When its generated +dispatcher is available, pass `--bios-dispatch ` so the +scoreboard preserves overlay-cache-only recall and also reports combined native +coverage from the dispatch table plus guarded relocated-kernel body ranges. Do not +manufacture overlay shards merely to duplicate those already-native BIOS bodies. + ### 1.3 Compile (offline, `tools/compile_overlays.py`) 1. Python does function-boundary discovery from the seeds (`classify_overlay_seeds`, `_walk_overlay_function`) with a strict callable-prologue gate; interior/ diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 5d16070e..2c281f8f 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -130,7 +130,11 @@ one-entry fragments at consecutive instructions, while static AOT emits one broa function with the same PCs in its guarded `R` ranges. Exact-entry parity is retained for continuity; code-range recall is the useful uncovered-code roadmap. At the current Tomba 2 checkpoint these are 1270/1856 (68.4%) exact entries and 1811/1856 -(97.6%) code-range coverage. All 45 remaining code-range gaps are in low/kernel RAM. +(97.6%) overlay-cache code-range coverage. Pass the generated base BIOS dispatcher +with `--bios-dispatch generated/SCPH1001_dispatch.c` to report separate combined +metrics. The base recompiler already covers relocated kernel bodies with a live-byte +guard; including it raises Tomba 2 to 1852/1856 (99.8%) native code-range coverage. +This avoids generating duplicate overlay shards for code that is already native. ## Next to raise coverage (future session) @@ -147,6 +151,10 @@ current Tomba 2 checkpoint these are 1270/1856 (68.4%) exact entries and 1811/18 A09/A0J/OPN; GAME's 8-byte near-tie resolves through the exact base independently proved by DEMO. The played vault byte-proves GAME at 0x80106228. Ambiguous files without one unique trusted match still fail closed. -4. Extract the kernel `0x80000000` region once from the BIOS (game-independent): +4. ~~Extract the kernel `0x80000000` region once from the BIOS~~ — NOT NEEDED for + the measured body gaps. The base BIOS recompiler already emits 499 relocated, + guarded kernel bodies. Of the four combined gaps, A0/B0/C0 are tiny installed + vector thunks and EE7C is an all-zero runtime-fragment artifact; keep the latter + interpreted rather than treating padding/data as code. this is now the complete 45-PC Tomba 2 code-range gap set. 5. Generalize into a real `tools/` producer (no hardcoded paths; disc + game.toml in). diff --git a/tools/aot_overlay_spike/coverage_report.py b/tools/aot_overlay_spike/coverage_report.py index 982f25e5..55294c1e 100644 --- a/tools/aot_overlay_spike/coverage_report.py +++ b/tools/aot_overlay_spike/coverage_report.py @@ -19,6 +19,7 @@ Recall is reported at three strengths: Usage: coverage_report.py --static --vault + [--bios-dispatch ] [--captures ] [--prior-report --assume-static-superset] --out-md --out-json [--game ] @@ -35,6 +36,15 @@ def region_of(a): """Page-region label for grouping (aligns to the overlay window bases).""" return norm(a) & 0xFFFFF000 +def merge_intervals(intervals): + merged = [] + for lo, hi in sorted(intervals): + if not merged or lo > merged[-1][1]: + merged.append([lo, hi]) + elif hi > merged[-1][1]: + merged[-1][1] = hi + return [(lo, hi) for lo, hi in merged] + def parse_ranges_dir(d): """Return ({entry_masked: code_crc}, merged compiled-code intervals).""" ent = {} @@ -49,13 +59,53 @@ def parse_ranges_dir(d): if m: lo = norm(int(m.group(1), 16)) intervals.append((lo, lo + int(m.group(2), 16))) - merged = [] - for lo, hi in sorted(intervals): - if not merged or lo > merged[-1][1]: - merged.append([lo, hi]) - elif hi > merged[-1][1]: - merged[-1][1] = hi - return ent, [(lo, hi) for lo, hi in merged] + return ent, merge_intervals(intervals) + +def _c_initializer(text, declaration_pattern): + """Return the braced initializer following a unique C declaration.""" + match = re.search(declaration_pattern, text) + if not match: + raise ValueError(f'missing generated-C declaration: {declaration_pattern}') + start = text.find('{', match.end()) + if start < 0: + raise ValueError('generated-C declaration has no initializer') + depth = 0 + for pos in range(start, len(text)): + if text[pos] == '{': + depth += 1 + elif text[pos] == '}': + depth -= 1 + if depth == 0: + return text[start + 1:pos] + raise ValueError('unterminated generated-C initializer') + +def parse_bios_dispatch(path): + """Return base-BIOS dispatch entries and relocated kernel body ranges. + + The generated dispatcher has many unrelated numeric initializers, so parse + only the two named tables instead of matching address triples globally. + """ + with open(path, encoding='utf-8', errors='ignore') as generated: + text = generated.read() + dispatch = _c_initializer( + text, r'\bDispatchEntry\s+dispatch_table\s*\[[^]]*\]\s*=') + bodies = _c_initializer( + text, r'\bPsxKernelBody\s+psx_bios_kernel_bodies\s*\[[^]]*\]\s*=') + entries = { + norm(int(value, 16)) + for value in re.findall( + r'\{\s*0x([0-9A-Fa-f]+)u?\s*,\s*func_[0-9A-Fa-f]+\s*\}', dispatch) + } + intervals = [] + for _, lo, hi in re.findall( + r'\{\s*0x([0-9A-Fa-f]+)u?\s*,\s*' + r'0x([0-9A-Fa-f]+)u?\s*,\s*0x([0-9A-Fa-f]+)u?\s*\}', bodies): + lo, hi = norm(int(lo, 16)), norm(int(hi, 16)) + if hi > lo: + intervals.append((lo, hi)) + if not entries or not intervals: + raise ValueError('generated BIOS dispatcher contained an empty static table') + return entries, merge_intervals(intervals) def parse_captures_executed(path): """-> set of FUNCTION-ENTRY addrs (masked) the runtime dispatched to. @@ -101,6 +151,9 @@ def group_by_region(addrs): def main(): ap = argparse.ArgumentParser() ap.add_argument('--static', required=True, help='static shard cache dir (.ranges)') + ap.add_argument('--bios-dispatch', + help='generated base-BIOS dispatch C; adds already-native BIOS ' + 'entries/ranges to separate combined metrics') ap.add_argument('--vault', help='played/coverage vault cache dir (.ranges)') ap.add_argument('--captures', help='runtime overlay_captures.json (executed_pcs)') ap.add_argument('--prior-report', help='roll forward a persisted live gap set') @@ -116,6 +169,18 @@ def main(): static_ent, static_ranges = parse_ranges_dir(a.static) report = {'game': a.game, 'static_entries': len(static_ent), 'sources': {}} + combined_ent = dict(static_ent) + combined_ranges = static_ranges + if a.bios_dispatch: + try: + bios_ent, bios_ranges = parse_bios_dispatch(a.bios_dispatch) + except (OSError, ValueError) as exc: + ap.error(f'cannot parse --bios-dispatch: {exc}') + combined_ent.update({entry: None for entry in bios_ent}) + combined_ranges = merge_intervals(static_ranges + bios_ranges) + report['bios_dispatch'] = a.bios_dispatch + report['bios_static_entries'] = len(bios_ent) + report['bios_static_ranges'] = len(bios_ranges) md = [] md.append(f'# AOT static-coverage recall — {a.game}\n') @@ -123,6 +188,11 @@ def main(): 'extractor reproduce, and how much lies in compiled static code?_\n') md.append(f'- Static shard cache: `{a.static}`') md.append(f'- Static manifest entries: **{len(static_ent)}**\n') + if a.bios_dispatch: + md.append(f'- Base BIOS native dispatch entries: **{len(bios_ent)}**; ' + f'relocated kernel body ranges: **{len(bios_ranges)}**') + md.append('- Combined metrics below count both the play-free overlay cache ' + 'and the separately generated, live-byte-guarded base BIOS.\n') if a.vault: vault_ent, _ = parse_ranges_dir(a.vault) @@ -132,6 +202,13 @@ def main(): report['vault_misses'] = [f'0x{x|0x80000000:08X}' for x in r['misses']] report['vault_range_misses'] = [ f'0x{x|0x80000000:08X}' for x in r['range_misses']] + combined = None + if a.bios_dispatch: + combined = recall(set(vault_ent), combined_ent, combined_ranges) + report['sources']['vault_combined_bios'] = { + k: v for k, v in combined.items() if k not in ('misses', 'range_misses')} + report['vault_combined_range_misses'] = [ + f'0x{x|0x80000000:08X}' for x in combined['range_misses']] md.append('## vs played vault (most complete needed-set)\n') md.append(f'- Manifest entries in the full-playthrough vault: **{r["needed"]}**') md.append(f'- Discovered by static: **{r["covered_here"]}** ' @@ -149,6 +226,24 @@ def main(): md.append(f'- **MISSED exact entries: {r["missed"]}**') md.append(f'- **TRUE CODE-RANGE GAPS: {r["missed_code_range"]}** played ' 'entry PCs outside all compiled static ranges\n') + if combined: + md.append('### Combined with base recompiled BIOS\n') + md.append(f'- Exact native dispatch entries: **{combined["covered_here"]}** ' + f'(**{combined["recall_entry"]*100:.1f}%**)') + md.append(f'- Covered by native code ranges: ' + f'**{combined["covered_by_code_range"]}** ' + f'(**{combined["recall_code_range"]*100:.1f}%**)') + md.append(f'- **COMBINED CODE-RANGE GAPS: ' + f'{combined["missed_code_range"]}**\n') + md.append('#### Combined gaps grouped by region\n') + for reg, addrs in group_by_region(combined['range_misses']).items(): + md.append(f'- region `0x{reg|0x80000000:08X}`: {len(addrs)} misses') + md.append(' ```') + for i in range(0, len(addrs), 8): + md.append(' ' + ' '.join( + f'{x|0x80000000:08X}' for x in addrs[i:i+8])) + md.append(' ```') + md.append('') md.append('### Code-range gaps grouped by overlay region\n') for reg, addrs in group_by_region(r['range_misses']).items(): md.append(f'- region `0x{reg|0x80000000:08X}`: {len(addrs)} misses') @@ -166,6 +261,13 @@ def main(): report['live_misses'] = [f'0x{x|0x80000000:08X}' for x in r['misses']] report['live_range_misses'] = [ f'0x{x|0x80000000:08X}' for x in r['range_misses']] + combined = None + if a.bios_dispatch: + combined = recall(live, combined_ent, combined_ranges) + report['sources']['live_session_combined_bios'] = { + k: v for k, v in combined.items() if k not in ('misses', 'range_misses')} + report['live_combined_range_misses'] = [ + f'0x{x|0x80000000:08X}' for x in combined['range_misses']] md.append('## vs live session captures (this drive)\n') md.append(f'- Dispatch entries this session exercised: **{r["needed"]}**') md.append(f'- Discovered by static: **{r["covered_here"]}** ' @@ -173,6 +275,10 @@ def main(): md.append(f'- Covered by compiled static code ranges: ' f'**{r["covered_by_code_range"]}** ' f'(**{r["recall_code_range"]*100:.1f}%** code-range recall)') + if combined: + md.append(f'- Including base BIOS native code ranges: ' + f'**{combined["covered_by_code_range"]}** ' + f'(**{combined["recall_code_range"]*100:.1f}%**)') md.append(f'- **MISSED live: {r["missed"]}**\n') elif a.prior_report: with open(a.prior_report, encoding='utf-8') as prior_in: @@ -204,6 +310,29 @@ def main(): report['live_misses'] = [f'0x{x|0x80000000:08X}' for x in sorted(miss)] report['live_range_misses'] = [ f'0x{x|0x80000000:08X}' for x in sorted(range_miss)] + combined = None + if a.bios_dispatch: + combined_miss = old_miss - set(combined_ent) + combined_starts = [lo for lo, _ in combined_ranges] + def in_combined_range(entry): + i = bisect.bisect_right(combined_starts, entry) - 1 + return i >= 0 and entry < combined_ranges[i][1] + combined_range_miss = { + entry for entry in old_miss if not in_combined_range(entry)} + combined_covered = needed - len(combined_miss) + combined = { + 'needed': needed, + 'covered_here': combined_covered, + 'missed': len(combined_miss), + 'recall_entry': (combined_covered/needed if needed else 0.0), + 'covered_by_code_range': needed - len(combined_range_miss), + 'missed_code_range': len(combined_range_miss), + 'recall_code_range': ( + (needed - len(combined_range_miss))/needed if needed else 0.0), + 'provenance': r['provenance']} + report['sources']['live_session_combined_bios'] = combined + report['live_combined_range_misses'] = [ + f'0x{x|0x80000000:08X}' for x in sorted(combined_range_miss)] md.append('## vs persisted live-session gaps (monotonic roll-forward)\n') md.append(f'- Dispatch entries the persisted session exercised: **{needed}**') md.append(f'- Discovered by current static: **{covered}** ' @@ -211,6 +340,10 @@ def main(): md.append(f'- Covered by current static code ranges: ' f'**{r["covered_by_code_range"]}** ' f'(**{r["recall_code_range"]*100:.1f}%** code-range recall)') + if combined: + md.append(f'- Including base BIOS native code ranges: ' + f'**{combined["covered_by_code_range"]}** ' + f'(**{combined["recall_code_range"]*100:.1f}%**)') md.append(f'- **MISSED live: {len(miss)}**') md.append('- Provenance: caller explicitly asserted that the current static ' 'entry set retains every prior static entry.\n') diff --git a/tools/aot_overlay_spike/test_coverage_report.py b/tools/aot_overlay_spike/test_coverage_report.py new file mode 100644 index 00000000..28c67c7b --- /dev/null +++ b/tools/aot_overlay_spike/test_coverage_report.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +import os +import tempfile +import unittest + +import coverage_report + + +class CoverageReportTests(unittest.TestCase): + def test_bios_parser_is_scoped_to_named_tables(self): + source = r''' +static const DispatchEntry dispatch_table[2] = { + { 0x00000500u, func_00000500 }, + { 0xBFC00180u, func_BFC00180 }, +}; +static const unsigned unrelated[][3] = { + { 0xDEADBEEFu, 0x00000001u, 0x1FFFFFFFu }, +}; +const PsxKernelBody psx_bios_kernel_bodies[2] = { + { 0x00000500u, 0x00000500u, 0x00000510u }, + { 0x00000510u, 0x00000510u, 0x00000520u }, +}; +''' + with tempfile.NamedTemporaryFile( + mode='w', suffix='.c', delete=False, encoding='utf-8') as generated: + generated.write(source) + path = generated.name + try: + entries, ranges = coverage_report.parse_bios_dispatch(path) + finally: + os.unlink(path) + self.assertEqual(entries, {0x500, 0x1FC00180}) + self.assertEqual(ranges, [(0x500, 0x520)]) + + def test_code_range_recall_is_distinct_from_exact_entry_recall(self): + result = coverage_report.recall( + {0x100, 0x104}, {0x100: 0x1234}, [(0x100, 0x108)]) + self.assertEqual(result['covered_here'], 1) + self.assertEqual(result['covered_by_code_range'], 2) + + +if __name__ == '__main__': + unittest.main() From 23de1d5bcd2512ba89b8996019be12f5d79cec2b Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 01:55:05 -0700 Subject: [PATCH 10/37] aot: seed dense function pointer tables --- recompiler/tests/test_aot_base_recovery.py | 18 +++++++ tools/aot_overlay_spike/README.md | 6 +++ tools/aot_overlay_spike/extract_generic.py | 63 ++++++++++++++++++++-- 3 files changed, 83 insertions(+), 4 deletions(-) diff --git a/recompiler/tests/test_aot_base_recovery.py b/recompiler/tests/test_aot_base_recovery.py index 5c95e9a3..52d574d3 100644 --- a/recompiler/tests/test_aot_base_recovery.py +++ b/recompiler/tests/test_aot_base_recovery.py @@ -35,6 +35,24 @@ def main(): struct.pack_into("> 2) & 0x03FFFFFF)) assert MOD.jal_targets(data) == {0x80105678} + # Dense function-pointer tables may recover framed or frameless functions, + # but only when every target has independent callable-boundary evidence. + base = 0x80100000 + data = bytearray(0x100) + # Three adjacent leaves, each immediately after a previous return. + for off in (0x18, 0x28, 0x38): + struct.pack_into('>16)==0x27BD and (struct.unpack_from('> 16) == 0x27BD and (w & 0x8000): + return True + # A return, arbitrary architectural delay slot, and up to six alignment + # NOPs is the same Psy-Q boundary accepted by overlay exact-entry mode. + for padding in range(7): + jr_off = off - 8 - padding*4 + if _word(data, jr_off) != 0x03E00008: + continue + if all(_word(data, p) == 0 for p in range(jr_off+8, off, 4)): + return True + return False + +def pointer_table_targets(data, base): + """Callable targets from dense in-file function-pointer tables. + + A run needs at least three adjacent in-range pointers, and each promoted + target independently needs a prologue/return boundary. This deliberately + refuses isolated pointer-shaped data and mid-function dispatch labels. + """ + lo, hi = base, base + len(data) + words = [_word(data, off) for off in range(0, len(data)-3, 4)] + in_range = lambda value: value is not None and (value & 3) == 0 and lo <= value < hi + targets = set() + start = 0 + while start < len(words): + if not in_range(words[start]): + start += 1 + continue + end = start + 1 + while end < len(words) and in_range(words[end]): + end += 1 + if end - start >= 3: + for value in words[start:end]: + if _callable_boundary(data, value-base): + targets.add(value) + start = end + return targets + +def supplemental_callable_seeds(data, base): + return pointer_table_targets(data, base) + def jal_targets(data): """Distinct absolute jal call targets. These are BASE-INDEPENDENT: the target is 0x80000000 | (imm26<<2) regardless of where the file loads, because all @@ -295,7 +344,9 @@ def main(): # the old prologue-density delta-sweep, which matched export pointers # against prologues they never point at -> wrong base by 2-16KB). base,score=rb - seeds=prologues(data,base) # seed via full-file prologue scan + prologue_seeds=prologues(data,base) + supplemental=supplemental_callable_seeds(data,base) + seeds=sorted(set(prologue_seeds)|supplemental) # SUPPLEMENT the prologue seeds with the overlay's OWN export/dispatch # table (the header pointers) — the game's authoritative list of live # entry points, sitting on the disc (play-free). These are exactly the @@ -319,7 +370,8 @@ def main(): extra=len(set(hdr_entries)-set(seeds)) print(f" [header-table] {p}: {len(data)}B file@0x{base:08X} " f"region@0x{page_base:08X}(+{fill}) " - f"({fit_method} score={score}), prologue seeds={len(seeds)} " + f"({fit_method} score={score}), prologue seeds={len(prologue_seeds)} " + f"+{len(supplemental-set(prologue_seeds))} pointer-table seeds " f"+{extra} export-table dispatch entries") # Some engines keep call-dense position-fixed code in raw siblings whose @@ -331,7 +383,9 @@ def main(): rb=recover_raw_base(data, raw_base_lo) if rb is None: continue base,score,second=rb - seeds=prologues(data,base) + prologue_seeds=prologues(data,base) + supplemental=supplemental_callable_seeds(data,base) + seeds=sorted(set(prologue_seeds)|supplemental) page_base=base&~0xFFF fill=base-page_base region=b'\x00'*fill+data @@ -341,7 +395,8 @@ def main(): print(f" [raw-code] {p}: {len(data)}B file@0x{base:08X} " f"region@0x{page_base:08X}(+{fill}) " f"(strict jal-fit score={score}, runner-up={second}), " - f"prologue seeds={len(seeds)}") + f"prologue seeds={len(prologue_seeds)} " + f"+{len(supplemental-set(prologue_seeds))} pointer-table seeds") # Runtime dirty-page capture coalesces directly adjacent producers into one # region keyed by the earlier page. Emit each provable two-file composition From 495bf2de53402cbdc0f17eb854f672b157972f0b Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 02:56:42 -0700 Subject: [PATCH 11/37] aot: constrain calls across composite producers --- recompiler/include/function_analysis.h | 7 +- recompiler/src/function_analysis.cpp | 33 +++- recompiler/src/main_psx.cpp | 58 +++++- recompiler/tests/reachable_discovery_test.cpp | 13 ++ .../tests/test_aot_overlay_discovery.py | 74 ++++++++ tools/aot_overlay_spike/extract_generic.py | 22 ++- tools/compile_overlays.py | 166 +++++++++++++++++- 7 files changed, 348 insertions(+), 25 deletions(-) diff --git a/recompiler/include/function_analysis.h b/recompiler/include/function_analysis.h index bdb87486..d5a914db 100644 --- a/recompiler/include/function_analysis.h +++ b/recompiler/include/function_analysis.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include #include #include "ps1_exe_parser.h" @@ -52,7 +54,10 @@ public: // reachable from them. Used by runtime-loaded overlays and opt-in main-EXE // reachable discovery. Unresolved jalr/indirect targets do not mint // functions; evidence-backed entries must be supplied explicitly. - FunctionAnalysisResult analyze_exact_entries(const std::vector& entries); + FunctionAnalysisResult analyze_exact_entries( + const std::vector& entries, + const std::vector>& producer_ranges = {}, + const std::set& cross_call_allow = {}); // Add a forced entry point address that is treated as a function start // even if it has no standard ADDIU $sp prologue. The function will be diff --git a/recompiler/src/function_analysis.cpp b/recompiler/src/function_analysis.cpp index 3ac817c0..570e5ead 100644 --- a/recompiler/src/function_analysis.cpp +++ b/recompiler/src/function_analysis.cpp @@ -644,7 +644,10 @@ static ExactCf exact_classify_cf(uint32_t pc, uint32_t instr) { } // namespace -FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries(const std::vector& entries) { +FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( + const std::vector& entries, + const std::vector>& producer_ranges, + const std::set& cross_call_allow) { FunctionAnalysisResult result; result.total_instructions = 0; result.jr_ra_count = 0; @@ -658,13 +661,28 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries(const std::vector return addr >= exe_.header.load_address && addr < exe_.end_address() && (addr & 3u) == 0; }; - auto callable_direct_jal_target = [&](uint32_t addr) { + auto producer_for = [&](uint32_t addr) -> int { + for (size_t i = 0; i < producer_ranges.size(); i++) { + if (addr >= producer_ranges[i].first && addr < producer_ranges[i].second) + return static_cast(i); + } + return -1; + }; + + auto callable_direct_jal_target = [&](uint32_t source, uint32_t addr) { auto word_opt = exe_.read_word(addr); // A JAL encountered by the reachable CFG walk is direct call evidence. // Requiring a stack prologue here discarded legitimate frameless leaf // functions and tiny library wrappers. The target still has to decode // as a valid PS1 instruction and lie inside the verified image bound. - return word_opt.has_value() && exact_is_valid_mips_word(*word_opt); + if (!word_opt.has_value() || !exact_is_valid_mips_word(*word_opt)) + return false; + if (producer_ranges.empty()) return true; + int source_producer = producer_for(source); + int target_producer = producer_for(addr); + if (source_producer >= 0 && source_producer == target_producer) + return true; + return cross_call_allow.count(addr) != 0; }; auto find_jump_table_targets = [&](uint32_t entry, uint32_t hard_cap, @@ -788,7 +806,8 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries(const std::vector case ExactCfKind::Jal: if (in_function(delay)) wr.visited.insert(delay); if (in_function(pc + 8u)) work.push(pc + 8u); - if (in_exe(cf.target) && callable_direct_jal_target(cf.target)) { + if (in_exe(cf.target) && + callable_direct_jal_target(pc, cf.target)) { wr.direct_jal_targets.insert(cf.target); } break; @@ -800,7 +819,8 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries(const std::vector uint32_t target_reg = (instr >> 21) & 0x1Fu; if (exact_resolve_constant_transfer(exe_, entry, pc, target_reg, target) && - in_exe(target)) { + in_exe(target) && + callable_direct_jal_target(pc, target)) { wr.direct_jal_targets.insert(target); } } @@ -819,7 +839,8 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries(const std::vector if (in_function(target)) { wr.jump_table_targets.insert(target); work.push(target); - } else if (in_exe(target)) { + } else if (in_exe(target) && + callable_direct_jal_target(pc, target)) { // An absolute JR out of the current function is a // statically proven tail-call entry. wr.direct_jal_targets.insert(target); diff --git a/recompiler/src/main_psx.cpp b/recompiler/src/main_psx.cpp index 68268caa..bc9472af 100644 --- a/recompiler/src/main_psx.cpp +++ b/recompiler/src/main_psx.cpp @@ -432,6 +432,8 @@ int main(int argc, char** argv) { std::vector interior_seeds; std::set trusted_root_seeds; std::set trusted_call_root_seeds; + std::vector> producer_ranges; + std::set cross_call_allow; if (extra_funcs_path) { std::ifstream ef(extra_funcs_path); if (ef.is_open()) { @@ -440,6 +442,33 @@ int main(int argc, char** argv) { std::string line; while (std::getline(ef, line)) { if (line.empty() || line[0] == '#') continue; + if (line.rfind("producer_range", 0) == 0) { + std::istringstream in(line); + std::string tag, lo_text, hi_text; + in >> tag >> lo_text >> hi_text; + uint32_t lo = static_cast( + std::strtoul(lo_text.c_str(), nullptr, 16)); + uint32_t hi = static_cast( + std::strtoul(hi_text.c_str(), nullptr, 16)); + if (lo_text.empty() || hi_text.empty() || + lo < seed_lo || lo >= hi || hi > seed_hi) { + fmt::print(stderr, "ERROR: invalid producer_range: {}\n", line); + return 1; + } + producer_ranges.emplace_back(lo, hi); + continue; + } + if (line.rfind("cross_call_allow", 0) == 0) { + const char* p = line.c_str() + 16; + uint32_t addr = static_cast( + std::strtoul(p, nullptr, 16)); + if (addr < seed_lo || addr >= seed_hi) { + fmt::print(stderr, "ERROR: invalid cross_call_allow: {}\n", line); + return 1; + } + cross_call_allow.insert(addr); + continue; + } bool interior = false; bool trusted_root = false; bool trusted_call_root = false; @@ -466,11 +495,35 @@ int main(int argc, char** argv) { } } } + std::sort(producer_ranges.begin(), producer_ranges.end()); + for (size_t i = 1; i < producer_ranges.size(); i++) { + if (producer_ranges[i - 1].second > producer_ranges[i].first) { + fmt::print(stderr, "ERROR: overlapping producer_range entries\n"); + return 1; + } + } + for (uint32_t addr : cross_call_allow) { + bool contained = false; + for (const auto& range : producer_ranges) { + if (addr >= range.first && addr < range.second) { + contained = true; + break; + } + } + if (!contained) { + fmt::print(stderr, + "ERROR: cross_call_allow 0x{:08X} is outside " + "producer ranges\n", addr); + return 1; + } + } fmt::print("Loaded {} extra function addresses ({} interior, " - "{} dispatch-root, {} call-root) from {}\n", + "{} dispatch-root, {} call-root, {} producer ranges, " + "{} cross-call allows) from {}\n", file_seeds.size() + interior_seeds.size(), interior_seeds.size(), trusted_root_seeds.size(), trusted_call_root_seeds.size(), + producer_ranges.size(), cross_call_allow.size(), extra_funcs_path); } else { fmt::print("WARNING: Cannot open extra-funcs file: {}\n", extra_funcs_path); @@ -560,7 +613,8 @@ int main(int argc, char** argv) { { PSXRecomp::FunctionAnalyzer analyzer(*exe); std::vector roots_vec(roots.begin(), roots.end()); - analysis_result = analyzer.analyze_exact_entries(roots_vec); + analysis_result = analyzer.analyze_exact_entries( + roots_vec, producer_ranges, cross_call_allow); } std::vector alias_entries; diff --git a/recompiler/tests/reachable_discovery_test.cpp b/recompiler/tests/reachable_discovery_test.cpp index 5ea74b2b..6f635f73 100644 --- a/recompiler/tests/reachable_discovery_test.cpp +++ b/recompiler/tests/reachable_discovery_test.cpp @@ -111,6 +111,19 @@ int main() { CHECK(!reachable.count(kLoad + 0x1100), "direct target beyond verified bound fails closed"); + // A synthetic adjacent-producer envelope is not one linked image. The + // caller supplies the cross-boundary targets that passed its independent + // callable-CFG proof; other valid-looking targets stay with the interpreter. + PSXRecomp::FunctionAnalyzer composite_analyzer(exe); + const std::vector> producer_ranges = { + {kLoad, kLoad + 0x80}, {kLoad + 0x80, kLoad + 0x1000}}; + const auto composite = starts(composite_analyzer.analyze_exact_entries( + {kLoad}, producer_ranges, {kLoad + 0x100})); + CHECK(composite.count(kLoad) && composite.count(kLoad + 0x100), + "approved cross-producer call target is discovered"); + CHECK(!composite.count(kLoad + 0x300), + "unapproved cross-producer JAL target fails closed"); + // A constant-register JR out of the current function is a statically // proven tail call. Resolve the common lui/addiu/jr sequence without an // execution-derived seed. diff --git a/recompiler/tests/test_aot_overlay_discovery.py b/recompiler/tests/test_aot_overlay_discovery.py index df679e72..66707d70 100644 --- a/recompiler/tests/test_aot_overlay_discovery.py +++ b/recompiler/tests/test_aot_overlay_discovery.py @@ -37,6 +37,42 @@ def make_psxexe(entry, data): return bytes(header) + data +def check_composite_call_boundaries(): + data = bytearray(0x100) + cross_target = LOAD + 0x50 + put(data, 0x00, jal(cross_target)) + put(data, 0x04, 0x00000000) + put(data, 0x08, 0x03E00008) + put(data, 0x0C, 0x00000000) + put(data, 0x50, 0x24020001) # valid MIPS, but no callable boundary + ranges = ((LOAD, LOAD + 0x40), (LOAD + 0x40, LOAD + 0x100)) + + # A direct JAL remains sufficient for frameless leaves inside one producer, + # but not when an adjacent-producer variant merely puts pointer/data-shaped + # bytes at the encoded address. + unscoped = MOD._walk_overlay_function(data, LOAD, len(data), LOAD, LOAD + 0x100) + assert cross_target in unscoped['direct_jals'] + scoped = MOD._walk_overlay_function( + data, LOAD, len(data), LOAD, LOAD + 0x100, ranges) + assert cross_target not in scoped['direct_jals'] + assert cross_target in scoped['rejected_cross_producer_calls'] + + # A bounded frameless body with a real return is independently code-like + # and remains discoverable across the producer boundary. + put(data, 0x54, 0x03E00008) + put(data, 0x58, 0x00000000) + scoped = MOD._walk_overlay_function( + data, LOAD, len(data), LOAD, LOAD + 0x100, ranges) + assert cross_target in scoped['direct_jals'] + + # Independent target-local boundary evidence makes a real cross-producer + # export safe to retain. + put(data, 0x50, 0x27BDFFE0) + scoped = MOD._walk_overlay_function( + data, LOAD, len(data), LOAD, LOAD + 0x100, ranges) + assert cross_target in scoped['direct_jals'] + + def check_padded_return_boundary(recompiler): # Exact-entry mode re-verifies every untrusted seed. Psy-Q aligns frameless # leaves with NOPs after the prior function's JR delay slot; those NOPs must @@ -67,6 +103,41 @@ def check_padded_return_boundary(recompiler): assert f"F {LOAD + 0x14:08X}" in ranges +def check_recompiler_composite_contract(recompiler): + data = bytearray(0x80) + target = LOAD + 0x50 + put(data, 0x00, jal(target)) + put(data, 0x04, 0x00000000) + put(data, 0x08, 0x03E00008) + put(data, 0x0C, 0x00000000) + put(data, 0x50, 0x24020001) + put(data, 0x54, 0x03E00008) + put(data, 0x58, 0x00000000) + + with tempfile.TemporaryDirectory() as tmp: + psx = os.path.join(tmp, "composite.psx") + with open(psx, "wb") as f: + f.write(make_psxexe(LOAD, data)) + + def generated(allow): + seeds = os.path.join(tmp, f"seeds_{allow}.txt") + out = os.path.join(tmp, f"out_{allow}") + with open(seeds, "w", encoding="utf-8") as f: + f.write(f"producer_range 0x{LOAD:08X} 0x{LOAD + 0x40:08X}\n") + f.write(f"producer_range 0x{LOAD + 0x40:08X} 0x{LOAD + 0x80:08X}\n") + if allow: + f.write(f"cross_call_allow 0x{target:08X}\n") + f.write(f"0x{LOAD:08X}\n") + result = subprocess.run( + [recompiler, psx, "--seeds", seeds, "--out-dir", out, "--overlay"], + capture_output=True, text=True) + assert result.returncode == 0, result.stderr or result.stdout + return ''.join(path.read_text() for path in pathlib.Path(out).glob("*_full*.c")) + + assert f"void func_{target:08X}" not in generated(False) + assert f"void func_{target:08X}" in generated(True) + + def main(): default_recompiler = ROOT / "recompiler" / "build" / "psxrecomp-game.exe" parser = argparse.ArgumentParser() @@ -75,6 +146,9 @@ def main(): if not os.path.isfile(args.recompiler): raise SystemExit(f"recompiler not found: {args.recompiler}") + check_composite_call_boundaries() + check_recompiler_composite_contract(args.recompiler) + data = bytearray(0x200) # A framed root directly calls a frameless leaf. JAL is the boundary diff --git a/tools/aot_overlay_spike/extract_generic.py b/tools/aot_overlay_spike/extract_generic.py index 001265ac..9f558699 100644 --- a/tools/aot_overlay_spike/extract_generic.py +++ b/tools/aot_overlay_spike/extract_generic.py @@ -245,7 +245,7 @@ def full_discovery_seeds(data, recompiler, tmp): a=int(m.group(1),16); seeds.add(a if a>=0x80000000 else (0x80000000|a)) return sorted(seeds) or None -def rec(load_addr, data, seeds, dispatch_extra=None): +def rec(load_addr, data, seeds, dispatch_extra=None, producer_ranges=None): # function_entry_pcs = prologue-scanned function starts (walk roots). # dispatch_entry_pcs = those PLUS any extra dispatch targets (e.g. the # overlay's own export table). compile_overlays promotes clean starts to @@ -253,11 +253,16 @@ def rec(load_addr, data, seeds, dispatch_extra=None): # (impossible_entry_start guards garbage), so passing the game's authoritative # dispatch table here recovers indirect-only entries a prologue scan misses. disp = sorted(set(seeds) | set(dispatch_extra or ())) - return {"schema":"psxrecomp overlay capture v2","load_addr":f"0x{load_addr:08X}", - "size":len(data),"bytes_b64":base64.b64encode(data).decode(), - "executed_pcs":[],"dispatch_entry_pcs":[f"0x{a:08X}" for a in disp], - "function_entry_pcs":[f"0x{a:08X}" for a in seeds], - "seeds":[f"0x{a:08X}" for a in seeds]} + out={"schema":"psxrecomp overlay capture v2","load_addr":f"0x{load_addr:08X}", + "size":len(data),"bytes_b64":base64.b64encode(data).decode(), + "executed_pcs":[],"dispatch_entry_pcs":[f"0x{a:08X}" for a in disp], + "function_entry_pcs":[f"0x{a:08X}" for a in seeds], + "seeds":[f"0x{a:08X}" for a in seeds]} + if producer_ranges: + out["producer_ranges"]=[ + {"start":f"0x{lo:08X}","end":f"0x{hi:08X}"} + for lo,hi in producer_ranges] + return out def main(): ap=argparse.ArgumentParser() @@ -417,7 +422,10 @@ def main(): composite_keys.add(key) seeds=sorted(set(lseeds)|set(rseeds)) dispatch=sorted(set(ldispatch)|set(rdispatch)) - records.append(rec(page_base,region,seeds,dispatch_extra=dispatch)) + records.append(rec( + page_base,region,seeds,dispatch_extra=dispatch, + producer_ranges=((lbase,lbase+len(ldata)), + (rbase,rbase+len(rdata))))) nc+=1 print(f" [composite] {lp} + {rp}: {len(region)}B " f"region@0x{page_base:08X} (exact adjacent producers), " diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index a39f0000..0f2e4aac 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -563,8 +563,81 @@ def _resolve_constant_transfer(data: bytes, load_addr: int, return None +def _plausible_callable_target(data: bytes, load_addr: int, size: int, + target: int, producer_hi: int) -> bool: + """CFG proof for a cross-producer call target without a classic prologue. + + Direct calls are sufficient boundary evidence inside one linked producer. + Across a synthetic adjacent-producer envelope, additionally require a + bounded, valid CFG with a reachable return and no sequential fallthrough. + This retains real frameless exports while rejecting pointer tables/NOP + runways that only happen to decode as instructions. + """ + first = _word_at(data, load_addr, target) + if first in (None, 0) or not _is_valid_mips_word(first): + return False + cap = min(load_addr + size, producer_hi, target + 0x4000) + work = deque([target]) + visited = set() + saw_return = False + bad = False + + def inside(addr: int) -> bool: + return target <= addr < cap and (addr & 3) == 0 + + def add_delay(pc: int) -> None: + nonlocal bad + delay = pc + 4 + if not inside(delay): + bad = True + return + word = _word_at(data, load_addr, delay) + if not _is_valid_mips_word(word): + bad = True + else: + visited.add(delay) + + while work and not bad and len(visited) < 2048: + pc = work.popleft() + if pc in visited: + continue + if not inside(pc): + bad = True # sequential fallthrough past the bounded probe + break + word = _word_at(data, load_addr, pc) + if not _is_valid_mips_word(word): + bad = True + break + visited.add(pc) + kind, branch_target = _classify_cf(pc, word) + if kind == 'normal': + work.append(pc + 4) + elif kind == 'branch': + add_delay(pc) + work.append(pc + 8) # conditional fallthrough must remain bounded + if inside(branch_target): + work.append(branch_target) + # An out-of-probe branch target is a shared external code edge. + elif kind in ('jal', 'jalr'): + add_delay(pc) + work.append(pc + 8) + elif kind == 'j': + add_delay(pc) + if inside(branch_target): + work.append(branch_target) + # Otherwise this is a bounded tail edge. + elif kind == 'jr_ra': + add_delay(pc) + saw_return = True + elif kind == 'jr': + add_delay(pc) # bounded indirect/tail exit + + return saw_return and not bad and not work and len(visited) < 2048 + + def _walk_overlay_function(data: bytes, load_addr: int, size: int, - entry: int, hard_cap: int) -> dict: + entry: int, hard_cap: int, + producer_ranges=()) -> dict: lo = load_addr hi = load_addr + size @@ -577,6 +650,37 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, static_indirect_targets = set() branch_targets = set() jump_table_targets = set() + rejected_cross_producer_calls = set() + accepted_cross_producer_calls = set() + + def producer_for(addr: int): + for range_lo, range_hi in producer_ranges: + if range_lo <= addr < range_hi: + return range_lo, range_hi + return None + + def callable_transfer_target(source: int, target: int) -> bool: + if not producer_ranges: + return True + source_range = producer_for(source) + target_range = producer_for(target) + if source_range is not None and source_range == target_range: + return True + # Adjacent-producer composites are a byte envelope, not proof that a + # call encoded by producer A is linked to arbitrary bytes at the same + # address in every producer-B variant. Cross-boundary calls therefore + # need independent local boundary evidence. This keeps real exports + # while rejecting pointer/data words that happen to be valid MIPS. + if _callable_direct_jal_target(data, load_addr, target): + accepted_cross_producer_calls.add(target) + return True + if (target_range is not None and + _plausible_callable_target( + data, load_addr, size, target, target_range[1])): + accepted_cross_producer_calls.add(target) + return True + rejected_cross_producer_calls.add(target) + return False while work: pc = work.popleft() @@ -614,7 +718,8 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, work.append(pc + 8) branch_targets.add(pc + 8) if (lo <= target < hi and (target & 3) == 0 and - _is_valid_mips_word(_word_at(data, load_addr, target))): + _is_valid_mips_word(_word_at(data, load_addr, target)) and + callable_transfer_target(pc, target)): direct_jals.add(target) elif kind == 'jalr': if in_function(delay): @@ -625,7 +730,8 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, target_reg = (word >> 21) & 0x1F target = _resolve_constant_transfer(data, load_addr, entry, pc, target_reg) - if target is not None and lo <= target < hi and (target & 3) == 0: + if (target is not None and lo <= target < hi and (target & 3) == 0 + and callable_transfer_target(pc, target)): static_indirect_targets.add(target) elif kind == 'jr': if in_function(delay): @@ -636,7 +742,7 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, if in_function(target): branch_targets.add(target) work.append(target) - else: + elif callable_transfer_target(pc, target): static_indirect_targets.add(target) else: for jt in _find_jump_table_targets(data, load_addr, size, @@ -655,6 +761,8 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, 'static_indirect_targets': static_indirect_targets, 'branch_targets': branch_targets, 'jump_table_targets': jump_table_targets, + 'rejected_cross_producer_calls': rejected_cross_producer_calls, + 'accepted_cross_producer_calls': accepted_cross_producer_calls, } @@ -685,6 +793,24 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, lo = load_addr hi = load_addr + size region = lambda a: lo <= a < hi and (a & 3) == 0 + producer_ranges = [] + for raw_range in cap.get('producer_ranges', []) or []: + if not isinstance(raw_range, dict): + raise RuntimeError('producer_ranges entries must be objects') + try: + range_lo = _parse_addr(raw_range['start']) + range_hi = _parse_addr(raw_range['end']) + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeError(f'invalid producer_ranges entry: {raw_range}') from exc + if not (lo <= range_lo < range_hi <= hi): + raise RuntimeError( + f'producer range 0x{range_lo:08X}..0x{range_hi:08X} ' + f'outside capture 0x{lo:08X}..0x{hi:08X}') + producer_ranges.append((range_lo, range_hi)) + producer_ranges.sort() + if any(producer_ranges[i-1][1] > producer_ranges[i][0] + for i in range(1, len(producer_ranges))): + raise RuntimeError('producer_ranges overlap') schema_keys = {'executed_pcs', 'observed_pcs', 'dispatch_entry_pcs', 'function_entry_pcs'} @@ -714,7 +840,8 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, pre_roots_sorted = sorted(a for a in pre_roots if region(a)) for i, entry in enumerate(pre_roots_sorted): hard_cap = pre_roots_sorted[i + 1] if i + 1 < len(pre_roots_sorted) else hi - walk = _walk_overlay_function(data, load_addr, size, entry, hard_cap) + walk = _walk_overlay_function( + data, load_addr, size, entry, hard_cap, producer_ranges) jump_table_targets.update(walk['jump_table_targets']) def impossible_entry_start(addr: int) -> bool: @@ -800,7 +927,8 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, processed.add(entry) sorted_known = sorted(known) hard_cap = next((x for x in sorted_known if x > entry), hi) - walk = _walk_overlay_function(data, load_addr, size, entry, hard_cap) + walk = _walk_overlay_function( + data, load_addr, size, entry, hard_cap, producer_ranges) all_branch_targets.update(walk['branch_targets']) all_branch_targets.update(walk['jump_table_targets']) for target in sorted(walk['direct_jals']): @@ -828,7 +956,8 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, sorted_known = sorted(known) for i, entry in enumerate(sorted_known): hard_cap = sorted_known[i + 1] if i + 1 < len(sorted_known) else hi - walk = _walk_overlay_function(data, load_addr, size, entry, hard_cap) + walk = _walk_overlay_function( + data, load_addr, size, entry, hard_cap, producer_ranges) covered |= walk['visited'] promoted = sorted(a for a, r in included.items() if r == 'DISPATCH_INTERIOR' and a not in covered @@ -847,12 +976,19 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, # Re-walk with the final function set so the branch-target exclusion count # matches the actual compilation boundaries. all_branch_targets.clear() + rejected_cross_producer_calls = set() + accepted_cross_producer_calls = set() sorted_known = sorted(known) for i, entry in enumerate(sorted_known): hard_cap = sorted_known[i + 1] if i + 1 < len(sorted_known) else hi - walk = _walk_overlay_function(data, load_addr, size, entry, hard_cap) + walk = _walk_overlay_function( + data, load_addr, size, entry, hard_cap, producer_ranges) all_branch_targets.update(walk['branch_targets']) all_branch_targets.update(walk['jump_table_targets']) + rejected_cross_producer_calls.update( + walk['rejected_cross_producer_calls']) + accepted_cross_producer_calls.update( + walk['accepted_cross_producer_calls']) candidates = {a for a in (executed_pcs | dispatch_entry_pcs | captured_function_entries | legacy_seeds | toml_entries) @@ -885,6 +1021,9 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, 'branch_targets_excluded_count': len(all_branch_targets - set(included)), 'counts': counts, 'excluded_counts': excluded_counts, + 'producer_ranges': producer_ranges, + 'rejected_cross_producer_calls': rejected_cross_producer_calls, + 'accepted_cross_producer_calls': accepted_cross_producer_calls, } # Interior entries carry the 'interior' marker so the recompiler emits # them as overlapping aliases, never as walk roots. Promoted kernel @@ -899,7 +1038,12 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, if r in ('DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET'): return f'call_root 0x{addr:08X}' return f'0x{addr:08X}' - seeds = [seed_line(addr) for addr in sorted(included)] + seeds = [] + for range_lo, range_hi in producer_ranges: + seeds.append(f'producer_range 0x{range_lo:08X} 0x{range_hi:08X}') + for addr in sorted(accepted_cross_producer_calls): + seeds.append(f'cross_call_allow 0x{addr:08X}') + seeds.extend(seed_line(addr) for addr in sorted(included)) return seeds, audit @@ -915,6 +1059,10 @@ def print_seed_audit(audit: dict) -> None: print(f'toml_entries_included: {audit["counts"].get("TOML_DECLARED_ENTRY", 0)}') print(f'dispatch_interior_included: {audit["counts"].get("DISPATCH_INTERIOR", 0)}') print(f'dispatch_roots_promoted: {audit["counts"].get("DISPATCH_ROOT", 0)}') + print(f'cross_producer_calls_rejected: ' + f'{len(audit["rejected_cross_producer_calls"])}') + print(f'cross_producer_calls_accepted: ' + f'{len(audit["accepted_cross_producer_calls"])}') print(f'branch_targets_excluded: {audit["branch_targets_excluded_count"]}') print(f'observed_only_excluded: {audit["excluded_counts"].get("OBSERVED_PC_ONLY", 0)}') print(f'unknown_excluded: {audit["excluded_counts"].get("UNKNOWN", 0)}') From 65d89a06d7eaf385693e91d9bd2566dbcb852e9f Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 02:56:53 -0700 Subject: [PATCH 12/37] runtime: compact persisted capture addenda --- docs/AOT_OVERLAY_PLAN.md | 15 ++- runtime/src/overlay_capture.c | 56 +++++++++- tools/aot_overlay_spike/README.md | 22 +++- tools/coverage_vault.py | 170 ++++++++++++++++++++++++++++-- tools/test_coverage_vault.py | 98 +++++++++++++++++ 5 files changed, 346 insertions(+), 15 deletions(-) create mode 100644 tools/test_coverage_vault.py diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index 027b5b64..5bd6b002 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -131,9 +131,13 @@ compiler, but it is published through a same-directory temporary file plus atomi replace so readers never observe a half-written manifest. Games may additionally set `overlay_capture_history = true` to append each changed coherent snapshot as an independent JSONL record to `overlay_captures.addendum.jsonl` beside the -executable. A hard kill can damage only the last line; the next append quarantines -that tail, and `tools/coverage_vault.py merge --addendum ...` ignores malformed -lines while additively merging every valid record. +executable. Without a persist directory the v1 record embeds the full snapshot. +With dev immutable persistence enabled, a compact v2 record references the +atomically published JSON and vault merge verifies its FNV signature before use; +this avoids repeatedly embedding multi-megabyte base64 snapshots. A hard kill can +damage only the last line; the next append quarantines that tail, and +`tools/coverage_vault.py merge --addendum ...` ignores malformed records while +additively merging every valid snapshot. Development configs may also set a project-relative `overlay_capture_persist_dir`. This creates one atomically published, immutable @@ -141,6 +145,11 @@ JSON snapshot per changed capture. Absolute paths and `..` components are reject Production configs normally omit this key and retain only the naive executable- local addendum. Both history modes are off by default and contain game-derived bytes, so their outputs remain private/gitignored artifacts. +Previously embedded development histories can be converted to verified compact +references with `coverage_vault.py compact-addendum --addendum ... +--persist-dir ...`. The tool verifies every valid record against its exact +immutable snapshot before atomically replacing the addendum, and leaves the +source untouched if any snapshot is missing or mismatched. The recall scoreboard can roll a persisted live gap set forward after a verified monotonic static expansion with diff --git a/runtime/src/overlay_capture.c b/runtime/src/overlay_capture.c index 516be488..fbf17b95 100644 --- a/runtime/src/overlay_capture.c +++ b/runtime/src/overlay_capture.c @@ -492,6 +492,55 @@ static int append_history_record(const char *snapshot_path, const char *reason, return ok; } +static int write_json_string(FILE *dst, const char *value) +{ + const unsigned char *p = (const unsigned char *)(value ? value : ""); + if (fputc('"', dst) == EOF) return 0; + for (; *p; p++) { + if (*p == '"' || *p == '\\') { + if (fputc('\\', dst) == EOF || fputc(*p, dst) == EOF) return 0; + } else if (*p < 0x20) { + if (fprintf(dst, "\\u%04X", (unsigned)*p) < 0) return 0; + } else if (fputc(*p, dst) == EOF) { + return 0; + } + } + return fputc('"', dst) != EOF; +} + +/* Dev persistence already owns an atomically published immutable full JSON + * snapshot. Append only a durable reference to it instead of embedding the + * same multi-megabyte capture array again every autocap interval. Production + * configs without persist_dir continue to use the self-contained v1 record. */ +static int append_history_reference(const char *persist_path, const char *reason, + uint64_t sig, uint32_t sequence) +{ + FILE *dst, *tail; + int c, needs_separator = 0, ok = 1; + tail = fopen(s_history_addendum, "rb"); + if (tail) { + if (fseek(tail, -1, SEEK_END) == 0) { + c = fgetc(tail); + needs_separator = c != '\n'; + } + fclose(tail); + } + dst = fopen(s_history_addendum, "ab"); + if (!dst) return 0; + if (needs_separator && fputc('\n', dst) == EOF) ok = 0; + if (ok && fprintf(dst, + "{\"schema\":\"psxrecomp overlay capture addendum v2\"," + "\"game\":\"%s\",\"session\":\"%s\",\"sequence\":%u," + "\"reason\":\"%s\",\"fnv64\":\"%016llX\",\"snapshot\":", + s_history_game_id, s_history_session, sequence, + reason ? reason : "snapshot", (unsigned long long)sig) < 0) ok = 0; + if (ok) ok = write_json_string(dst, persist_path); + if (ok && fputs("}\n", dst) == EOF) ok = 0; + if (ok) ok = sync_file(dst); + if (fclose(dst) != 0) ok = 0; + return ok; +} + static void persist_history_snapshot(const char *snapshot_path, const char *reason, uint64_t sig) { @@ -513,7 +562,12 @@ static void persist_history_snapshot(const char *snapshot_path, "psxrecomp: failed to persist overlay capture snapshot %s\n", persist_path); } - addendum_saved = append_history_record(snapshot_path, reason, sig, sequence); + if (s_history_persist_dir[0] && persist_saved) + addendum_saved = append_history_reference( + persist_path, reason, sig, sequence); + else + addendum_saved = append_history_record( + snapshot_path, reason, sig, sequence); if (!addendum_saved) fprintf(stderr, "psxrecomp: failed to append overlay capture history %s\n", s_history_addendum); diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 40973158..06dfb77b 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -49,10 +49,13 @@ misses, production autocompile (tcc/gcc) self-heals on first visit. The canonical `overlay_captures.json` is atomically replaced for live consumers. With `[runtime] overlay_capture_history = true`, each changed coherent snapshot is -also appended as one independent record to -`overlay_captures.addendum.jsonl` beside the executable. A torn final line does -not invalidate earlier or later records. Merge it into the private additive vault -with: +also represented by an independent record in `overlay_captures.addendum.jsonl` +beside the executable. Production's simple mode embeds the full snapshot (v1). +When an immutable persist directory is configured, dev mode writes a small v2 +record referencing the already-atomically-published snapshot instead of duplicating +megabytes of base64 every autocap interval. Vault merge verifies the referenced +file's FNV signature before ingesting it. A torn final line does not invalidate +earlier or later records. Merge it into the private additive vault with: ```sh python tools/coverage_vault.py merge --vault \ @@ -62,6 +65,17 @@ python tools/coverage_vault.py merge --vault \ Dev configs can additionally set the project-relative `overlay_capture_persist_dir` to retain immutable per-snapshot JSON files. Both settings are opt-in; capture artifacts contain game bytes and must stay ignored. +Legacy dev addenda that embedded full snapshots before v2 can be compacted only +after every immutable copy passes its signature check: + +```sh +python tools/coverage_vault.py compact-addendum \ + --addendum /overlay_captures.addendum.jsonl \ + --persist-dir +``` + +The replacement is atomic and aborts without changing the addendum if any valid +record lacks its exact immutable snapshot. ## Multi-game sweep findings (2026-07-17) diff --git a/tools/coverage_vault.py b/tools/coverage_vault.py index 22c9bda6..0df9f4fb 100644 --- a/tools/coverage_vault.py +++ b/tools/coverage_vault.py @@ -22,6 +22,8 @@ Usage: coverage_vault.py merge --vault DIR [--captures captures.json] [--addendum captures.addendum.jsonl] [--cache CACHE_DIR] coverage_vault.py stats --vault DIR + coverage_vault.py compact-addendum --addendum captures.addendum.jsonl + --persist-dir IMMUTABLE_DIR """ import argparse, json, os, shutil, hashlib, sys @@ -51,6 +53,7 @@ def _load_addendum(path): quarantines the bad tail with a newline. """ regions = [] + seen_refs = set() if not path or not os.path.exists(path): return regions with open(path, encoding="utf-8", errors="replace") as history: @@ -63,16 +66,56 @@ def _load_addendum(path): print(" warn: ignoring invalid addendum line %d in %s (%s)" % (lineno, path, e)) continue - if not isinstance(record, dict) or record.get("schema") != ( - "psxrecomp overlay capture addendum v1"): + if not isinstance(record, dict): print(" warn: ignoring unknown addendum record on line %d in %s" % (lineno, path)) continue - captures = record.get("captures", []) - if isinstance(captures, list): - regions.extend(r for r in captures if isinstance(r, dict)) + schema = record.get("schema") + if schema == "psxrecomp overlay capture addendum v1": + captures = record.get("captures", []) + if isinstance(captures, list): + regions.extend(r for r in captures if isinstance(r, dict)) + continue + if schema == "psxrecomp overlay capture addendum v2": + snapshot = record.get("snapshot") + expected = str(record.get("fnv64", "")).upper() + if not isinstance(snapshot, str) or not snapshot: + print(" warn: v2 addendum line %d has no snapshot" % lineno) + continue + if not os.path.isabs(snapshot): + snapshot = os.path.join(os.path.dirname(path), snapshot) + ref_key = (os.path.normcase(os.path.abspath(snapshot)), expected) + if ref_key in seen_refs: + continue + seen_refs.add(ref_key) + if not os.path.exists(snapshot): + print(" warn: v2 snapshot missing on line %d: %s" % + (lineno, snapshot)) + continue + actual = _fnv64_file(snapshot) + if expected and expected != "%016X" % actual: + print(" warn: v2 snapshot signature mismatch on line %d: %s" % + (lineno, snapshot)) + continue + regions.extend(r for r in _load_list(snapshot) + if isinstance(r, dict)) + continue + print(" warn: ignoring unknown addendum record on line %d in %s" % + (lineno, path)) return regions +def _fnv64_file(path): + value = 1469598103934665603 + with open(path, "rb") as source: + while True: + chunk = source.read(65536) + if not chunk: + break + for byte in chunk: + value ^= byte + value = (value * 1099511628211) & 0xFFFFFFFFFFFFFFFF + return value + def _write_list_atomic(path, value): os.makedirs(os.path.dirname(path) or ".", exist_ok=True) tmp = "%s.tmp-%d" % (path, os.getpid()) @@ -87,6 +130,111 @@ def _write_list_atomic(path, value): if os.path.exists(tmp): os.unlink(tmp) +def _verified_v2_record(record, addendum, lineno): + snapshot = record.get("snapshot") + expected = str(record.get("fnv64", "")).upper() + if not isinstance(snapshot, str) or not snapshot: + raise ValueError("v2 addendum line %d has no snapshot" % lineno) + if not os.path.isabs(snapshot): + snapshot = os.path.join(os.path.dirname(addendum), snapshot) + snapshot = os.path.abspath(snapshot) + if not os.path.isfile(snapshot): + raise ValueError("v2 snapshot missing on line %d: %s" % + (lineno, snapshot)) + actual = "%016X" % _fnv64_file(snapshot) + if not expected or expected != actual: + raise ValueError("v2 snapshot signature mismatch on line %d: %s" % + (lineno, snapshot)) + result = dict(record) + result["fnv64"] = actual + result["snapshot"] = snapshot + return result + +def compact_addendum(addendum, persist_dir): + """Atomically replace embedded v1 snapshots with verified v2 references. + + Immutable snapshots are the authority during this operation. Every valid + v1 record must have its exact runtime-named file and matching FNV signature; + otherwise the original addendum is left byte-for-byte untouched. + """ + if not addendum or not os.path.isfile(addendum): + raise ValueError("addendum does not exist: %s" % addendum) + if not persist_dir or not os.path.isdir(persist_dir): + raise ValueError("persist directory does not exist: %s" % persist_dir) + addendum = os.path.abspath(addendum) + persist_dir = os.path.abspath(persist_dir) + old_bytes = os.path.getsize(addendum) + records = [] + seen_refs = set() + v1_count = v2_count = invalid_count = duplicate_count = 0 + with open(addendum, encoding="utf-8", errors="replace") as history: + for lineno, line in enumerate(history, 1): + if not line.strip(): + continue + try: + record = json.loads(line) + except Exception as e: + print(" warn: dropping invalid addendum line %d (%s)" % + (lineno, e)) + invalid_count += 1 + continue + if not isinstance(record, dict): + raise ValueError("unknown addendum record on line %d" % lineno) + schema = record.get("schema") + if schema == "psxrecomp overlay capture addendum v1": + game = record.get("game") + session = record.get("session") + sequence = record.get("sequence") + expected = str(record.get("fnv64", "")).upper() + if (not isinstance(game, str) or not game or + not isinstance(session, str) or not session or + not isinstance(sequence, int) or sequence < 0 or + len(expected) != 16 or + any(c not in "0123456789ABCDEF" for c in expected)): + raise ValueError("malformed v1 metadata on line %d" % lineno) + basename = "%s_%s_%04d_%s.json" % ( + game, session, sequence, expected) + snapshot = os.path.join(persist_dir, basename) + converted = dict(record) + converted.pop("captures", None) + converted["schema"] = "psxrecomp overlay capture addendum v2" + converted["snapshot"] = snapshot + converted = _verified_v2_record(converted, addendum, lineno) + v1_count += 1 + elif schema == "psxrecomp overlay capture addendum v2": + converted = _verified_v2_record(record, addendum, lineno) + v2_count += 1 + else: + raise ValueError("unknown addendum schema on line %d: %r" % + (lineno, schema)) + ref_key = (os.path.normcase(converted["snapshot"]), + converted["fnv64"]) + if ref_key in seen_refs: + duplicate_count += 1 + continue + seen_refs.add(ref_key) + records.append(converted) + + tmp = "%s.tmp-%d" % (addendum, os.getpid()) + try: + with open(tmp, "w", encoding="utf-8", newline="\n") as out: + for record in records: + out.write(json.dumps(record, separators=(",", ":"))) + out.write("\n") + out.flush() + os.fsync(out.fileno()) + os.replace(tmp, addendum) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + new_bytes = os.path.getsize(addendum) + print("coverage_vault: compacted %s" % addendum) + print(" records: %d v1 converted, %d v2 retained, %d duplicates and %d invalid lines dropped" % + (v1_count, v2_count, duplicate_count, invalid_count)) + print(" bytes: %d -> %d (saved %d)" % + (old_bytes, new_bytes, old_bytes - new_bytes)) + return v1_count, v2_count, duplicate_count, invalid_count + def merge_capture_regions(vault_json, src): if not src: return 0, 0 @@ -158,12 +306,20 @@ def cmd_stats(vault): def main(): ap = argparse.ArgumentParser() - ap.add_argument("cmd", choices=["merge", "stats"]) - ap.add_argument("--vault", required=True, help="vault directory (kept gitignored/private)") + ap.add_argument("cmd", choices=["merge", "stats", "compact-addendum"]) + ap.add_argument("--vault", help="vault directory (kept gitignored/private)") ap.add_argument("--captures", help="source overlay_captures.json to merge in") ap.add_argument("--addendum", help="append-only overlay capture history (.jsonl)") ap.add_argument("--cache", help="source cache dir (e.g. build/cache/) to merge in") + ap.add_argument("--persist-dir", help="immutable capture snapshot directory") a = ap.parse_args() + if a.cmd == "compact-addendum": + if not a.addendum or not a.persist_dir: + ap.error("compact-addendum requires --addendum and --persist-dir") + compact_addendum(a.addendum, a.persist_dir) + return 0 + if not a.vault: + ap.error("%s requires --vault" % a.cmd) if a.cmd == "stats": cmd_stats(a.vault) return 0 diff --git a/tools/test_coverage_vault.py b/tools/test_coverage_vault.py new file mode 100644 index 00000000..a689cc28 --- /dev/null +++ b/tools/test_coverage_vault.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +import importlib.util +import json +import os +import pathlib +import tempfile +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "coverage_vault", ROOT / "tools" / "coverage_vault.py") +MOD = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MOD) + + +class CoverageVaultHistoryTests(unittest.TestCase): + def test_v1_and_verified_v2_records_survive_a_torn_tail(self): + with tempfile.TemporaryDirectory() as tmp: + snapshot = os.path.join(tmp, "immutable.json") + addendum = os.path.join(tmp, "history.jsonl") + v1_region = {"load_addr": "0x80010000", "bytes_b64": "AA=="} + v2_region = {"load_addr": "0x80100000", "bytes_b64": "AQ=="} + with open(snapshot, "w", encoding="utf-8", newline="\n") as out: + json.dump([v2_region], out) + signature = "%016X" % MOD._fnv64_file(snapshot) + records = [ + {"schema": "psxrecomp overlay capture addendum v1", + "captures": [v1_region]}, + {"schema": "psxrecomp overlay capture addendum v2", + "snapshot": snapshot, "fnv64": signature}, + # A repeated reference is idempotent and should not inflate RAM. + {"schema": "psxrecomp overlay capture addendum v2", + "snapshot": snapshot, "fnv64": signature}, + # A bad signature must fail closed instead of ingesting corruption. + {"schema": "psxrecomp overlay capture addendum v2", + "snapshot": snapshot, "fnv64": "0000000000000000"}, + ] + with open(addendum, "w", encoding="utf-8", newline="\n") as out: + for record in records: + out.write(json.dumps(record) + "\n") + out.write('{"schema":"torn') + self.assertEqual(MOD._load_addendum(addendum), + [v1_region, v2_region]) + + def test_compact_addendum_converts_v1_and_is_idempotent(self): + with tempfile.TemporaryDirectory() as tmp: + persist = os.path.join(tmp, "immutable") + os.mkdir(persist) + addendum = os.path.join(tmp, "history.jsonl") + snapshot = os.path.join( + persist, "GAME_session_0001_0000000000000000.json") + region = {"load_addr": "0x80010000", "bytes_b64": "AA=="} + with open(snapshot, "w", encoding="utf-8", newline="\n") as out: + json.dump([region], out) + signature = "%016X" % MOD._fnv64_file(snapshot) + corrected = os.path.join( + persist, "GAME_session_0001_%s.json" % signature) + os.replace(snapshot, corrected) + v1 = {"schema": "psxrecomp overlay capture addendum v1", + "game": "GAME", "session": "session", "sequence": 1, + "reason": "autocap", "fnv64": signature, + "captures": [region]} + with open(addendum, "w", encoding="utf-8", newline="\n") as out: + out.write(json.dumps(v1) + "\n") + out.write('{"schema":"torn') + self.assertEqual(MOD.compact_addendum(addendum, persist), + (1, 0, 0, 1)) + with open(addendum, encoding="utf-8") as source: + compacted = json.loads(source.readline()) + self.assertNotIn("captures", compacted) + self.assertEqual(compacted["snapshot"], corrected) + self.assertEqual(MOD._load_addendum(addendum), [region]) + self.assertEqual(MOD.compact_addendum(addendum, persist), + (0, 1, 0, 0)) + self.assertEqual(MOD._load_addendum(addendum), [region]) + + def test_compact_addendum_failure_preserves_source(self): + with tempfile.TemporaryDirectory() as tmp: + persist = os.path.join(tmp, "immutable") + os.mkdir(persist) + addendum = os.path.join(tmp, "history.jsonl") + v1 = {"schema": "psxrecomp overlay capture addendum v1", + "game": "GAME", "session": "missing", "sequence": 1, + "reason": "autocap", "fnv64": "1234567890ABCDEF", + "captures": []} + original = (json.dumps(v1) + "\n").encode() + with open(addendum, "wb") as out: + out.write(original) + with self.assertRaisesRegex(ValueError, "snapshot missing"): + MOD.compact_addendum(addendum, persist) + with open(addendum, "rb") as source: + self.assertEqual(source.read(), original) + + +if __name__ == "__main__": + unittest.main() From 9ab17d084bd17d23495dde8ef021a42a7504a9da Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 03:24:56 -0700 Subject: [PATCH 13/37] aot: close guarded BIOS vector coverage gaps --- recompiler/src/full_function_emitter.cpp | 58 +++++++++- tools/aot_overlay_spike/coverage_report.py | 109 ++++++++++++++---- .../aot_overlay_spike/test_coverage_report.py | 26 ++++- 3 files changed, 164 insertions(+), 29 deletions(-) diff --git a/recompiler/src/full_function_emitter.cpp b/recompiler/src/full_function_emitter.cpp index b838ce92..587a7946 100644 --- a/recompiler/src/full_function_emitter.cpp +++ b/recompiler/src/full_function_emitter.cpp @@ -1421,6 +1421,51 @@ void FullFunctionEmitter::emit_dispatch( out += fmt::format("const uint32_t psx_bios_kernel_body_count = {}u;\n\n", kb_count); } + // --- Runtime-installed BIOS call-vector stubs --- + // Every retail PSX BIOS installs the A0/B0/C0 ABI gates as the same + // four-instruction shape: lui/addiu $t0,target; jr $t0; nop. The target is + // BIOS-specific, so decode it from live RAM and require the complete shape + // before taking the native tail transfer. A game/BIOS patch that changes + // even one word fails closed to dirty_ram_interp below. + out += "typedef struct {\n"; + out += " uint32_t key;\n"; + out += " uint32_t body_lo;\n"; + out += " uint32_t body_hi;\n"; + out += "} PsxNativeStub;\n\n"; + out += "const PsxNativeStub psx_bios_native_stubs[3] = {\n"; + out += " { 0x000000A0u, 0x000000A0u, 0x000000B0u },\n"; + out += " { 0x000000B0u, 0x000000B0u, 0x000000C0u },\n"; + out += " { 0x000000C0u, 0x000000C0u, 0x000000D0u },\n"; + out += "};\n"; + out += "const uint32_t psx_bios_native_stub_count = 3u;\n\n"; + out += "static int psx_bios_try_native_call_stub(CPUState* cpu, " + "uint32_t addr) {\n"; + out += " uint32_t phys = addr & 0x1FFFFFFFu;\n"; + out += " if (phys != 0xA0u && phys != 0xB0u && phys != 0xC0u) return 0;\n"; + out += " uint32_t w0 = cpu->read_word(phys + 0u);\n"; + out += " uint32_t w1 = cpu->read_word(phys + 4u);\n"; + out += " uint32_t w2 = cpu->read_word(phys + 8u);\n"; + out += " uint32_t w3 = cpu->read_word(phys + 12u);\n"; + out += " if ((w0 & 0xFFFF0000u) != 0x3C080000u ||\n"; + out += " (w1 & 0xFFFF0000u) != 0x25080000u ||\n"; + out += " w2 != 0x01000008u || w3 != 0u) return 0;\n"; + out += "#ifdef PSX_ENABLE_BLOCK_CYCLES\n"; + out += " psx_icache_fetch(cpu, addr);\n"; + out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", + psx_cyc_dep_res_mask(0x3C080000u)); + out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", + psx_cyc_dep_res_mask(0x25080000u)); + out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", + psx_cyc_dep_res_mask(0x01000008u)); + out += fmt::format(" psx_cyc_step(cpu, 0x{:X}u);\n", + psx_cyc_dep_res_mask(0x00000000u)); + out += "#endif\n"; + out += " cpu->gpr[8] = ((w0 & 0xFFFFu) << 16) +\n"; + out += " (uint32_t)(int32_t)(int16_t)(w1 & 0xFFFFu);\n"; + out += " cpu->pc = cpu->gpr[8];\n"; + out += " return 1;\n"; + out += "}\n\n"; + // Dispatch function with binary search. out += "static uint32_t normalize(uint32_t addr) {\n"; out += " uint32_t phys = addr & 0x1FFFFFFFu;\n"; @@ -1475,11 +1520,14 @@ void FullFunctionEmitter::emit_dispatch( out += " * backends. Handled (rc 1) \xE2\x87\x92 the service completed against guest\n"; out += " * state and the guest resumes at $ra via the trampoline's normal\n"; out += " * return/tail contract below. rc 0 \xE2\x87\x92 pure LLE fall-through. */\n"; - out += " if (g_psx_bios_hle_hook &&\n"; - out += " g_psx_bios_hle_hook(cpu, addr & 0x1FFFFFFFu)) {\n"; - out += " cpu->pc = cpu->gpr[31];\n"; - out += " found = 1;\n"; - out += " }\n"; + out += " if (g_psx_bios_hle_hook &&\n"; + out += " g_psx_bios_hle_hook(cpu, addr & 0x1FFFFFFFu)) {\n"; + out += " cpu->pc = cpu->gpr[31];\n"; + out += " found = 1;\n"; + out += " }\n"; + out += " /* Byte-guarded A0/B0/C0 call-vector tail stubs. */\n"; + out += " if (!found && psx_bios_try_native_call_stub(cpu, addr))\n"; + out += " found = 1;\n"; out += "#ifdef PSX_HAS_GAME_DISPATCH\n"; out += " /* Game EXEs can overlap the BIOS shell copy window at\n"; out += " * physical 0x30000-0x5AFFF. If the target belongs to the\n"; diff --git a/tools/aot_overlay_spike/coverage_report.py b/tools/aot_overlay_spike/coverage_report.py index 55294c1e..56d97049 100644 --- a/tools/aot_overlay_spike/coverage_report.py +++ b/tools/aot_overlay_spike/coverage_report.py @@ -25,7 +25,7 @@ Usage: --out-md --out-json [--game ] Cache dirs are scanned recursively for *.ranges; a captures json is the v2 schema. """ -import argparse, bisect, os, sys, json, re, glob +import argparse, bisect, os, sys, json, re, glob, zlib from collections import defaultdict MASK = 0x1FFFFFFF @@ -45,21 +45,61 @@ def merge_intervals(intervals): merged[-1][1] = hi return [(lo, hi) for lo, hi in merged] -def parse_ranges_dir(d): - """Return ({entry_masked: code_crc}, merged compiled-code intervals).""" +def _zero_filled_crc(size): + crc = 0 + zeroes = bytes(min(size, 65536)) + remaining = size + while remaining: + take = min(remaining, len(zeroes)) + crc = zlib.crc32(zeroes[:take], crc) + remaining -= take + return crc & 0xFFFFFFFF + +def parse_ranges_dir(d, with_audit=False): + """Return entries/ranges, rejecting proven all-zero data-as-code records. + + A valid function cannot be an entire compiled range of MIPS NOP words: it + has no return or transfer. Historical runtime compilers nevertheless minted + such ranges from observed sequential PCs. The v2 manifest has enough proof + to quarantine them without needing the original capture bytes: F carries + the range CRC and its following R carries the exact byte count. + """ ent = {} intervals = [] + quarantined = set() for rf in glob.glob(os.path.join(d, '**', '*.ranges'), recursive=True): - for line in open(rf, errors='ignore'): - m = re.match(r'F\s+([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+)', line) - if m: - ent[norm(int(m.group(1), 16))] = int(m.group(2), 16) - continue - m = re.match(r'R\s+([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+)', line) - if m: - lo = norm(int(m.group(1), 16)) - intervals.append((lo, lo + int(m.group(2), 16))) - return ent, merge_intervals(intervals) + pending = None + with open(rf, errors='ignore') as manifest: + for line in manifest: + m = re.match(r'F\s+([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+)', line) + if m: + if pending is not None: + ent[pending[0]] = pending[1] + pending = (norm(int(m.group(1), 16)), int(m.group(2), 16)) + continue + m = re.match(r'R\s+([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+)', line) + if m: + lo = norm(int(m.group(1), 16)) + size = int(m.group(2), 16) + if (pending is not None and size > 0 and size % 4 == 0 and + pending[1] == _zero_filled_crc(size)): + quarantined.add((pending[0], pending[1], size)) + else: + if pending is not None: + ent[pending[0]] = pending[1] + intervals.append((lo, lo + size)) + pending = None + if pending is not None: + ent[pending[0]] = pending[1] + result = (ent, merge_intervals(intervals)) + if not with_audit: + return result + audit = [ + {'entry': f'0x{entry | 0x80000000:08X}', + 'code_crc': f'{crc:08X}', 'size': size} + for entry, crc, size in sorted(quarantined) + ] + return result + (audit,) def _c_initializer(text, declaration_pattern): """Return the braced initializer following a unique C declaration.""" @@ -103,6 +143,18 @@ def parse_bios_dispatch(path): lo, hi = norm(int(lo, 16)), norm(int(hi, 16)) if hi > lo: intervals.append((lo, hi)) + try: + native_stubs = _c_initializer( + text, r'\bPsxNativeStub\s+psx_bios_native_stubs\s*\[[^]]*\]\s*=') + except ValueError: + native_stubs = '' + for key, lo, hi in re.findall( + r'\{\s*0x([0-9A-Fa-f]+)u?\s*,\s*' + r'0x([0-9A-Fa-f]+)u?\s*,\s*0x([0-9A-Fa-f]+)u?\s*\}', native_stubs): + key, lo, hi = norm(int(key, 16)), norm(int(lo, 16)), norm(int(hi, 16)) + if hi > lo and lo <= key < hi: + entries.add(key) + intervals.append((lo, hi)) if not entries or not intervals: raise ValueError('generated BIOS dispatcher contained an empty static table') return entries, merge_intervals(intervals) @@ -167,8 +219,10 @@ def main(): if a.prior_report and not a.assume_static_superset: ap.error('--prior-report requires the explicit --assume-static-superset assertion') - static_ent, static_ranges = parse_ranges_dir(a.static) + static_ent, static_ranges, static_zero = parse_ranges_dir( + a.static, with_audit=True) report = {'game': a.game, 'static_entries': len(static_ent), 'sources': {}} + report['static_quarantined_zero_ranges'] = static_zero combined_ent = dict(static_ent) combined_ranges = static_ranges if a.bios_dispatch: @@ -188,6 +242,9 @@ def main(): 'extractor reproduce, and how much lies in compiled static code?_\n') md.append(f'- Static shard cache: `{a.static}`') md.append(f'- Static manifest entries: **{len(static_ent)}**\n') + if static_zero: + md.append(f'- Quarantined proven all-zero static ranges: ' + f'**{len(static_zero)}** (excluded as data, not code)\n') if a.bios_dispatch: md.append(f'- Base BIOS native dispatch entries: **{len(bios_ent)}**; ' f'relocated kernel body ranges: **{len(bios_ranges)}**') @@ -195,7 +252,8 @@ def main(): 'and the separately generated, live-byte-guarded base BIOS.\n') if a.vault: - vault_ent, _ = parse_ranges_dir(a.vault) + vault_ent, _, vault_zero = parse_ranges_dir(a.vault, with_audit=True) + report['vault_quarantined_zero_ranges'] = vault_zero r = recall(set(vault_ent), static_ent, static_ranges, vault_ent) report['sources']['vault'] = { k: v for k, v in r.items() if k not in ('misses', 'range_misses')} @@ -210,6 +268,9 @@ def main(): report['vault_combined_range_misses'] = [ f'0x{x|0x80000000:08X}' for x in combined['range_misses']] md.append('## vs played vault (most complete needed-set)\n') + if vault_zero: + md.append(f'- Quarantined proven all-zero played ranges: ' + f'**{len(vault_zero)}** (excluded from the needed set)') md.append(f'- Manifest entries in the full-playthrough vault: **{r["needed"]}**') md.append(f'- Discovered by static: **{r["covered_here"]}** ' f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') @@ -236,13 +297,17 @@ def main(): md.append(f'- **COMBINED CODE-RANGE GAPS: ' f'{combined["missed_code_range"]}**\n') md.append('#### Combined gaps grouped by region\n') - for reg, addrs in group_by_region(combined['range_misses']).items(): - md.append(f'- region `0x{reg|0x80000000:08X}`: {len(addrs)} misses') - md.append(' ```') - for i in range(0, len(addrs), 8): - md.append(' ' + ' '.join( - f'{x|0x80000000:08X}' for x in addrs[i:i+8])) - md.append(' ```') + grouped_combined = group_by_region(combined['range_misses']) + if not grouped_combined: + md.append('- None.') + else: + for reg, addrs in grouped_combined.items(): + md.append(f'- region `0x{reg|0x80000000:08X}`: {len(addrs)} misses') + md.append(' ```') + for i in range(0, len(addrs), 8): + md.append(' ' + ' '.join( + f'{x|0x80000000:08X}' for x in addrs[i:i+8])) + md.append(' ```') md.append('') md.append('### Code-range gaps grouped by overlay region\n') for reg, addrs in group_by_region(r['range_misses']).items(): diff --git a/tools/aot_overlay_spike/test_coverage_report.py b/tools/aot_overlay_spike/test_coverage_report.py index 28c67c7b..0d29cc6a 100644 --- a/tools/aot_overlay_spike/test_coverage_report.py +++ b/tools/aot_overlay_spike/test_coverage_report.py @@ -2,6 +2,7 @@ import os import tempfile import unittest +import zlib import coverage_report @@ -20,6 +21,9 @@ const PsxKernelBody psx_bios_kernel_bodies[2] = { { 0x00000500u, 0x00000500u, 0x00000510u }, { 0x00000510u, 0x00000510u, 0x00000520u }, }; +const PsxNativeStub psx_bios_native_stubs[1] = { + { 0x000000A0u, 0x000000A0u, 0x000000B0u }, +}; ''' with tempfile.NamedTemporaryFile( mode='w', suffix='.c', delete=False, encoding='utf-8') as generated: @@ -29,8 +33,8 @@ const PsxKernelBody psx_bios_kernel_bodies[2] = { entries, ranges = coverage_report.parse_bios_dispatch(path) finally: os.unlink(path) - self.assertEqual(entries, {0x500, 0x1FC00180}) - self.assertEqual(ranges, [(0x500, 0x520)]) + self.assertEqual(entries, {0xA0, 0x500, 0x1FC00180}) + self.assertEqual(ranges, [(0xA0, 0xB0), (0x500, 0x520)]) def test_code_range_recall_is_distinct_from_exact_entry_recall(self): result = coverage_report.recall( @@ -38,6 +42,24 @@ const PsxKernelBody psx_bios_kernel_bodies[2] = { self.assertEqual(result['covered_here'], 1) self.assertEqual(result['covered_by_code_range'], 2) + def test_all_zero_compiled_range_is_quarantined(self): + with tempfile.TemporaryDirectory() as tmp: + zero_size = 0x184 + zero_crc = zlib.crc32(bytes(zero_size)) & 0xFFFFFFFF + with open(os.path.join(tmp, 'mixed.ranges'), 'w', encoding='utf-8') as out: + out.write('# psxrecomp overlay code-range manifest v2\n') + out.write('F 8000EE7C %08X\n' % zero_crc) + out.write('R 8000EE7C %X\n' % zero_size) + out.write('F 80000100 12345678\n') + out.write('R 80000100 10\n') + entries, ranges, audit = coverage_report.parse_ranges_dir( + tmp, with_audit=True) + self.assertEqual(entries, {0x100: 0x12345678}) + self.assertEqual(ranges, [(0x100, 0x110)]) + self.assertEqual(audit, [{ + 'entry': '0x8000EE7C', 'code_crc': '%08X' % zero_crc, + 'size': zero_size}]) + if __name__ == '__main__': unittest.main() From e2a1037dad3b3e4206059c4ca0e559b1537096e7 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 04:16:51 -0700 Subject: [PATCH 14/37] aot: score verified capture history --- tools/aot_overlay_spike/coverage_report.py | 147 ++++++++++++++++-- .../aot_overlay_spike/test_coverage_report.py | 28 ++++ 2 files changed, 161 insertions(+), 14 deletions(-) diff --git a/tools/aot_overlay_spike/coverage_report.py b/tools/aot_overlay_spike/coverage_report.py index 56d97049..a83ffc0e 100644 --- a/tools/aot_overlay_spike/coverage_report.py +++ b/tools/aot_overlay_spike/coverage_report.py @@ -21,6 +21,7 @@ Usage: coverage_report.py --static --vault [--bios-dispatch ] [--captures ] + [--addendum ] [--prior-report --assume-static-superset] --out-md --out-json [--game ] Cache dirs are scanned recursively for *.ranges; a captures json is the v2 schema. @@ -171,6 +172,88 @@ def parse_captures_executed(path): ent.add(norm(int(a, 16))) return ent +def _fnv64_file(path): + value = 1469598103934665603 + with open(path, 'rb') as source: + while True: + chunk = source.read(65536) + if not chunk: + break + for byte in chunk: + value ^= byte + value = (value * 1099511628211) & 0xFFFFFFFFFFFFFFFF + return value + +def parse_addendum_entries(path): + """Union dispatch/function entries from verified v1/v2 history records.""" + entries = set() + seen_refs = set() + audit = {'v1_records': 0, 'v2_records': 0, 'invalid_records': 0, + 'duplicate_refs': 0, 'verified_snapshots': 0} + + def add_captures(captures): + if not isinstance(captures, list): + return + for cap in captures: + if not isinstance(cap, dict): + continue + for key in ('dispatch_entry_pcs', 'function_entry_pcs'): + for addr in cap.get(key, []): + entries.add(norm(int(addr, 16) if isinstance(addr, str) + else int(addr))) + + with open(path, encoding='utf-8', errors='replace') as history: + for lineno, line in enumerate(history, 1): + if not line.strip(): + continue + try: + record = json.loads(line) + except Exception as exc: + print(f' warn: ignoring invalid addendum line {lineno} ({exc})') + audit['invalid_records'] += 1 + continue + if not isinstance(record, dict): + audit['invalid_records'] += 1 + continue + schema = record.get('schema') + if schema == 'psxrecomp overlay capture addendum v1': + audit['v1_records'] += 1 + add_captures(record.get('captures')) + continue + if schema != 'psxrecomp overlay capture addendum v2': + audit['invalid_records'] += 1 + continue + audit['v2_records'] += 1 + snapshot = record.get('snapshot') + expected = str(record.get('fnv64', '')).upper() + if not isinstance(snapshot, str) or not snapshot: + audit['invalid_records'] += 1 + continue + if not os.path.isabs(snapshot): + snapshot = os.path.join(os.path.dirname(path), snapshot) + snapshot = os.path.abspath(snapshot) + ref = (os.path.normcase(snapshot), expected) + if ref in seen_refs: + audit['duplicate_refs'] += 1 + continue + seen_refs.add(ref) + if (not os.path.isfile(snapshot) or not expected or + expected != '%016X' % _fnv64_file(snapshot)): + print(f' warn: ignoring missing/mismatched snapshot on line ' + f'{lineno}: {snapshot}') + audit['invalid_records'] += 1 + continue + try: + with open(snapshot, encoding='utf-8') as source: + add_captures(json.load(source)) + except Exception as exc: + print(f' warn: ignoring unreadable snapshot on line {lineno} ' + f'({exc})') + audit['invalid_records'] += 1 + continue + audit['verified_snapshots'] += 1 + return entries, audit + def recall(needed_entries, static_ent, static_ranges, needed_crc=None): """needed_entries: set; static_ent: {entry->crc}. Returns dict of metrics.""" covered = set(static_ent) @@ -208,6 +291,7 @@ def main(): 'entries/ranges to separate combined metrics') ap.add_argument('--vault', help='played/coverage vault cache dir (.ranges)') ap.add_argument('--captures', help='runtime overlay_captures.json (executed_pcs)') + ap.add_argument('--addendum', help='append-only runtime capture history (.jsonl)') ap.add_argument('--prior-report', help='roll forward a persisted live gap set') ap.add_argument('--assume-static-superset', action='store_true', help='assert current static entries retain every prior static entry') @@ -318,8 +402,17 @@ def main(): md.append(' ```') md.append('') + live = set() + live_sources = [] if a.captures and os.path.exists(a.captures): - live = parse_captures_executed(a.captures) + live.update(parse_captures_executed(a.captures)) + live_sources.append('latest capture') + if a.addendum and os.path.exists(a.addendum): + addendum_entries, addendum_audit = parse_addendum_entries(a.addendum) + live.update(addendum_entries) + live_sources.append('verified append-only history') + report['live_addendum_audit'] = addendum_audit + if live_sources: r = recall(live, static_ent, static_ranges) report['sources']['live_session'] = { k: v for k, v in r.items() if k not in ('misses', 'range_misses')} @@ -333,8 +426,14 @@ def main(): k: v for k, v in combined.items() if k not in ('misses', 'range_misses')} report['live_combined_range_misses'] = [ f'0x{x|0x80000000:08X}' for x in combined['range_misses']] - md.append('## vs live session captures (this drive)\n') - md.append(f'- Dispatch entries this session exercised: **{r["needed"]}**') + md.append('## vs live capture history\n') + md.append(f'- Sources: **{" + ".join(live_sources)}**') + if a.addendum and 'live_addendum_audit' in report: + aa = report['live_addendum_audit'] + md.append(f'- Verified immutable snapshots: ' + f'**{aa["verified_snapshots"]}**; invalid records: ' + f'**{aa["invalid_records"]}**') + md.append(f'- Dispatch entries exercised: **{r["needed"]}**') md.append(f'- Discovered by static: **{r["covered_here"]}** ' f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') md.append(f'- Covered by compiled static code ranges: ' @@ -345,11 +444,19 @@ def main(): f'**{combined["covered_by_code_range"]}** ' f'(**{combined["recall_code_range"]*100:.1f}%**)') md.append(f'- **MISSED live: {r["missed"]}**\n') - elif a.prior_report: - with open(a.prior_report, encoding='utf-8') as prior_in: - prior = json.load(prior_in) - prior_live = prior.get('sources', {}).get('live_session') - prior_misses = prior.get('live_misses') + if a.prior_report: + if a.prior_report == '-': + prior = json.load(sys.stdin) + else: + with open(a.prior_report, encoding='utf-8') as prior_in: + prior = json.load(prior_in) + prior_sources = prior.get('sources', {}) + if 'prior_live_session' in prior_sources: + prior_live = prior_sources.get('prior_live_session') + prior_misses = prior.get('prior_live_misses') + else: + prior_live = prior_sources.get('live_session') + prior_misses = prior.get('live_misses') if not isinstance(prior_live, dict) or not isinstance(prior_misses, list): ap.error('--prior-report has no persisted live_session/live_misses') if len(static_ent) < int(prior.get('static_entries', 0)): @@ -371,9 +478,12 @@ def main(): 'recall_code_range': ((needed - len(range_miss))/needed if needed else 0.0)} r['provenance'] = ('rolled forward from the persisted live gap manifest; ' 'caller asserted the current static entry set is a superset') - report['sources']['live_session'] = r - report['live_misses'] = [f'0x{x|0x80000000:08X}' for x in sorted(miss)] - report['live_range_misses'] = [ + prior_key = 'prior_live_session' if live_sources else 'live_session' + prior_prefix = 'prior_live' if live_sources else 'live' + report['sources'][prior_key] = r + report[f'{prior_prefix}_misses'] = [ + f'0x{x|0x80000000:08X}' for x in sorted(miss)] + report[f'{prior_prefix}_range_misses'] = [ f'0x{x|0x80000000:08X}' for x in sorted(range_miss)] combined = None if a.bios_dispatch: @@ -395,10 +505,19 @@ def main(): 'recall_code_range': ( (needed - len(combined_range_miss))/needed if needed else 0.0), 'provenance': r['provenance']} - report['sources']['live_session_combined_bios'] = combined - report['live_combined_range_misses'] = [ + combined_key = ('prior_live_session_combined_bios' if live_sources + else 'live_session_combined_bios') + report['sources'][combined_key] = combined + report[f'{prior_prefix}_combined_range_misses'] = [ f'0x{x|0x80000000:08X}' for x in sorted(combined_range_miss)] - md.append('## vs persisted live-session gaps (monotonic roll-forward)\n') + heading = ('## vs pre-history persisted live-session gaps ' + '(separate monotonic roll-forward)\n' if live_sources else + '## vs persisted live-session gaps (monotonic roll-forward)\n') + md.append(heading) + if live_sources: + md.append('- This older needed set predates append-only history. It is ' + 'reported separately because the old report retained misses, ' + 'not enough addresses to compute an honest set union.\n') md.append(f'- Dispatch entries the persisted session exercised: **{needed}**') md.append(f'- Discovered by current static: **{covered}** ' f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') diff --git a/tools/aot_overlay_spike/test_coverage_report.py b/tools/aot_overlay_spike/test_coverage_report.py index 0d29cc6a..f6016235 100644 --- a/tools/aot_overlay_spike/test_coverage_report.py +++ b/tools/aot_overlay_spike/test_coverage_report.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import os +import json import tempfile import unittest import zlib @@ -60,6 +61,33 @@ const PsxNativeStub psx_bios_native_stubs[1] = { 'entry': '0x8000EE7C', 'code_crc': '%08X' % zero_crc, 'size': zero_size}]) + def test_addendum_unions_v1_and_verified_v2(self): + with tempfile.TemporaryDirectory() as tmp: + snapshot = os.path.join(tmp, 'immutable.json') + addendum = os.path.join(tmp, 'history.jsonl') + with open(snapshot, 'w', encoding='utf-8') as out: + json.dump([{'dispatch_entry_pcs': ['0x80000200']}], out) + signature = '%016X' % coverage_report._fnv64_file(snapshot) + records = [ + {'schema': 'psxrecomp overlay capture addendum v1', + 'captures': [{'function_entry_pcs': ['0x80000100']}]}, + {'schema': 'psxrecomp overlay capture addendum v2', + 'snapshot': snapshot, 'fnv64': signature}, + {'schema': 'psxrecomp overlay capture addendum v2', + 'snapshot': snapshot, 'fnv64': signature}, + {'schema': 'psxrecomp overlay capture addendum v2', + 'snapshot': snapshot, 'fnv64': '0000000000000000'}, + ] + with open(addendum, 'w', encoding='utf-8', newline='\n') as out: + for record in records: + out.write(json.dumps(record) + '\n') + out.write('{"schema":"torn') + entries, audit = coverage_report.parse_addendum_entries(addendum) + self.assertEqual(entries, {0x100, 0x200}) + self.assertEqual(audit, { + 'v1_records': 1, 'v2_records': 3, 'invalid_records': 2, + 'duplicate_refs': 1, 'verified_snapshots': 1}) + if __name__ == '__main__': unittest.main() From 4fb144a4fd9f1ec2142bc566c5b5542e397d8f8c Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 04:16:51 -0700 Subject: [PATCH 15/37] aot: preserve additive static discovery --- docs/AOT_OVERLAY_PLAN.md | 6 ++ recompiler/src/main_psx.cpp | 48 ++++++++++- .../tests/test_aot_overlay_discovery.py | 51 ++++++++++++ tools/aot_overlay_spike/README.md | 12 +++ tools/aot_overlay_spike/extract_generic.py | 11 ++- tools/compile_overlays.py | 80 ++++++++++++++++--- 6 files changed, 196 insertions(+), 12 deletions(-) diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index 5bd6b002..a4fb65eb 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -150,6 +150,9 @@ references with `coverage_vault.py compact-addendum --addendum ... --persist-dir ...`. The tool verifies every valid record against its exact immutable snapshot before atomically replacing the addendum, and leaves the source untouched if any snapshot is missing or mismatched. +`coverage_report.py --addendum ...` streams those verified history snapshots and +unions their dispatch/function entries into the live scoreboard, preserving +cross-launch recall without loading their game bytes into the static needed set. The recall scoreboard can roll a persisted live gap set forward after a verified monotonic static expansion with @@ -178,6 +181,9 @@ manufacture overlay shards merely to duplicate those already-native BIOS bodies. 4. Per-function identity = CRC32 over its exact compiled code ranges. Output: `_.dll` + a `.ranges` manifest (`F ` / `R `). + Rebuilding identical region bytes also imports prior overlapping alias groups + as non-root `retained_alias` bodies. This makes discovery additive: newly + proven frameless roots cannot hard-cap and erase an older indirect-entry body. 5. `--static` mode instead namespaces the C and folds variants into `overlays_static.c` linked directly into the binary, dispatched via a generated `psx_overlay_dispatch()` — **this is already an AOT delivery path**; it moves diff --git a/recompiler/src/main_psx.cpp b/recompiler/src/main_psx.cpp index bc9472af..0eb8f949 100644 --- a/recompiler/src/main_psx.cpp +++ b/recompiler/src/main_psx.cpp @@ -430,6 +430,7 @@ int main(int argc, char** argv) { * valid seeds. */ std::vector file_seeds; std::vector interior_seeds; + std::vector retained_alias_seeds; std::set trusted_root_seeds; std::set trusted_call_root_seeds; std::vector> producer_ranges; @@ -469,6 +470,24 @@ int main(int argc, char** argv) { cross_call_allow.insert(addr); continue; } + if (line.rfind("retained_alias", 0) == 0) { + std::istringstream in(line); + std::string tag, addr_text, lo_text, hi_text; + in >> tag >> addr_text >> lo_text >> hi_text; + uint32_t addr = static_cast( + std::strtoul(addr_text.c_str(), nullptr, 16)); + uint32_t lo = static_cast( + std::strtoul(lo_text.c_str(), nullptr, 16)); + uint32_t hi = static_cast( + std::strtoul(hi_text.c_str(), nullptr, 16)); + if (addr_text.empty() || lo_text.empty() || hi_text.empty() || + lo < seed_lo || addr < lo || addr >= hi || hi > seed_hi) { + fmt::print(stderr, "ERROR: invalid retained_alias: {}\n", line); + return 1; + } + retained_alias_seeds.push_back({addr, lo, hi}); + continue; + } bool interior = false; bool trusted_root = false; bool trusted_call_root = false; @@ -519,11 +538,12 @@ int main(int argc, char** argv) { } fmt::print("Loaded {} extra function addresses ({} interior, " "{} dispatch-root, {} call-root, {} producer ranges, " - "{} cross-call allows) from {}\n", + "{} cross-call allows, {} retained aliases) from {}\n", file_seeds.size() + interior_seeds.size(), interior_seeds.size(), trusted_root_seeds.size(), trusted_call_root_seeds.size(), producer_ranges.size(), cross_call_allow.size(), + retained_alias_seeds.size(), extra_funcs_path); } else { fmt::print("WARNING: Cannot open extra-funcs file: {}\n", extra_funcs_path); @@ -673,6 +693,32 @@ int main(int argc, char** argv) { fmt::print(" +{} in-function jal-target alias entries " "(range-ownership completeness)\n", jal_alias_added); + // An existing manifest for these exact image bytes is durable proof + // that its overlapping alias groups compiled successfully before. + // Preserve those groups without feeding their entries back as walk + // roots: roots would hard-cap newly discovered siblings, while a + // retained overlapping body lets old indirect entries and new static + // call roots coexist. Ordinary current functions win at an identical + // entry; retained aliases replace only an ordinary alias at that PC. + if (!retained_alias_seeds.empty()) { + std::set current_starts; + for (const auto& f : analysis_result.functions) + current_starts.insert(f.start_addr); + size_t retained_added = 0; + for (const auto& retained : retained_alias_seeds) { + if (current_starts.count(retained.addr)) continue; + alias_entries.erase( + std::remove_if(alias_entries.begin(), alias_entries.end(), + [&](const AliasEntry& ae) { return ae.addr == retained.addr; }), + alias_entries.end()); + alias_entries.push_back(retained); + retained_added++; + } + if (retained_added) + fmt::print(" +{} retained prior-manifest alias entries " + "(monotonic identical-image coverage)\n", retained_added); + } + materialize_alias_groups(analysis_result, alias_entries); fmt::print("Exact-entry alias entries emitted: {}\n\n", alias_entries.size()); } else { diff --git a/recompiler/tests/test_aot_overlay_discovery.py b/recompiler/tests/test_aot_overlay_discovery.py index 66707d70..4604f07b 100644 --- a/recompiler/tests/test_aot_overlay_discovery.py +++ b/recompiler/tests/test_aot_overlay_discovery.py @@ -73,6 +73,23 @@ def check_composite_call_boundaries(): assert cross_target in scoped['direct_jals'] +def check_static_discovery_provenance(): + data = bytearray(0x80) + entry = LOAD + 0x20 + put(data, 0x20, 0x30620008) # andi v0,v1,8 -- frameless body + put(data, 0x24, 0x03E00008) + put(data, 0x28, 0x00000000) + cap = { + 'schema': 'psxrecomp overlay capture v2', + 'function_entry_pcs': [f'0x{entry:08X}'], + 'static_discovery_entry_pcs': [f'0x{entry:08X}'], + } + seeds, audit = MOD.classify_overlay_seeds( + cap, bytes(data), LOAD, len(data), 0, {}) + assert f'call_root 0x{entry:08X}' in seeds + assert audit['included_reasons'][entry] == 'STATIC_DISCOVERY_ROOT' + + def check_padded_return_boundary(recompiler): # Exact-entry mode re-verifies every untrusted seed. Psy-Q aligns frameless # leaves with NOPs after the prior function's JR delay slot; those NOPs must @@ -138,6 +155,38 @@ def check_recompiler_composite_contract(recompiler): assert f"void func_{target:08X}" in generated(True) +def check_retained_alias_contract(recompiler): + # A previously compiled indirect entry retains its overlapping host body + # without becoming a root that caps the newly discovered ordinary walk. + data = bytearray(0x80) + put(data, 0x00, 0x03E00008) + put(data, 0x04, 0x00000000) + put(data, 0x20, 0x24020001) + put(data, 0x24, 0x03E00008) + put(data, 0x28, 0x00000000) + + with tempfile.TemporaryDirectory() as tmp: + psx = os.path.join(tmp, "retained_alias.psx") + seeds = os.path.join(tmp, "seeds.txt") + out = os.path.join(tmp, "out") + with open(psx, "wb") as f: + f.write(make_psxexe(LOAD, data)) + with open(seeds, "w", encoding="utf-8") as f: + f.write(f"0x{LOAD:08X}\n") + f.write( + f"retained_alias 0x{LOAD + 0x20:08X} " + f"0x{LOAD:08X} 0x{LOAD + 0x2C:08X}\n") + result = subprocess.run( + [recompiler, psx, "--seeds", seeds, "--out-dir", out, "--overlay"], + capture_output=True, text=True) + assert result.returncode == 0, result.stderr or result.stdout + full = ''.join(path.read_text() for path in pathlib.Path(out).glob("*_full*.c")) + assert f"void func_{LOAD + 0x20:08X}" in full + ranges = next(pathlib.Path(out).glob("*_full.ranges")).read_text() + assert f"F {LOAD + 0x20:08X}" in ranges + assert f"R {LOAD:08X} 2C" in ranges + + def main(): default_recompiler = ROOT / "recompiler" / "build" / "psxrecomp-game.exe" parser = argparse.ArgumentParser() @@ -147,7 +196,9 @@ def main(): raise SystemExit(f"recompiler not found: {args.recompiler}") check_composite_call_boundaries() + check_static_discovery_provenance() check_recompiler_composite_contract(args.recompiler) + check_retained_alias_contract(args.recompiler) data = bytearray(0x200) diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 06dfb77b..a68886aa 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -77,6 +77,18 @@ python tools/coverage_vault.py compact-addendum \ The replacement is atomic and aborts without changing the addendum if any valid record lacks its exact immutable snapshot. +For a live-recall scoreboard that survives individual launches, pass the same +history to `coverage_report.py --addendum ...`. It unions only dispatch/function +entries from every verified snapshot, so the report no longer depends on which +latest capture happened to be present when it ran. + +PS-X EXE extraction records normal-mode discovery provenance separately from +generic function-pointer candidates. `compile_overlays.py` promotes only those +entries that also pass its local callable/CFG proof to trusted `call_root`s. +When rebuilding identical bytes, prior overlapping alias groups are retained as +non-root bodies, so stronger new roots add coverage without displacing already +compiled indirect entries. + ## Multi-game sweep findings (2026-07-17) `extract_generic.py` runs the improved extraction from a game.toml. Results: diff --git a/tools/aot_overlay_spike/extract_generic.py b/tools/aot_overlay_spike/extract_generic.py index 9f558699..da7ae02b 100644 --- a/tools/aot_overlay_spike/extract_generic.py +++ b/tools/aot_overlay_spike/extract_generic.py @@ -245,7 +245,8 @@ def full_discovery_seeds(data, recompiler, tmp): a=int(m.group(1),16); seeds.add(a if a>=0x80000000 else (0x80000000|a)) return sorted(seeds) or None -def rec(load_addr, data, seeds, dispatch_extra=None, producer_ranges=None): +def rec(load_addr, data, seeds, dispatch_extra=None, producer_ranges=None, + static_discovery=None): # function_entry_pcs = prologue-scanned function starts (walk roots). # dispatch_entry_pcs = those PLUS any extra dispatch targets (e.g. the # overlay's own export table). compile_overlays promotes clean starts to @@ -262,6 +263,9 @@ def rec(load_addr, data, seeds, dispatch_extra=None, producer_ranges=None): out["producer_ranges"]=[ {"start":f"0x{lo:08X}","end":f"0x{hi:08X}"} for lo,hi in producer_ranges] + if static_discovery: + out["static_discovery_entry_pcs"]=[ + f"0x{addr:08X}" for addr in sorted(set(static_discovery))] return out def main(): @@ -304,6 +308,7 @@ def main(): t_addr=struct.unpack_from(' None: print(f'function_entry_pcs: {len(audit["function_entry_pcs"])}') print(f'direct_jal_targets_included: {audit["counts"].get("DIRECT_JAL_TARGET", 0)}') print(f'static_indirect_targets_included: {audit["counts"].get("STATIC_INDIRECT_TARGET", 0)}') + print(f'static_discovery_roots_included: ' + f'{audit["counts"].get("STATIC_DISCOVERY_ROOT", 0)}') print(f'function_pointer_targets_included: {audit["counts"].get("FUNCTION_POINTER_TARGET", 0)}') print(f'toml_entries_included: {audit["counts"].get("TOML_DECLARED_ENTRY", 0)}') print(f'dispatch_interior_included: {audit["counts"].get("DISPATCH_INTERIOR", 0)}') @@ -2118,6 +2143,9 @@ def main(): help='TinyCC binary (used when --compiler tcc)') ap.add_argument('--force', action='store_true', help='recompile even if output already exists') + ap.add_argument('--only-region', action='append', default=[], + help='compile only captures whose normalized load address ' + 'matches this value (repeatable; manual validation aid)') ap.add_argument('--check', action='store_true', help='PREFLIGHT: build every shard into a throwaway temp dir ' '(implies --force, ignores any injected cache dir, never ' @@ -2232,6 +2260,14 @@ def main(): with open(args.captures) as f: captures = json.load(f) + if args.only_region: + only_regions = {int(value, 0) & 0x1FFFFFFF + for value in args.only_region} + captures = [cap for cap in captures + if (int(cap['load_addr'], 16) & 0x1FFFFFFF) in only_regions] + if not captures: + ap.error('--only-region did not match any capture') + print(f'Captures: {len(captures)} overlay(s) to process\n') # B-2 static mode: accumulate privately-namespaced generated C plus exact @@ -2291,13 +2327,16 @@ def main(): # entries (walk-root eligible); non-callable ones (alias wrappers # from a prior build) re-enter as dispatch entries so the classifier # re-derives their interior/alias disposition — feeding them back as - # roots would truncate their hosts. + # roots would truncate their hosts. Their prior R ranges additionally + # reconstruct overlapping alias groups through retained_alias lines, so + # stronger new roots and old indirect dispatch entries can coexist. if not args.static: ranges_name = f'{phys_addr:08X}_{crc32:08X}.ranges' - prior_ranges = os.path.join(cache_dir, ranges_name) - if os.path.exists(prior_ranges): + prior_ranges_path = os.path.join(cache_dir, ranges_name) + if os.path.exists(prior_ranges_path): prior_entries = [] - with open(prior_ranges) as pf: + prior_func_ranges = [] + with open(prior_ranges_path) as pf: for ln in pf: parts = ln.split() if parts and parts[0] == 'F': @@ -2305,6 +2344,12 @@ def main(): prior_entries.append(int(parts[1], 16)) except (IndexError, ValueError): pass + elif parts and parts[0] == 'R': + try: + prior_func_ranges.append( + (int(parts[1], 16), int(parts[2], 16))) + except (IndexError, ValueError): + pass if prior_entries: fe = _parse_addr_list(cap.get('function_entry_pcs', [])) de = _parse_addr_list(cap.get('dispatch_entry_pcs', [])) @@ -2315,9 +2360,26 @@ def main(): de.add(a) cap['function_entry_pcs'] = sorted(fe) cap['dispatch_entry_pcs'] = sorted(de) + alias_groups = {} + if len(prior_entries) != len(prior_func_ranges): + print(f' WARNING: malformed prior manifest has ' + f'{len(prior_entries)} F lines and ' + f'{len(prior_func_ranges)} R lines; alias groups ' + f'will not be retained') + else: + for entry, (range_lo, range_size) in zip( + prior_entries, prior_func_ranges): + alias_groups.setdefault( + (range_lo, range_lo + range_size), []).append(entry) + cap['_prior_aliases'] = [ + (entry, range_lo, range_hi) + for (range_lo, range_hi), entries in alias_groups.items() + if len(entries) > 1 or any(e != range_lo for e in entries) + for entry in entries + ] cap.setdefault('schema', 'merged') print(f' merged {len(prior_entries)} prior-manifest entries ' - f'from {prior_ranges}') + f'from {prior_ranges_path}') seeds, seed_audit = classify_overlay_seeds(cap, data, load_addr, size, crc32, toml) From baf8b5928cbed15a8b455865841b2b1facc4c497 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 04:40:07 -0700 Subject: [PATCH 16/37] aot: recover cross-boundary branch roots --- docs/AOT_OVERLAY_PLAN.md | 3 ++ .../tests/test_aot_overlay_discovery.py | 24 +++++++++++ tools/aot_overlay_spike/README.md | 3 ++ tools/compile_overlays.py | 41 +++++++++++++++++++ 4 files changed, 71 insertions(+) diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index a4fb65eb..4ec1a14c 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -184,6 +184,9 @@ manufacture overlay shards merely to duplicate those already-native BIOS bodies. Rebuilding identical region bytes also imports prior overlapping alias groups as non-root `retained_alias` bodies. This makes discovery additive: newly proven frameless roots cannot hard-cap and erase an older indirect-entry body. + A reachable direct branch that crosses a sibling-entry hard cap may add its + target as a static root only when a bounded target-local CFG reaches a return + without invalid words or sequential escape. 5. `--static` mode instead namespaces the C and folds variants into `overlays_static.c` linked directly into the binary, dispatched via a generated `psx_overlay_dispatch()` — **this is already an AOT delivery path**; it moves diff --git a/recompiler/tests/test_aot_overlay_discovery.py b/recompiler/tests/test_aot_overlay_discovery.py index 4604f07b..0c0e7b70 100644 --- a/recompiler/tests/test_aot_overlay_discovery.py +++ b/recompiler/tests/test_aot_overlay_discovery.py @@ -90,6 +90,29 @@ def check_static_discovery_provenance(): assert audit['included_reasons'][entry] == 'STATIC_DISCOVERY_ROOT' +def check_forward_branch_root(): + data = bytearray(0x80) + target = LOAD + 0x40 + put(data, 0x00, 0x1000000F) # beq zero,zero,target + put(data, 0x04, 0x00000000) + put(data, 0x20, 0x27BDFFF0) # sibling root that hard-caps the first walk + put(data, 0x24, 0x03E00008) + put(data, 0x28, 0x27BD0010) + put(data, 0x40, 0x24020001) # forward branch target, frameless + put(data, 0x44, 0x03E00008) + put(data, 0x48, 0x00000000) + cap = { + 'schema': 'psxrecomp overlay capture v2', + 'function_entry_pcs': [f'0x{LOAD:08X}', f'0x{LOAD + 0x20:08X}'], + 'static_discovery_entry_pcs': [ + f'0x{LOAD:08X}', f'0x{LOAD + 0x20:08X}'], + } + seeds, audit = MOD.classify_overlay_seeds( + cap, bytes(data), LOAD, len(data), 0, {}) + assert f'call_root 0x{target:08X}' in seeds + assert audit['included_reasons'][target] == 'STATIC_BRANCH_ROOT' + + def check_padded_return_boundary(recompiler): # Exact-entry mode re-verifies every untrusted seed. Psy-Q aligns frameless # leaves with NOPs after the prior function's JR delay slot; those NOPs must @@ -197,6 +220,7 @@ def main(): check_composite_call_boundaries() check_static_discovery_provenance() + check_forward_branch_root() check_recompiler_composite_contract(args.recompiler) check_retained_alias_contract(args.recompiler) diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index a68886aa..6db048ec 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -88,6 +88,9 @@ entries that also pass its local callable/CFG proof to trusted `call_root`s. When rebuilding identical bytes, prior overlapping alias groups are retained as non-root bodies, so stronger new roots add coverage without displacing already compiled indirect entries. +Reachable direct branches that cross a sibling-entry hard cap are also promoted +only after the target passes the same bounded CFG proof. This recovers out-of-line +switch/state blocks without a blind byte or pointer sweep. ## Multi-game sweep findings (2026-07-17) diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index c694f788..6a4f3f01 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -282,6 +282,7 @@ INCLUDE_REASONS = { 'DISPATCH_ENTRY', 'DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET', + 'STATIC_BRANCH_ROOT', # Play-free normal-mode discovery entry that also passes a target-local # callable-boundary/CFG proof. Stronger than a generic captured pointer. 'STATIC_DISCOVERY_ROOT', @@ -653,6 +654,7 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, static_indirect_targets = set() branch_targets = set() jump_table_targets = set() + forward_branch_targets = set() rejected_cross_producer_calls = set() accepted_cross_producer_calls = set() @@ -708,12 +710,19 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, if in_function(target): work.append(target) branch_targets.add(target) + elif lo <= target < hi and target >= hard_cap and (target & 3) == 0: + # A reachable direct branch may jump across a sibling entry. + # Exact-entry partitioning hard-caps the current walk there, + # but the target remains mechanically proven code reachability. + forward_branch_targets.add(target) elif kind == 'j': if in_function(delay): visited.add(delay) if in_function(target): work.append(target) branch_targets.add(target) + elif lo <= target < hi and target >= hard_cap and (target & 3) == 0: + forward_branch_targets.add(target) elif kind == 'jal': if in_function(delay): visited.add(delay) @@ -764,6 +773,7 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, 'static_indirect_targets': static_indirect_targets, 'branch_targets': branch_targets, 'jump_table_targets': jump_table_targets, + 'forward_branch_targets': forward_branch_targets, 'rejected_cross_producer_calls': rejected_cross_producer_calls, 'accepted_cross_producer_calls': accepted_cross_producer_calls, } @@ -887,6 +897,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, included.setdefault(addr, 'DISPATCH_INTERIOR') return elif (reason not in ('DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET', + 'STATIC_BRANCH_ROOT', 'STATIC_DISCOVERY_ROOT', 'TOML_DECLARED_ENTRY') and addr in jump_table_targets and @@ -897,6 +908,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, raise RuntimeError(f'BUG: refusing to include 0x{addr:08X} as {reason}') old = included.get(addr) if reason in ('DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET', + 'STATIC_BRANCH_ROOT', 'STATIC_DISCOVERY_ROOT'): # Static call-edge evidence outranks heuristic capture metadata. included[addr] = reason @@ -962,6 +974,31 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, if target in included and target not in known: known.add(target) pending.append(target) + for target in sorted(walk['forward_branch_targets']): + source_reason = included.get(entry) + strong_source = source_reason in ( + 'DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET', + 'STATIC_BRANCH_ROOT', 'STATIC_DISCOVERY_ROOT', + 'TOML_DECLARED_ENTRY', 'DISPATCH_ROOT') + if not strong_source and not _callable_legacy_seed( + data, load_addr, entry): + continue + source_range = next( + ((lo, hi) for lo, hi in producer_ranges + if lo <= entry < hi), None) + target_range = next( + ((lo, hi) for lo, hi in producer_ranges + if lo <= target < hi), None) + if producer_ranges and source_range != target_range: + continue + producer_hi = target_range[1] if target_range else hi + if not _plausible_callable_target( + data, load_addr, size, target, producer_hi): + continue + include(target, 'STATIC_BRANCH_ROOT') + if target in included and target not in known: + known.add(target) + pending.append(target) if not kernel_window: break @@ -1003,6 +1040,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, data, load_addr, size, entry, hard_cap, producer_ranges) all_branch_targets.update(walk['branch_targets']) all_branch_targets.update(walk['jump_table_targets']) + all_branch_targets.update(walk['forward_branch_targets']) rejected_cross_producer_calls.update( walk['rejected_cross_producer_calls']) accepted_cross_producer_calls.update( @@ -1055,6 +1093,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, if r == 'DISPATCH_ROOT': return f'dispatch_root 0x{addr:08X}' if r in ('DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET', + 'STATIC_BRANCH_ROOT', 'STATIC_DISCOVERY_ROOT'): return f'call_root 0x{addr:08X}' return f'0x{addr:08X}' @@ -1078,6 +1117,8 @@ def print_seed_audit(audit: dict) -> None: print(f'function_entry_pcs: {len(audit["function_entry_pcs"])}') print(f'direct_jal_targets_included: {audit["counts"].get("DIRECT_JAL_TARGET", 0)}') print(f'static_indirect_targets_included: {audit["counts"].get("STATIC_INDIRECT_TARGET", 0)}') + print(f'static_branch_roots_included: ' + f'{audit["counts"].get("STATIC_BRANCH_ROOT", 0)}') print(f'static_discovery_roots_included: ' f'{audit["counts"].get("STATIC_DISCOVERY_ROOT", 0)}') print(f'function_pointer_targets_included: {audit["counts"].get("FUNCTION_POINTER_TARGET", 0)}') From 8f90df9b43b0275b0504fc32bd17eae1c75fef69 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 05:31:03 -0700 Subject: [PATCH 17/37] aot: preload exact-hash BIOS resident shards --- docs/AOT_OVERLAY_PLAN.md | 16 +++ recompiler/tests/test_aot_base_recovery.py | 39 ++++++ recompiler/tests/test_bios_resident_shard.py | 59 ++++++++ runtime/src/overlay_loader.c | 63 ++++++++- tools/aot_overlay_spike/README.md | 61 ++++++--- .../aot_overlay_spike/bios_resident_code.json | 35 +++++ tools/aot_overlay_spike/extract_generic.py | 126 +++++++++++++++++- tools/compile_overlays.py | 34 ++++- tools/coverage_vault.py | 5 +- tools/test_coverage_vault.py | 16 +++ 10 files changed, 428 insertions(+), 26 deletions(-) create mode 100644 recompiler/tests/test_bios_resident_shard.py create mode 100644 tools/aot_overlay_spike/bios_resident_code.json diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index 4ec1a14c..f6071394 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -167,6 +167,16 @@ scoreboard preserves overlay-cache-only recall and also reports combined native coverage from the dispatch table plus guarded relocated-kernel body ranges. Do not manufacture overlay shards merely to duplicate those already-native BIOS bodies. +Boot-installed RAM helpers that are not contiguous ROM relocations use a separate +exact-hash recipe (`tools/aot_overlay_spike/bios_resident_code.json`). The generic +extractor emits only the listed code words/entries when the supplied BIOS SHA-256 +matches. `compile_overlays.py` adds a `.resident` sidecar, and the runtime preloads +only explicitly marked resident shards. Preload is registration, not trust: every +entry still passes the same live code-range CRC before native execution. The +current SCPH-1001 recipe is exactly `0x8000DF80-0x8000DFF0`; it deliberately omits +page fill and the `0x8000DFFC` callback-pointer data. This closes the six durable +Tomba 2 kernel gaps without turning adjacent zeros/data into code. + ### 1.3 Compile (offline, `tools/compile_overlays.py`) 1. Python does function-boundary discovery from the seeds (`classify_overlay_seeds`, `_walk_overlay_function`) with a strict callable-prologue gate; interior/ @@ -187,6 +197,8 @@ manufacture overlay shards merely to duplicate those already-native BIOS bodies. A reachable direct branch that crosses a sibling-entry hard cap may add its target as a static root only when a bounded target-local CFG reaches a return without invalid words or sequential escape. + Exact-hash BIOS resident captures additionally receive a `.resident` sidecar; + ordinary captures actively remove a stale sidecar for the same output stem. 5. `--static` mode instead namespaces the C and folds variants into `overlays_static.c` linked directly into the binary, dispatched via a generated `psx_overlay_dispatch()` — **this is already an AOT delivery path**; it moves @@ -202,6 +214,10 @@ manufacture overlay shards merely to duplicate those already-native BIOS bodies. `:595`). Cache is **additive** — every variant ever seen for an address stays loaded; per-dispatch CRC picks the one that currently matches (`:1844`). This is how one address (`0x800E7xxx`) hosts village vs overworld safely. +- Marked BIOS-resident shards are registered during cache scan because their exact + compact code envelope is not the runtime's page-coalesced kernel region key. + They begin invalid while boot RAM differs, then revalidate after the BIOS writes + the expected bytes. Unmarked shards retain lazy region loading. --- diff --git a/recompiler/tests/test_aot_base_recovery.py b/recompiler/tests/test_aot_base_recovery.py index 52d574d3..28d63a4b 100644 --- a/recompiler/tests/test_aot_base_recovery.py +++ b/recompiler/tests/test_aot_base_recovery.py @@ -2,9 +2,12 @@ """Focused regression checks for generic AOT overlay base selection.""" import collections import importlib.util +import hashlib +import json import pathlib import struct import sys +import tempfile ROOT = pathlib.Path(__file__).resolve().parents[2] @@ -53,6 +56,42 @@ def main(): struct.pack_into('mtime = file->mtime; e->func_count = 0; e->indexed_func_count = 0; + e->resident = cache_path_is_bios_resident(file->path); snprintf(e->path, sizeof(e->path), "%s", file->path); return 0; } @@ -973,6 +1007,7 @@ static void scan_one_cache_dir(const char *dir) { e->func_count = 0; e->indexed_func_count = 0; snprintf(e->path, sizeof(e->path), "%s", full); + e->resident = cache_path_is_bios_resident(full); } while (FindNextFileA(h, &fd)); FindClose(h); #else @@ -1201,6 +1236,7 @@ static void scan_cache_dir(void) { scan_one_cache_dir(dir); abi_preflight_sweep(dir); + refresh_bios_resident_flags(); rebuild_lazy_manifest_index(); /* Never-again guard: if we loaded NOTHING but wrong-hash shards exist, shout. */ @@ -1219,9 +1255,10 @@ static void scan_cache_dir(void) { for (int i = 0; i < s_cache_idx_count; i++) { const char *base = strrchr(s_cache_idx[i].path, '/'); base = base ? base + 1 : s_cache_idx[i].path; - fprintf(stderr, " [%d] %s (region=0x%08X, functions=%d)\n", + fprintf(stderr, " [%d] %s (region=0x%08X, functions=%d%s)\n", i, base, s_cache_idx[i].region_start, - s_cache_idx[i].indexed_func_count); + s_cache_idx[i].indexed_func_count, + s_cache_idx[i].resident ? ", BIOS-resident" : ""); } fflush(stderr); } @@ -1706,6 +1743,26 @@ static int load_overlay_dll(const char *dll_path, ManFn *man, int man_n, int dll /* ---- Public API -------------------------------------------------------- */ +static int dll_already_loaded(const char *path); +static int load_one_dll(const char *dll_path); + +static int load_bios_resident_shards(void) { + int bundles = 0, functions = 0; + for (int i = 0; i < s_cache_idx_count; i++) { + CacheEntry *e = &s_cache_idx[i]; + if (!e->resident || e->func_count <= 0 || + e->indexed_func_count != e->func_count || + dll_already_loaded(e->path)) + continue; + int loaded = load_one_dll(e->path); + if (loaded > 0) { bundles++; functions += loaded; } + } + if (bundles > 0) + loader_log("preloaded %d exact-guarded BIOS resident shard(s), %d candidates", + bundles, functions); + return functions; +} + void overlay_loader_init(const char *cache_dir, const char *game_id) { for (uint32_t p = 0; p < RANGE_PAGE_COUNT; p++) { s_range_page_head[p] = -1; @@ -1721,6 +1778,7 @@ void overlay_loader_init(const char *cache_dir, const char *game_id) { { extern void ds_init(const char*, const char*); ds_init(cache_dir, game_id); } init_callbacks(); scan_cache_dir(); + load_bios_resident_shards(); overlay_image_warm_init(); overlay_image_warm_seed_boot_text(); /* (sljit removed 2026-07-15: the persisted-shard reload + off-thread compile @@ -1846,6 +1904,7 @@ static void mark_checked(uint32_t region_start) { void overlay_loader_rescan(void) { if (!s_active) return; scan_cache_dir(); + load_bios_resident_shards(); s_nchecked = 0; } diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 6db048ec..15ed6897 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -5,10 +5,10 @@ disc image at build time, so a fresh install ships with overlay coverage instead of relying entirely on runtime (tcc/gcc) autocompile. See `docs/AOT_OVERLAY_PLAN.md` for the full design, findings, and honest coverage numbers. -**Status: SPIKE. Provably-correct for clean producers, not exhaustive.** These -scripts carry hardcoded paths (disc, scratchpad) and are Tomba-specific proofs of -concept, not a finished general tool. They exist so a future session can improve -discovery quality, not to be run as-is in a build. +**Status: ENHANCEMENT SPIKE. Provably-correct for clean producers, not exhaustive.** +`extract_generic.py` is path-parameterized and suitable for the current build +pipeline. Disc-container discovery remains engine-specific; unsupported producers +fail closed instead of being guessed as code. ## What each does @@ -92,6 +92,34 @@ Reachable direct branches that cross a sibling-entry hard cap are also promoted only after the target passes the same bounded CFG proof. This recovers out-of-line switch/state blocks without a blind byte or pointer sweep. +## Exact-hash BIOS resident code + +The base BIOS emitter covers ordinary ROM-resident code and the relocated kernel, +but SCPH-1001 also assembles a small device helper directly into RAM at +`0x8000DF80`. It is not one contiguous ROM relocation. The bundled +`bios_resident_code.json` records the exact 112-byte installed code image and its +six observed dispatch entries for BIOS SHA-256 +`71af94d1...f1e99d3`. Evidence is invariant across 415 snapshots from 10 sessions. + +`extract_generic.py` automatically appends the matching recipe when the framework +BIOS exists (or `--bios` / `PSXRECOMP_BIOS_ROM` names it). A different/missing BIOS +emits no resident record. `--only-bios-resident` materializes just these shared +recipes without scanning a game disc. The capture contains the code bytes only: +no page padding, no adjacent callback-pointer data, and no synthetic execution +claims. Its sole producer range is `DF80-DFF0`. + +`compile_overlays.py` marks only manifest-produced resident shards with a +`.resident` sidecar. The runtime preloads those marked DLLs because their compact +synthetic envelope cannot equal the kernel dirty-run key. Preload only registers +candidates: normal per-function live-byte CRC checks remain authoritative. At +boot the candidate initially mismatches and is invalid; after the BIOS installs +the helper, page-generation invalidation rechecks it and enables native dispatch. +An exact T2 static-only attract soak verified all six entries native, zero +interpreter appearances, zero aborts/guard yields, correct scene transitions, and +60-61 fps at BelowNormal priority. The shard emits exactly six functions with +zero unsupported instructions or bad targets; a rejected earlier page-envelope +prototype exposed padding as a seventh function and was not retained. + ## Multi-game sweep findings (2026-07-17) `extract_generic.py` runs the improved extraction from a game.toml. Results: @@ -158,12 +186,14 @@ code-range recall. The distinction matters: runtime/autocompile vaults may conta one-entry fragments at consecutive instructions, while static AOT emits one broad function with the same PCs in its guarded `R` ranges. Exact-entry parity is retained for continuity; code-range recall is the useful uncovered-code roadmap. At the -current Tomba 2 checkpoint these are 1270/1856 (68.4%) exact entries and 1811/1856 -(97.6%) overlay-cache code-range coverage. Pass the generated base BIOS dispatcher -with `--bios-dispatch generated/SCPH1001_dispatch.c` to report separate combined -metrics. The base recompiler already covers relocated kernel bodies with a live-byte -guard; including it raises Tomba 2 to 1852/1856 (99.8%) native code-range coverage. -This avoids generating duplicate overlay shards for code that is already native. +current Tomba 2 checkpoint the play-free overlay cache plus the exact-hash +resident shard covers 97.6% of vault code ranges. Pass the generated base BIOS +dispatcher with `--bios-dispatch generated/SCPH1001_dispatch.c` to report separate +combined metrics. The base recompiler covers relocated kernel bodies with a +live-byte guard; including it raises both the full-playthrough vault and the +verified append-only live-history needed set to **100% native code-range +coverage**. This avoids generating duplicate overlay shards for code that is +already native. ## Next to raise coverage (future session) @@ -186,10 +216,9 @@ This avoids generating duplicate overlay shards for code that is already native. A09/A0J/OPN; GAME's 8-byte near-tie resolves through the exact base independently proved by DEMO. The played vault byte-proves GAME at 0x80106228. Ambiguous files without one unique trusted match still fail closed. -4. ~~Extract the kernel `0x80000000` region once from the BIOS~~ — NOT NEEDED for - the measured body gaps. The base BIOS recompiler already emits 499 relocated, - guarded kernel bodies. Of the four combined gaps, A0/B0/C0 are tiny installed - vector thunks and EE7C is an all-zero runtime-fragment artifact; keep the latter - interpreted rather than treating padding/data as code. - this is now the complete 45-PC Tomba 2 code-range gap set. +4. ~~AOT the game-independent kernel region once from the BIOS~~ — + DONE without compiling the whole dirty kernel capture. The base BIOS recompiler + supplies relocated guarded bodies and A0/B0/C0 stubs. The only invariant + installed-code gap is the exact-hash `DF80-DFF0` resident recipe above. Padding + and the adjacent `DFFC` callback pointer are intentionally excluded from code. 5. Generalize into a real `tools/` producer (no hardcoded paths; disc + game.toml in). diff --git a/tools/aot_overlay_spike/bios_resident_code.json b/tools/aot_overlay_spike/bios_resident_code.json new file mode 100644 index 00000000..5089a251 --- /dev/null +++ b/tools/aot_overlay_spike/bios_resident_code.json @@ -0,0 +1,35 @@ +{ + "schema": "psxrecomp bios resident code v1", + "images": [ + { + "name": "SCPH-1001 boot-installed device helper", + "bios_sha256": "71af94d1e47a68c11e8fdb9f8368040601514a42a5a399cda48c7d3bff1e99d3", + "region_start": "0x8000DF80", + "region_size": "0x70", + "evidence": { + "verified_snapshots": 415, + "verified_sessions": 10, + "distinct_code_images": 1, + "code_window_crc32": "0x4EE6AC69" + }, + "code_fragments": [ + { + "address": "0x8000DF80", + "words": [ + "0x946F000A", "0x3C080000", "0x01E2C025", "0x37190012", + "0xA479000A", "0x24080028", "0x2508FFFF", "0x1500FFFE", + "0x00000000", "0x03E00008", "0x00000000", "0x8C621074", + "0x00000000", "0x30420080", "0x1040000B", "0x00000000", + "0x8C621044", "0x00000000", "0x30420080", "0x1440FFFC", + "0x00000000", "0x3C020001", "0x8C42DFFC", "0x00000000", + "0x00400008", "0x00000000", "0x03E00008", "0x00000000" + ] + } + ], + "dispatch_entry_pcs": [ + "0x8000DF80", "0x8000DF98", "0x8000DFA4", + "0x8000DFAC", "0x8000DFC0", "0x8000DFD4" + ] + } + ] +} diff --git a/tools/aot_overlay_spike/extract_generic.py b/tools/aot_overlay_spike/extract_generic.py index da7ae02b..daee8fda 100644 --- a/tools/aot_overlay_spike/extract_generic.py +++ b/tools/aot_overlay_spike/extract_generic.py @@ -20,7 +20,7 @@ Usage: --out [--tmp ] Requires extract_overlays.py (same dir chain) for the DiscReader/ISO9660 walker. """ -import argparse, os, sys, json, base64, struct, subprocess, tempfile, re, binascii +import argparse, os, sys, json, base64, struct, subprocess, tempfile, re, binascii, hashlib sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) import extract_overlays as eo try: @@ -268,13 +268,118 @@ def rec(load_addr, data, seeds, dispatch_extra=None, producer_ranges=None, f"0x{addr:08X}" for addr in sorted(set(static_discovery))] return out +def bios_resident_records(bios_path=None, manifest_path=None): + """Build guarded captures for BIOS-installed RAM code known by exact hash. + + Some BIOS helpers are assembled or patched into RAM during boot rather than + copied as one contiguous ROM range. They therefore cannot be emitted by + the ordinary ROM->RAM relocation path. The manifest records only verified + code words and dispatch entries for an exact BIOS SHA-256. Any gap in a + declared image is inert zero fill, is outside ``producer_ranges``, and can never + become a discovery root. compile_overlays.py still emits its normal + per-function live-byte CRC guards, so a different install image fails closed + to the interpreter. + """ + here=os.path.dirname(os.path.abspath(__file__)) + if manifest_path is None: + manifest_path=os.path.join(here, 'bios_resident_code.json') + if bios_path is None: + bios_path=os.environ.get('PSXRECOMP_BIOS_ROM') or os.path.abspath( + os.path.join(here, '..', '..', 'bios', 'SCPH1001.BIN')) + if not os.path.isfile(manifest_path) or not os.path.isfile(bios_path): + return [] + + with open(bios_path, 'rb') as f: + bios_sha=hashlib.sha256(f.read()).hexdigest().lower() + with open(manifest_path, encoding='utf-8') as f: + manifest=json.load(f) + if manifest.get('schema') != 'psxrecomp bios resident code v1': + raise ValueError(f'{manifest_path}: unsupported BIOS resident manifest schema') + + def number(value, label): + try: + return value if isinstance(value, int) else int(str(value), 0) + except (TypeError, ValueError) as e: + raise ValueError(f'{manifest_path}: invalid {label}: {value!r}') from e + + records=[] + for image in manifest.get('images', []): + if str(image.get('bios_sha256', '')).lower() != bios_sha: + continue + start=number(image.get('region_start'), 'region_start') + size=number(image.get('region_size'), 'region_size') + if (start & 3) or size <= 0 or size > 0x10000 or (size & 3): + raise ValueError(f'{manifest_path}: resident region must be ' + 'word-aligned, word-sized, and at most 64 KiB') + end=start+size + data=bytearray(size) + written={} + producer_ranges=[] + + def put_word(addr, word, label): + if (addr & 3) or not (start <= addr <= end-4): + raise ValueError(f'{manifest_path}: {label} address 0x{addr:08X} ' + 'is outside/alignment-invalid for its region') + word &= 0xFFFFFFFF + if addr in written and written[addr] != word: + raise ValueError(f'{manifest_path}: conflicting resident word at ' + f'0x{addr:08X}') + written[addr]=word + struct.pack_into(' {a.out}") + return + if not a.game_toml or not a.recompiler: + ap.error('--game-toml and --recompiler are required unless ' + '--only-bios-resident is used') doc=tomllib.loads(open(a.game_toml, encoding='utf-8-sig').read()) # utf-8-sig strips BOM game=doc.get('game',doc) root=os.path.dirname(os.path.abspath(a.game_toml)) @@ -437,9 +542,20 @@ def main(): print(f" [composite] {lp} + {rp}: {len(region)}B " f"region@0x{page_base:08X} (exact adjacent producers), " f"seeds={len(seeds)}") - json.dump(records, open(a.out,'w')) + nb=0 + if not a.no_bios_resident: + resident=bios_resident_records(a.bios, a.bios_resident_manifest) + records.extend(resident) + nb=len(resident) + for record in resident: + print(f" [BIOS resident] {record['producer_name']}: " + f"{record['size']}B @0x{int(record['load_addr'],16):08X}, " + f"dispatch entries={len(record['dispatch_entry_pcs'])}, " + f"sha256={record['bios_sha256'][:12]}...") + with open(a.out, 'w') as f: + json.dump(records, f) print(f"producers: {np} PS-X EXE (full-discovery), {nh} header-table, " - f"{nr} raw-code, {nc} adjacent composites; " + f"{nr} raw-code, {nc} adjacent composites, {nb} BIOS resident; " f"{len(records)} regions -> {a.out}") if __name__=='__main__': main() diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index 6a4f3f01..decd0983 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -309,6 +309,35 @@ INCLUDE_REASONS = { 'DISPATCH_ROOT', } FATAL_SEED_REASONS = {'BRANCH_TARGET_ONLY', 'OBSERVED_PC_ONLY', 'UNKNOWN'} +BIOS_RESIDENT_PRODUCER = 'bios_resident_manifest' +BIOS_RESIDENT_MARKER = 'psxrecomp bios resident shard v1' + + +def update_bios_resident_marker(dll_path: str, cap: dict) -> None: + """Publish/remove the opt-in marker consumed by the runtime preloader. + + Resident BIOS helpers use a synthetic page envelope, so their filename's + region key cannot equal the dirty-run start recovered at runtime. The + marker tells the loader to register the shard at cache scan time; normal + per-function code-CRC validation remains the execution authority. + """ + marker = os.path.splitext(dll_path)[0] + '.resident' + if cap.get('producer') != BIOS_RESIDENT_PRODUCER: + try: + os.remove(marker) + except FileNotFoundError: + pass + return + payload = { + 'schema': BIOS_RESIDENT_MARKER, + 'bios_sha256': str(cap.get('bios_sha256', '')), + 'producer_name': str(cap.get('producer_name', 'BIOS resident code')), + } + tmp = marker + '.tmp' + with open(tmp, 'w', encoding='utf-8') as f: + json.dump(payload, f, sort_keys=True) + f.write('\n') + os.replace(tmp, marker) def _parse_addr(value) -> int: @@ -2631,7 +2660,9 @@ def main(): covered = load_region_coverage(cache_dir, phys_addr) region_coverage_cache[phys_addr] = covered fully_covered = (bool(this_set) and this_set <= covered - and not args.force) + and not args.force + and cap.get('producer') != + BIOS_RESIDENT_PRODUCER) if fully_covered: print(f' SKIP: all {len(this_set)} function(s) already ' f'covered by existing DLL(s) at this region — no new ' @@ -2657,6 +2688,7 @@ def main(): ranges_out = os.path.splitext(dll_path)[0] + '.ranges' if this_ids: nfn = write_overlay_ranges_from(this_ids, ranges_out) + update_bios_resident_marker(dll_path, cap) print(f' ranges: {nfn} functions -> {ranges_out}') # New identities are now available for this region_start; # keep the warm coverage set current so later captures in diff --git a/tools/coverage_vault.py b/tools/coverage_vault.py index 0df9f4fb..4a37f8df 100644 --- a/tools/coverage_vault.py +++ b/tools/coverage_vault.py @@ -266,7 +266,7 @@ def merge_addendum(vault_json, addendum): return merge_capture_regions(vault_json, _load_addendum(addendum)) def merge_cache(vault_cache, src_cache): - """Mirror compiled DLLs/.ranges into the vault, PRESERVING the relative + """Mirror compiled DLLs/.ranges/.resident into the vault, preserving relative layout (//cg_/file). The layout is load- bearing: the same content-keyed filename exists under different cg dirs with DIFFERENT compiled bytes (per-emitter generations), so a flat copy @@ -279,7 +279,8 @@ def merge_cache(vault_cache, src_cache): for root, _dirs, files in os.walk(src_cache): rel = os.path.relpath(root, src_cache) for fn in files: - if not (fn.endswith(".dll") or fn.endswith(".ranges")): + if not (fn.endswith(".dll") or fn.endswith(".ranges") or + fn.endswith(".resident")): continue src = os.path.join(root, fn) dstdir = os.path.join(vault_cache, rel) if rel != "." else vault_cache diff --git a/tools/test_coverage_vault.py b/tools/test_coverage_vault.py index a689cc28..dd2cb0e3 100644 --- a/tools/test_coverage_vault.py +++ b/tools/test_coverage_vault.py @@ -16,6 +16,22 @@ SPEC.loader.exec_module(MOD) class CoverageVaultHistoryTests(unittest.TestCase): + def test_cache_merge_preserves_resident_sidecar(self): + with tempfile.TemporaryDirectory() as tmp: + src = os.path.join(tmp, 'source') + dst = os.path.join(tmp, 'vault') + rel = os.path.join('gcc', 'win-x64', 'cg5_12345678') + os.makedirs(os.path.join(src, rel)) + stem = os.path.join(src, rel, '0000DF80_4EE6AC69') + for ext, contents in (('.dll', b'dll'), ('.ranges', b'ranges'), + ('.resident', b'resident')): + with open(stem + ext, 'wb') as out: + out.write(contents) + self.assertEqual(MOD.merge_cache(dst, src), 1) + for ext in ('.dll', '.ranges', '.resident'): + self.assertTrue(os.path.exists(os.path.join( + dst, rel, '0000DF80_4EE6AC69' + ext))) + def test_v1_and_verified_v2_records_survive_a_torn_tail(self): with tempfile.TemporaryDirectory() as tmp: snapshot = os.path.join(tmp, "immutable.json") From 731f43235e892dc0c715475055bb135c2e232c75 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 06:30:37 -0700 Subject: [PATCH 18/37] tools: make large disc reads linear --- tools/extract_overlays.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/extract_overlays.py b/tools/extract_overlays.py index 0ad3804a..3558181f 100644 --- a/tools/extract_overlays.py +++ b/tools/extract_overlays.py @@ -65,10 +65,12 @@ class DiscReader: def read_file_bytes(self, lba, size): """Read exactly `size` bytes starting at `lba`.""" sectors_needed = (size + SECTOR_DATA - 1) // SECTOR_DATA - data = b'' - for i in range(sectors_needed): - data += self.read_sector_data(lba + i) - return data[:size] + # Do not repeatedly append to an immutable bytes object: large archive + # files (MMX6's ROCK_X6.DAT is ~51 MB / ~25k sectors) turn that into an + # O(n^2) copy loop. Accumulate references and copy the payload once. + chunks = [self.read_sector_data(lba + i) + for i in range(sectors_needed)] + return b''.join(chunks)[:size] # --------------------------------------------------------------------------- # CUE parser — extracts BIN path and track mode From 41f498fd09980050cf4b9534eddca47e1c792f59 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 07:25:49 -0700 Subject: [PATCH 19/37] aot: recover linked archives and static aliases --- docs/AOT_OVERLAY_PLAN.md | 25 ++ recompiler/tests/test_aot_base_recovery.py | 95 ++++- .../tests/test_aot_overlay_discovery.py | 25 ++ tools/aot_overlay_spike/README.md | 56 ++- tools/aot_overlay_spike/extract_generic.py | 395 ++++++++++++++++-- tools/compile_overlays.py | 26 +- 6 files changed, 577 insertions(+), 45 deletions(-) diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index f6071394..4027253f 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -24,6 +24,31 @@ continue to fail safely to the interpreter. --- +## 2026-07-18: byte-proven archive producers + +- The generic extractor now recognizes strict 0x800-aligned `{id,size}` member + archives (MMX6 `ROCK_X6.BIN`) and companion HED sector tables spanning DAT/BNS + payload namespaces (Ape Escape `KKIIDDZZ.*`). +- Link bases are not guessed from title constants. Multiple independent members + must anchor one sharp jal/prologue consensus; a sibling then needs at least two + votes for that exact trusted base. +- Mixed archive members seed only locally callable direct-JAL targets. The first + MMX6 pointer-table expansion demonstrated why: seven variants hit the generated-C + unsupported-instruction audit. Direct-only roots compile all 30 members cleanly, + plus the exact-hash BIOS resident shard. +- Ape's two PS-X EXE mirrors are deduplicated in favor of the self-describing EXE + producer. Its remaining 44 HED/BNS members plus two minigame EXEs and the BIOS + recipe are also audit-clean in a disposable cache. +- Normal PS-X discovery now exports its exact overlapping `F/R` alias recipes. + Clean overlay builds therefore preserve decoder-proven indirect entries without + promoting every entry to a hard root or borrowing a previous cache manifest. + T2 compiled 53/53 disposable candidates with zero audit failures and restored + 100% combined BIOS code-range coverage for both vault and live history. +- Large archive reads are now linear rather than quadratic sector concatenation. + These validations wrote only `%TEMP%` caches; title caches remain untouched. + +--- + ## ⭐ SESSION-2 BREAKTHROUGH (2026-07-17, VERIFIED FROM DISC + CORPUS) Adversarial disc analysis overturned the pessimistic framing below. For Tomba's diff --git a/recompiler/tests/test_aot_base_recovery.py b/recompiler/tests/test_aot_base_recovery.py index 28d63a4b..4fc88bbf 100644 --- a/recompiler/tests/test_aot_base_recovery.py +++ b/recompiler/tests/test_aot_base_recovery.py @@ -38,9 +38,48 @@ def main(): struct.pack_into("> 2) & 0x03FFFFFF)) assert MOD.jal_targets(data) == {0x80105678} + # Psy-Q may schedule a global load before allocating the stack frame. The + # exact function starts immediately after the previous return, not at the + # otherwise recognizable addiu-sp instruction eight bytes later. + base = 0x80100000 + data = bytearray(0x40) + struct.pack_into('> 2) & 0x03FFFFFF)) + struct.pack_into('> 2) & 0x03FFFFFF)) + struct.pack_into(' full-discovery 786 fns / 69% vault (3296 shards, 0 audit fails); - the tool now emits 3500 discovered seeds vs 1210 prologues. -- **Generality is ENGINE-SPECIFIC, not universal.** The two producer types + the current tool emits 3503 discovered entries vs 1210 prologues. It also + carries normal mode's exact overlapping-range recipes into clean overlay + builds, so alias coverage does not depend on an older cache manifest. +- **Generality is ENGINE-SPECIFIC, not universal.** The initial two producer types (self-describing PS-X-EXE + {count,ptr[]} header-table) cover the Whoopee Camp engine (Tomba 1 header-table; Tomba 2 MAIN.EXE + BIN) but detect ZERO overlays - on Vigilante 8 (Luxoflux) and Mega Man X6 (Capcom) — those load overlays in a - different container/format (likely compressed/archived). Each engine family - needs its own producer. Matches the Legaia-decomp finding: overlay enumeration - is bespoke per engine. + on Vigilante 8 (Luxoflux) and initially detected zero on Mega Man X6 (Capcom). + The indexed/HED archive producers below now cover MMX6 and Ape Escape; other + engine families still need their own byte-proven container parser. Matches the + Legaia-decomp finding: overlay enumeration is bespoke per engine. - **SOLVED (2026-07-17) — header-table base recovery via jal-target self-consistency.** The old delta-sweep matched the export-table pointers against prologues, but those pointers point at mid-function DISPATCH entries, not prologues (measured: 0/29 hit @@ -195,6 +197,48 @@ verified append-only live-history needed set to **100% native code-range coverage**. This avoids generating duplicate overlay shards for code that is already native. +## Indexed and companion archive recovery (2026-07-18) + +Large-container extraction first exposed an unrelated quadratic I/O bug: +`DiscReader.read_file_bytes()` appended immutable bytes once per 2 KiB sector. +It now joins sector chunks once; reading MMX6's 50.9 MB DAT fell from minutes to +about 0.25 seconds on the validation machine. + +`ROCK_X6.BIN` is a strict 0x800-aligned `{id,size}[]` archive, not compressed +code. Its members independently vote three link bases (`0x800E9060`, +`0x800F9800`, `0x801E9800`). A base is trusted only after at least two members +produce a sharp >=8-vote, >=2x jal-to-prologue peak; siblings need at least two +votes for that exact consensus. Mixed archive members use **direct-JAL roots +only**. A broader pointer-table experiment compiled 24/31 but correctly failed +seven generated-C audits; removing those speculative roots produced +`PSX_SHARD_RESULT ok=31 failed=0 skipped=0` (30 members + BIOS resident). Against +the union of 16 legacy MMX6 capture files this first conservative pass covers +71.8% of entries by combined native code range. The remaining gap is discovery +inside byte-proven members, not container enumeration. + +Ape Escape's `KKIIDDZZ.HED` encodes contiguous +`size_sectors:12 | logical_sector:20` runs spanning sibling DAT then BNS files. +The same cross-member vote gate proves `0x80136000` (10 anchors), `0x8013D000` +(27), and two mirrored minigame EXEs. Ordinary PS-X EXE handling owns the latter; +the HED producer emits 44 BNS members. Together with two self-describing minigame +EXEs and the resident BIOS recipe, the disposable validation cache has all 47 +candidates compiled with zero unsupported/bad targets. + +Normal-mode PS-X discovery quarantines its one provenance-free fallback: the +image body start is excluded when the PS-X header declares a different entry. +This removes Ape MINI2's body-leading address/string table at `0x80100000` while +retaining its declared `0x80100C84` entry. All decoder-classified interior +entries remain ordinary candidates, and the extractor serializes their `F/R` +overlapping alias recipes instead of promoting them to hard call roots. A clean +Tomba 2 disposable rebuild compiled 53/53 candidates with zero unsupported/bad +targets and, together with the base BIOS, reproduced **100% code-range coverage** +against both the full-playthrough vault and 765-entry append-only live history. +No prior MAIN manifest was present for the two regenerated shards. Position-fixed +framed scans likewise recover Psy-Q's exact +`lui R; load ...,off(R); addiu sp,sp,-N` entry after a previous return, replacing +the old root eight bytes late. This closes Tomba 1's historical `0x80110E30` gap +in the clean X00 variant without minting overlapping functions. + ## Next to raise coverage (future session) 1. ~~Direct-call frameless-leaf discovery~~ — DONE. Reachable `jal` targets are diff --git a/tools/aot_overlay_spike/extract_generic.py b/tools/aot_overlay_spike/extract_generic.py index daee8fda..9617c8fc 100644 --- a/tools/aot_overlay_spike/extract_generic.py +++ b/tools/aot_overlay_spike/extract_generic.py @@ -2,7 +2,7 @@ """Generic play-free overlay extractor (AOT spike, v2 — full-discovery producers). Given a game.toml (disc + load_address + text_size), enumerate the disc and emit -a synthetic overlay_captures.json for compile_overlays.py. Two producer types: +a synthetic overlay_captures.json for compile_overlays.py. Producer types: 1. PS-X EXE files (magic "PS-X EXE") — the load address is self-described in the header. Run the recompiler in NORMAL mode (full discovery: jr-$ra boundary @@ -15,6 +15,12 @@ a synthetic overlay_captures.json for compile_overlays.py. Two producer types: prologue-scan seeds at a delta-swept base. LESS robust (base recovery unreliable when header ptrs aren't clean function starts); see plan doc. + 3. Strict 0x800-aligned {id,size} member archives — multiple members must + independently vote the same link base; mixed members use direct-call roots. + + 4. Companion HED/DAT/BNS archives — contiguous packed-sector runs address a + logical namespace across payload siblings, with the same consensus gate. + Usage: extract_generic.py --game-toml --recompiler --out [--tmp ] @@ -44,9 +50,33 @@ def parse_cue_datatrack(cue_path): return eo.parse_cue(cue_path) # fallback def prologues(data, base): - return [base+off for off in range(0,len(data)-4,4) - if (struct.unpack_from('>16)==0x27BD - and (struct.unpack_from('= 16 and _word(data, off-16) == 0x03E00008: + lui=_word(data, off-8) + load=_word(data, off-4) + lui_rt=(lui >> 16) & 0x1F + if (lui >> 26) == 0x0F and lui_rt != 0 and \ + 0x20 <= (load >> 26) <= 0x26 and \ + ((load >> 21) & 0x1F) == lui_rt: + entry=off-8 + entries.append(base+entry) + return entries def prologue_offsets(data): """File offsets whose word is an `addiu $sp,$sp,-N` stack-frame prologue.""" @@ -213,6 +243,192 @@ def recover_raw_base(data, base_lo, base_hi=0x80200000): return None return base,score,second +def raw_base_votes(data, base_lo, base_hi=0x80200000): + """Return jal/prologue base votes for a position-fixed code blob.""" + import bisect + from collections import Counter + P=prologue_offsets(data) + T=jal_targets(data) + hist=Counter() + if not P or not T: + return hist + max_base=base_hi-len(data) + for t in T: + lo_i=bisect.bisect_left(P,t-max_base) + hi_i=bisect.bisect_right(P,t-base_lo) + for o in P[lo_i:hi_i]: + base=t-o + if (base&3)==0: + hist[base]+=1 + return hist + +def split_indexed_archive(data, alignment=0x800): + """Recognize a strict ``{id,size}[]`` + aligned-payload archive. + + Mega Man X6's ROCK_X6.BIN uses one monotonically increasing table in the + first sector. Payload members follow in table order, each rounded up to a + 0x800-byte boundary. Require the complete layout to account for the file + (apart from at most one format trailer sector) so ordinary data cannot + be mistaken for this container merely because its first words look small. + """ + if len(data) < alignment*2 or len(data) % alignment: + return None + entries=[] + for off in range(0, alignment, 8): + ident,size=struct.unpack_from(' 2_000_000: + return None + if entries and ident <= entries[-1][0]: + return None + entries.append((ident,size)) + if len(entries) < 4: + return None + pos=alignment + members=[] + for ident,size in entries: + if pos+size > len(data): + return None + members.append((ident,pos,data[pos:pos+size])) + pos=(pos+size+alignment-1)&~(alignment-1) + if pos > len(data) or len(data)-pos > alignment: + return None + return members + +def recover_consensus_members(members, base_lo, base_hi=0x80200000): + """Recover linked code from members that independently vote shared bases. + + An anchor member needs >=8 agreeing jal/prologue pairs and a 2x vote peak. + A base becomes trusted only when at least two independent members anchor it. + Other members may reuse that exact consensus with >=2 agreeing pairs. Only + locally callable direct-JAL targets become roots; pointer tables and apparent + prologues are deliberately insufficient inside a mixed code/data archive. + """ + from collections import Counter + if not members: + return [] + votes=[] + anchor_support=Counter() + for ident,off,body in members: + hist=raw_base_votes(body,base_lo,base_hi) + votes.append((ident,off,body,hist)) + top=hist.most_common(2) + if top: + base,score=top[0] + second=top[1][1] if len(top)>1 else 0 + if score >= 8 and score >= second*2: + anchor_support[base]+=1 + trusted={base for base,count in anchor_support.items() if count >= 2} + if not trusted: + return [] + + recovered=[] + for ident,off,body,hist in votes: + ranked=sorted(((hist.get(base,0),base) for base in trusted),reverse=True) + score,base=ranked[0] + runner=ranked[1][0] if len(ranked)>1 else 0 + if score < 2 or score < runner*2: + continue + lo,hi=base,base+len(body) + direct={t for t in jal_targets(body) + if lo <= t < hi and (t&3)==0 and + (_callable_boundary(body,t-base) or + ((_word(body,t-base) or 0)>>16)==0x27BD)} + seeds=sorted(direct) + if len(direct) < 2: + continue + recovered.append({ + 'id':ident, 'file_offset':off, 'data':body, 'base':base, + 'score':score, 'runner':runner, 'direct_seeds':sorted(direct), + 'seeds':seeds, + }) + return recovered + +def recover_indexed_archive(data, base_lo, base_hi=0x80200000): + """Recover independently linked code members from a strict archive.""" + members=split_indexed_archive(data) + return recover_consensus_members(members or (),base_lo,base_hi) + +def hed_companion_members(disc, files): + """Return strict sector-table members from sibling HED/DAT/BNS files. + + Ape Escape stores 32-bit descriptors as ``size_sectors:12 | lba:20``. + Descriptor runs are contiguous, while the payload namespace concatenates + same-stem DAT then BNS companions. Require runs of at least four entries, + exact sector continuity, sector-aligned companions, and no cross-file member. + """ + by_upper={p.upper():(p,l,s) for p,l,s in files} + groups=[] + for hed_path,hed_lba,hed_size in files: + if not hed_path.upper().endswith('.HED') or hed_size > 0x10000: + continue + stem=hed_path[:-4] + companions=[] + for ext in ('.DAT','.BNS'): + item=by_upper.get((stem+ext).upper()) + if item: + companions.append(item) + if not companions or any(size%0x800 for _,_,size in companions): + continue + hed=disc.read_file_bytes(hed_lba,hed_size) + words=[struct.unpack_from('>20 + if not count: + i+=1; continue + run=[]; expected=sector; j=i + while j < len(words): + value=words[j]; at=value&0xFFFFF; span=value>>20 + if not span or at != expected: + break + run.append((j,at,span)); expected+=span; j+=1 + if len(run) >= 4: + runs.extend(run); i=j + else: + i+=1 + if not runs: + continue + + logical=[]; cursor=0 + for path,lba,size in companions: + body=disc.read_file_bytes(lba,size) + logical.append((cursor,cursor+size//0x800,path,body)) + cursor+=size//0x800 + members=[]; seen=set() + for table_index,sector,count in runs: + key=(sector,count) + if key in seen: + continue + seen.add(key) + owner=next((x for x in logical + if x[0] <= sector and sector+count <= x[1]),None) + if owner is None: + continue + lo,hi,path,body=owner + off=(sector-lo)*0x800 + member=body[off:off+count*0x800] + members.append((table_index,sector*0x800,member)) + if members: + groups.append({ + 'hed_path':hed_path, + 'members':members, + 'consumed':{hed_path.upper(), + *(path.upper() for path,_,_ in companions)}, + }) + return groups + +def page_aligned_region(base, data): + """Build a runtime-keyed region and discard only proven all-zero lead pages.""" + page_base=base&~0xFFF + region=b'\x00'*(base-page_base)+data + while len(region)>0x1000 and not any(region[:0x1000]): + page_base+=0x1000 + region=region[0x1000:] + return page_base,region + def is_psx_exe(data): return len(data) >= 0x20 and data[:8] == b'PS-X EXE' @@ -225,28 +441,62 @@ def is_header_table(data): nul = sum(1 for p in ptrs if p == 0) return cnt if (inr+nul == cnt and inr >= 1) else None +def parse_full_discovery_ranges(lines): + """Return entries and overlapping alias recipes from normal-mode ranges.""" + seeds=set(); aliases=[]; pending=None + for line in lines: + m=re.match(r'F ([0-9A-Fa-f]+)',line) + if m: + pending=int(m.group(1),16) + seeds.add(pending if pending>=0x80000000 else (0x80000000|pending)) + continue + m=re.match(r'R ([0-9A-Fa-f]+) ([0-9A-Fa-f]+)',line) + if m and pending is not None: + lo=int(m.group(1),16); size=int(m.group(2),16) + entry=pending if pending>=0x80000000 else (0x80000000|pending) + lo=lo if lo>=0x80000000 else (0x80000000|lo) + if size and entry != lo and lo <= entry < lo+size: + aliases.append((entry,lo,lo+size)) + pending=None + return sorted(seeds),sorted(set(aliases)) + def full_discovery_seeds(data, recompiler, tmp): - """Run recompiler NORMAL mode on a PS-X EXE; return discovered function entries.""" + """Run NORMAL mode; return discovered entries plus exact alias recipes.""" exe_path = os.path.join(tmp, 'producer.exe') open(exe_path,'wb').write(data) - out = os.path.join(tmp, 'disc_out'); os.makedirs(out, exist_ok=True) + out = os.path.join(tmp, f'disc_out_{binascii.crc32(data)&0xFFFFFFFF:08X}') + os.makedirs(out, exist_ok=True) try: # errors='replace': the recompiler prints non-ASCII (✓) that would raise # a decode error under text=True and lose the (already-written) ranges. subprocess.run([recompiler, exe_path, '--out-dir', out], capture_output=True, text=True, errors='replace', timeout=300) except Exception as e: - print(f" full-discovery failed ({e}); falling back to prologue scan"); return None - seeds=set() + print(f" full-discovery failed ({e}); falling back to prologue scan") + return None,[] + lines=[] for rf in [f for f in os.listdir(out) if f.endswith('.ranges')]: - for line in open(os.path.join(out,rf), errors='ignore'): - m=re.match(r'F ([0-9A-Fa-f]+)', line) # normal-mode: "F " (no crc suffix) - if m: - a=int(m.group(1),16); seeds.add(a if a>=0x80000000 else (0x80000000|a)) - return sorted(seeds) or None + lines.extend(open(os.path.join(out,rf), errors='ignore')) + seeds,aliases=parse_full_discovery_ranges(lines) + return (seeds,aliases) if seeds else (None,[]) + +def filter_full_discovery_seeds(body, base, candidates, declared_entry): + """Quarantine normal mode's unproven image-body fallback entry. + + Normal mode derives interior entries from return/prologue/call/control-flow + evidence and classifies their reachable extents. Its one provenance-free + fallback is the image load address: a backward return scan that reaches the + beginning promotes that first body word. Ape MINI2 begins with a pointer + table there, while its PS-X header declares the real entry later. Quarantine + only that unproven body-start fallback; preserving the additive interior set + is essential because removing selected roots can split otherwise broad + functions and create native code-range holes. + """ + return sorted({addr for addr in candidates + if addr != base or addr == declared_entry}) def rec(load_addr, data, seeds, dispatch_extra=None, producer_ranges=None, - static_discovery=None): + static_discovery=None, static_alias_ranges=None): # function_entry_pcs = prologue-scanned function starts (walk roots). # dispatch_entry_pcs = those PLUS any extra dispatch targets (e.g. the # overlay's own export table). compile_overlays promotes clean starts to @@ -266,6 +516,11 @@ def rec(load_addr, data, seeds, dispatch_extra=None, producer_ranges=None, if static_discovery: out["static_discovery_entry_pcs"]=[ f"0x{addr:08X}" for addr in sorted(set(static_discovery))] + if static_alias_ranges: + out["static_alias_ranges"]=[ + {"entry":f"0x{entry:08X}", "start":f"0x{lo:08X}", + "end":f"0x{hi:08X}"} + for entry,lo,hi in sorted(set(static_alias_ranges))] return out def bios_resident_records(bios_path=None, manifest_path=None): @@ -393,13 +648,18 @@ def main(): bp,raw=parse_cue_datatrack(disc) # multi-bin safe: pick track-1 data bin dr=eo.DiscReader(bp,raw=raw) files=list(eo.enumerate_files(dr)) + hed_groups=hed_companion_members(dr,files) + hed_consumed=set().union(*(group['consumed'] for group in hed_groups)) \ + if hed_groups else set() tmp=a.tmp or tempfile.mkdtemp() os.makedirs(tmp, exist_ok=True) # The boot EXE (game.toml `exe`) is the STATIC-recompiled base, NOT an overlay. exe_base=os.path.basename(str(game.get('exe',''))).upper() - records=[]; np=nh=nr=nc=0; seen_crc=set(); header_files=[]; raw_files=[] + records=[]; np=nh=nr=na=ne=nc=0; seen_crc=set(); header_files=[]; raw_files=[] positioned=[] for p,l,s in sorted(files): + if p.upper() in hed_consumed: + continue if s<8 or s>2_000_000: continue if exe_base and os.path.basename(p).upper()==exe_base: continue # skip the statically-compiled base executable @@ -410,11 +670,15 @@ def main(): if dcrc in seen_crc: continue seen_crc.add(dcrc) if is_psx_exe(data): + entry_pc=struct.unpack_from(' {a.out}") if __name__=='__main__': main() diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index decd0983..46377f3c 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -854,6 +854,26 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, for i in range(1, len(producer_ranges))): raise RuntimeError('producer_ranges overlap') + static_alias_ranges = set() + for raw_alias in cap.get('static_alias_ranges', []) or []: + if not isinstance(raw_alias, dict): + raise RuntimeError('static_alias_ranges entries must be objects') + try: + entry = _parse_addr(raw_alias['entry']) + range_lo = _parse_addr(raw_alias['start']) + range_hi = _parse_addr(raw_alias['end']) + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeError( + f'invalid static_alias_ranges entry: {raw_alias}') from exc + if not (lo <= range_lo <= entry < range_hi <= hi and + (entry & 3) == 0 and (range_lo & 3) == 0 and + (range_hi & 3) == 0): + raise RuntimeError( + f'static alias 0x{entry:08X} -> ' + f'0x{range_lo:08X}..0x{range_hi:08X} outside/misaligned for ' + f'capture 0x{lo:08X}..0x{hi:08X}') + static_alias_ranges.add((entry, range_lo, range_hi)) + schema_keys = {'executed_pcs', 'observed_pcs', 'dispatch_entry_pcs', 'function_entry_pcs'} has_split_schema = any(k in cap for k in schema_keys) @@ -1108,6 +1128,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, 'counts': counts, 'excluded_counts': excluded_counts, 'producer_ranges': producer_ranges, + 'static_alias_ranges': static_alias_ranges, 'rejected_cross_producer_calls': rejected_cross_producer_calls, 'accepted_cross_producer_calls': accepted_cross_producer_calls, } @@ -1131,7 +1152,10 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, seeds.append(f'producer_range 0x{range_lo:08X} 0x{range_hi:08X}') for addr in sorted(accepted_cross_producer_calls): seeds.append(f'cross_call_allow 0x{addr:08X}') - for addr, range_lo, range_hi in cap.get('_prior_aliases', []): + retained_aliases = static_alias_ranges | { + (int(addr), int(range_lo), int(range_hi)) + for addr, range_lo, range_hi in cap.get('_prior_aliases', [])} + for addr, range_lo, range_hi in sorted(retained_aliases): seeds.append( f'retained_alias 0x{addr:08X} 0x{range_lo:08X} 0x{range_hi:08X}') seeds.extend(seed_line(addr) for addr in sorted(included)) From 5e8cc5b8e5775263d48ccaa6c07c99779866ca9e Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 07:49:29 -0700 Subject: [PATCH 20/37] aot: enrich archive discovery with safe fallback --- docs/AOT_OVERLAY_PLAN.md | 12 ++-- recompiler/tests/test_aot_base_recovery.py | 8 +++ .../tests/test_aot_overlay_discovery.py | 29 ++++++++ tools/aot_overlay_spike/README.md | 25 +++++-- tools/aot_overlay_spike/coverage_report.py | 17 +++-- tools/aot_overlay_spike/extract_generic.py | 71 ++++++++++++++++++- tools/compile_overlays.py | 37 ++++++++++ 7 files changed, 182 insertions(+), 17 deletions(-) diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index 4027253f..651df507 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -32,10 +32,14 @@ continue to fail safely to the interpreter. - Link bases are not guessed from title constants. Multiple independent members must anchor one sharp jal/prologue consensus; a sibling then needs at least two votes for that exact trusted base. -- Mixed archive members seed only locally callable direct-JAL targets. The first - MMX6 pointer-table expansion demonstrated why: seven variants hit the generated-C - unsupported-instruction audit. Direct-only roots compile all 30 members cleanly, - plus the exact-hash BIOS resident shard. +- Mixed archive members keep locally callable direct-JAL targets as a conservative + retry recipe. Normal-mode entries and exact aliases are optional enrichment: + generated-C audit or GCC rejection automatically retries the same byte-proven + member with direct roots only. The final disposable MMX6 build compiled all 30 + members plus the BIOS resident shard (`ok=31 failed=0 skipped=0`). Across 16 + legacy capture files it covers 636/677 entries by overlay code range (93.9%); + all 41 residual gaps are BIOS/kernel PCs, and the separately recompiled base + BIOS raises combined native code-range recall to 677/677 (100.0%). - Ape's two PS-X EXE mirrors are deduplicated in favor of the self-describing EXE producer. Its remaining 44 HED/BNS members plus two minigame EXEs and the BIOS recipe are also audit-clean in a disposable cache. diff --git a/recompiler/tests/test_aot_base_recovery.py b/recompiler/tests/test_aot_base_recovery.py index 4fc88bbf..7447a372 100644 --- a/recompiler/tests/test_aot_base_recovery.py +++ b/recompiler/tests/test_aot_base_recovery.py @@ -38,6 +38,14 @@ def main(): struct.pack_into("> 2) & 0x03FFFFFF)) assert MOD.jal_targets(data) == {0x80105678} + wrapped = MOD.make_psx_exe(b'\x11\x22\x33\x44', 0x80102000, + 0x80102040) + assert wrapped[:8] == b'PS-X EXE' + assert struct.unpack_from('=8-vote, >=2x jal-to-prologue peak; siblings need at least two -votes for that exact consensus. Mixed archive members use **direct-JAL roots -only**. A broader pointer-table experiment compiled 24/31 but correctly failed -seven generated-C audits; removing those speculative roots produced -`PSX_SHARD_RESULT ok=31 failed=0 skipped=0` (30 members + BIOS resident). Against -the union of 16 legacy MMX6 capture files this first conservative pass covers -71.8% of entries by combined native code range. The remaining gap is discovery -inside byte-proven members, not container enumeration. +votes for that exact consensus. Mixed archive members retain **direct-JAL roots** +as their conservative recipe. Normal-mode discovery may additionally contribute +decoder-classified entries and exact overlapping alias ranges. This enrichment +is explicitly optional: `compile_overlays.py` attempts it first, but any +generated-C audit or host-compiler rejection retries the exact same bytes with +direct-call roots only. A rejected recipe never contributes a DLL or a failure +to the final cache. + +The validation compile produced `PSX_SHARD_RESULT ok=31 failed=0 skipped=0` +(30 members + BIOS resident); unsafe enrichment fell back while clean members +retained broad ranges. Against the union of 16 legacy MMX6 capture files, overlay +code-range recall rose from the direct-only **71.8%** baseline to **93.9%** +(636/677). Every one of the remaining 41 PCs is in the BIOS/kernel window; +including the separately recompiled, live-byte-guarded base BIOS raises combined +native code-range recall to **100.0%** (677/677). Validation used only disposable +`%TEMP%` caches. Ape Escape's `KKIIDDZZ.HED` encodes contiguous `size_sectors:12 | logical_sector:20` runs spanning sibling DAT then BNS files. diff --git a/tools/aot_overlay_spike/coverage_report.py b/tools/aot_overlay_spike/coverage_report.py index a83ffc0e..873b0fc6 100644 --- a/tools/aot_overlay_spike/coverage_report.py +++ b/tools/aot_overlay_spike/coverage_report.py @@ -20,7 +20,7 @@ Recall is reported at three strengths: Usage: coverage_report.py --static --vault [--bios-dispatch ] - [--captures ] + [--captures ]... [--addendum ] [--prior-report --assume-static-superset] --out-md --out-json [--game ] @@ -290,7 +290,8 @@ def main(): help='generated base-BIOS dispatch C; adds already-native BIOS ' 'entries/ranges to separate combined metrics') ap.add_argument('--vault', help='played/coverage vault cache dir (.ranges)') - ap.add_argument('--captures', help='runtime overlay_captures.json (executed_pcs)') + ap.add_argument('--captures', action='append', + help='runtime overlay_captures.json (repeat for multiple sessions)') ap.add_argument('--addendum', help='append-only runtime capture history (.jsonl)') ap.add_argument('--prior-report', help='roll forward a persisted live gap set') ap.add_argument('--assume-static-superset', action='store_true', @@ -404,9 +405,15 @@ def main(): live = set() live_sources = [] - if a.captures and os.path.exists(a.captures): - live.update(parse_captures_executed(a.captures)) - live_sources.append('latest capture') + capture_source_count = 0 + for captures_path in a.captures or []: + if os.path.exists(captures_path): + live.update(parse_captures_executed(captures_path)) + capture_source_count += 1 + if capture_source_count: + live_sources.append( + f'{capture_source_count} capture file' + f'{"s" if capture_source_count != 1 else ""}') if a.addendum and os.path.exists(a.addendum): addendum_entries, addendum_audit = parse_addendum_entries(a.addendum) live.update(addendum_entries) diff --git a/tools/aot_overlay_spike/extract_generic.py b/tools/aot_overlay_spike/extract_generic.py index 9617c8fc..12e6d49a 100644 --- a/tools/aot_overlay_spike/extract_generic.py +++ b/tools/aot_overlay_spike/extract_generic.py @@ -432,6 +432,20 @@ def page_aligned_region(base, data): def is_psx_exe(data): return len(data) >= 0x20 and data[:8] == b'PS-X EXE' +def make_psx_exe(body, load_addr, entry_pc): + """Wrap positioned bytes for normal-mode discovery only. + + The wrapper is never emitted or loaded by the runtime. Its header merely + gives the recompiler the independently recovered link base and one locally + proven direct-call entry while preserving the member bytes verbatim. + """ + header=bytearray(0x800) + header[:8]=b'PS-X EXE' + struct.pack_into(' set[int]: return out +def optional_enrichment_fallback_capture(cap: dict) -> dict | None: + """Return a conservative retry recipe declared by a static extractor. + + The richer recipe is always attempted first. If its generated C or host + compile is rejected, these independently proven roots preserve the safe + shard while all optional aliases and normal-mode discoveries are dropped. + """ + key = 'optional_enrichment_fallback_entry_pcs' + entries = sorted(_parse_addr_list(cap.get(key, []))) + if not entries: + return None + encoded = [f'0x{entry:08X}' for entry in entries] + fallback = dict(cap) + fallback['function_entry_pcs'] = encoded + fallback['dispatch_entry_pcs'] = encoded + fallback['static_discovery_entry_pcs'] = encoded + fallback['seeds'] = encoded + fallback.pop('static_alias_ranges', None) + fallback.pop('_prior_aliases', None) + fallback.pop(key, None) + return fallback + + def _word_at(data: bytes, load_addr: int, addr: int): off = addr - load_addr if off < 0 or off + 4 > len(data): @@ -2657,6 +2680,13 @@ def main(): with open(debug_c, 'w') as f: f.write(src) if c_audit['unknown_bad'] or c_audit['unsupported_todo_addrs']: + fallback = optional_enrichment_fallback_capture(cap) + if fallback is not None: + print(' OPTIONAL ENRICHMENT AUDIT REJECTED; ' + 'retrying direct-call roots only\n') + _do_capture(fallback, region_coverage_cache, + interior_frag_jobs, stats) + return print(' GENERATED-C AUDIT FAILED\n') stats.add_fail(_label, 'audit', f'{len(c_audit["unknown_bad"])} unknown_bad, ' @@ -2735,6 +2765,13 @@ def main(): stats.add_fail(_label, 'no_ranges', 'DLL built but no _full.ranges (undispatchable)') else: + fallback = optional_enrichment_fallback_capture(cap) + if fallback is not None: + print(' OPTIONAL ENRICHMENT COMPILE REJECTED; ' + 'retrying direct-call roots only\n') + _do_capture(fallback, region_coverage_cache, + interior_frag_jobs, stats) + return print(f' FAILED\n') stats.add_fail(_label, 'compile', 'gcc/tcc compile failed (see COMPILE ERROR above)') From 32cc2e81f426436ff4bb169114f2e603d11cb0a7 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 08:27:09 -0700 Subject: [PATCH 21/37] aot: bound header dispatch fallback discovery --- docs/AOT_OVERLAY_PLAN.md | 7 ++ recompiler/tests/test_aot_base_recovery.py | 21 ++++++ .../tests/test_aot_overlay_discovery.py | 26 +++++++ tools/aot_overlay_spike/README.md | 12 ++++ tools/aot_overlay_spike/extract_generic.py | 70 ++++++++++++++++--- tools/compile_overlays.py | 62 ++++++++++++---- 6 files changed, 174 insertions(+), 24 deletions(-) diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index 651df507..d58e43a9 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -50,6 +50,13 @@ continue to fail safely to the interpreter. 100% combined BIOS code-range coverage for both vault and live history. - Large archive reads are now linear rather than quadratic sector concatenation. These validations wrote only `%TEMP%` caches; title caches remain untouched. +- Tomba 1 header exports are authoritative mid-function dispatch entries rather + than callable roots. Their conservative retry now uses only the nearest + preceding prologue host, hard-stops it at the next prologue, and rejects calls + outside those producer ranges. The clean 25-record corpus built 18 unique DLLs + with zero failures (the rest deduplicated); across 24 legacy sessions it covers + 86.2% of played PCs by overlay code range and 95.1% with base BIOS/kernel code, + leaving 22 genuine range misses rather than admitting embedded data as code. --- diff --git a/recompiler/tests/test_aot_base_recovery.py b/recompiler/tests/test_aot_base_recovery.py index 7447a372..3647d015 100644 --- a/recompiler/tests/test_aot_base_recovery.py +++ b/recompiler/tests/test_aot_base_recovery.py @@ -38,6 +38,27 @@ def main(): struct.pack_into("> 2) & 0x03FFFFFF)) assert MOD.jal_targets(data) == {0x80105678} + # A JAL target becomes a conservative shard root only when the target is + # local and has independent callable-boundary evidence. + rooted = bytearray(0x80) + struct.pack_into('> 2) & 0x03FFFFFF)) + struct.pack_into('>16)==0x27BD)) + +def bounded_dispatch_fallback(data, base, entries): + """Bound authoritative mid-function entries to their nearest hosts. + + Header-table exports are real dispatch targets but are commonly switch + cases rather than function starts. For a conservative retry, pair each + with only its nearest preceding prologue and stop that host at the very + next prologue. A host without a provable upper boundary is omitted. + """ + starts=prologues(data,base) + if len(starts)<2: + return None + hosts=set(); dispatch=set(); ranges=set() + for entry in sorted(set(entries)): + before=[start for start in starts if start<=entry] + if not before: + continue + host=before[-1] + end=next((start for start in starts if start>host),None) + if end is None or not (host<=entry1 else 0 if score < 2 or score < runner*2: continue - lo,hi=base,base+len(body) - direct={t for t in jal_targets(body) - if lo <= t < hi and (t&3)==0 and - (_callable_boundary(body,t-base) or - ((_word(body,t-base) or 0)>>16)==0x27BD)} - seeds=sorted(direct) + direct=direct_jal_roots(body,base) + seeds=list(direct) if len(direct) < 2: continue recovered.append({ 'id':ident, 'file_offset':off, 'data':body, 'base':base, - 'score':score, 'runner':runner, 'direct_seeds':sorted(direct), + 'score':score, 'runner':runner, 'direct_seeds':direct, 'seeds':seeds, }) return recovered @@ -815,7 +852,17 @@ def main(): page_base = base & ~0xFFF fill = base - page_base region = b'\x00'*fill + data - records.append(rec(page_base, region, seeds, dispatch_extra=hdr_entries)) + direct=direct_jal_roots(data,base) + record=rec(page_base, region, seeds, dispatch_extra=hdr_entries, + static_discovery=direct) + fallback=bounded_dispatch_fallback(data,base,hdr_entries) + if fallback: + # The on-disc export table is authoritative dispatch evidence but + # its entries are commonly mid-function. On audit rejection, + # classify those entries from scratch without retaining broad + # prologue/pointer guesses or trusting unrooted JAL-shaped data. + record['optional_enrichment_fallback_entry_pcs'] = fallback + records.append(record) positioned.append((p,data,base,seeds,hdr_entries)) nh+=1 extra=len(set(hdr_entries)-set(seeds)) @@ -838,7 +885,12 @@ def main(): supplemental=supplemental_callable_seeds(data,base) seeds=sorted(set(prologue_seeds)|supplemental) page_base,region=page_aligned_region(base,data) - records.append(rec(page_base,region,seeds)) + direct=direct_jal_roots(data,base) + record=rec(page_base,region,seeds,static_discovery=direct) + if direct and set(seeds) != set(direct): + record['optional_enrichment_fallback_entry_pcs'] = [ + f"0x{addr:08X}" for addr in direct] + records.append(record) positioned.append((p,data,base,seeds,())) nr+=1 print(f" [raw-code] {p}: {len(data)}B file@0x{base:08X} " diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index 2e976222..712c75a3 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -364,15 +364,38 @@ def optional_enrichment_fallback_capture(cap: dict) -> dict | None: shard while all optional aliases and normal-mode discoveries are dropped. """ key = 'optional_enrichment_fallback_entry_pcs' - entries = sorted(_parse_addr_list(cap.get(key, []))) - if not entries: - return None - encoded = [f'0x{entry:08X}' for entry in entries] + recipe = cap.get(key, []) fallback = dict(cap) - fallback['function_entry_pcs'] = encoded - fallback['dispatch_entry_pcs'] = encoded - fallback['static_discovery_entry_pcs'] = encoded - fallback['seeds'] = encoded + if isinstance(recipe, dict): + functions = sorted(_parse_addr_list( + recipe.get('function_entry_pcs', []))) + dispatch = sorted(_parse_addr_list( + recipe.get('dispatch_entry_pcs', []))) + static = sorted(_parse_addr_list( + recipe.get('static_discovery_entry_pcs', []))) + if not (functions or dispatch or static): + return None + fallback['function_entry_pcs'] = [ + f'0x{entry:08X}' for entry in functions] + fallback['dispatch_entry_pcs'] = [ + f'0x{entry:08X}' for entry in dispatch] + fallback['static_discovery_entry_pcs'] = [ + f'0x{entry:08X}' for entry in static] + fallback['seeds'] = fallback['function_entry_pcs'] + if 'producer_ranges' in recipe: + fallback['producer_ranges'] = recipe['producer_ranges'] + if 'strict_producer_ranges' in recipe: + fallback['strict_producer_ranges'] = bool( + recipe['strict_producer_ranges']) + else: + entries = sorted(_parse_addr_list(recipe)) + if not entries: + return None + encoded = [f'0x{entry:08X}' for entry in entries] + fallback['function_entry_pcs'] = encoded + fallback['dispatch_entry_pcs'] = encoded + fallback['static_discovery_entry_pcs'] = encoded + fallback['seeds'] = encoded fallback.pop('static_alias_ranges', None) fallback.pop('_prior_aliases', None) fallback.pop(key, None) @@ -693,7 +716,8 @@ def _plausible_callable_target(data: bytes, load_addr: int, size: int, def _walk_overlay_function(data: bytes, load_addr: int, size: int, entry: int, hard_cap: int, - producer_ranges=()) -> dict: + producer_ranges=(), + allow_cross_producer_calls=True) -> dict: lo = load_addr hi = load_addr + size @@ -723,6 +747,9 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, target_range = producer_for(target) if source_range is not None and source_range == target_range: return True + if not allow_cross_producer_calls: + rejected_cross_producer_calls.add(target) + return False # Adjacent-producer composites are a byte envelope, not proof that a # call encoded by producer A is linked to arbitrary bytes at the same # address in every producer-B variant. Cross-boundary calls therefore @@ -878,6 +905,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, raise RuntimeError('producer_ranges overlap') static_alias_ranges = set() + allow_cross_producer_calls = not bool(cap.get('strict_producer_ranges')) for raw_alias in cap.get('static_alias_ranges', []) or []: if not isinstance(raw_alias, dict): raise RuntimeError('static_alias_ranges entries must be objects') @@ -929,7 +957,8 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, for i, entry in enumerate(pre_roots_sorted): hard_cap = pre_roots_sorted[i + 1] if i + 1 < len(pre_roots_sorted) else hi walk = _walk_overlay_function( - data, load_addr, size, entry, hard_cap, producer_ranges) + data, load_addr, size, entry, hard_cap, producer_ranges, + allow_cross_producer_calls) jump_table_targets.update(walk['jump_table_targets']) def impossible_entry_start(addr: int) -> bool: @@ -1030,7 +1059,8 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, sorted_known = sorted(known) hard_cap = next((x for x in sorted_known if x > entry), hi) walk = _walk_overlay_function( - data, load_addr, size, entry, hard_cap, producer_ranges) + data, load_addr, size, entry, hard_cap, producer_ranges, + allow_cross_producer_calls) all_branch_targets.update(walk['branch_targets']) all_branch_targets.update(walk['jump_table_targets']) for target in sorted(walk['direct_jals']): @@ -1084,7 +1114,8 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, for i, entry in enumerate(sorted_known): hard_cap = sorted_known[i + 1] if i + 1 < len(sorted_known) else hi walk = _walk_overlay_function( - data, load_addr, size, entry, hard_cap, producer_ranges) + data, load_addr, size, entry, hard_cap, producer_ranges, + allow_cross_producer_calls) covered |= walk['visited'] promoted = sorted(a for a, r in included.items() if r == 'DISPATCH_INTERIOR' and a not in covered @@ -1109,7 +1140,8 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, for i, entry in enumerate(sorted_known): hard_cap = sorted_known[i + 1] if i + 1 < len(sorted_known) else hi walk = _walk_overlay_function( - data, load_addr, size, entry, hard_cap, producer_ranges) + data, load_addr, size, entry, hard_cap, producer_ranges, + allow_cross_producer_calls) all_branch_targets.update(walk['branch_targets']) all_branch_targets.update(walk['jump_table_targets']) all_branch_targets.update(walk['forward_branch_targets']) @@ -2683,7 +2715,7 @@ def main(): fallback = optional_enrichment_fallback_capture(cap) if fallback is not None: print(' OPTIONAL ENRICHMENT AUDIT REJECTED; ' - 'retrying direct-call roots only\n') + 'retrying conservative recipe\n') _do_capture(fallback, region_coverage_cache, interior_frag_jobs, stats) return @@ -2768,7 +2800,7 @@ def main(): fallback = optional_enrichment_fallback_capture(cap) if fallback is not None: print(' OPTIONAL ENRICHMENT COMPILE REJECTED; ' - 'retrying direct-call roots only\n') + 'retrying conservative recipe\n') _do_capture(fallback, region_coverage_cache, interior_frag_jobs, stats) return From ae1f1e7f197f5517744fe06897bb7ad094655607 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 08:59:26 -0700 Subject: [PATCH 22/37] tools: report true AOT coverage gaps --- docs/AOT_OVERLAY_PLAN.md | 7 +++++++ tools/aot_overlay_spike/README.md | 12 ++++++++++++ tools/aot_overlay_spike/coverage_report.py | 14 ++++++++++++-- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index d58e43a9..d16c9af1 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -43,6 +43,13 @@ continue to fail safely to the interpreter. - Ape's two PS-X EXE mirrors are deduplicated in favor of the self-describing EXE producer. Its remaining 44 HED/BNS members plus two minigame EXEs and the BIOS recipe are also audit-clean in a disposable cache. +- A fresh 47/47 clean Ape rebuild then completed a title-to-attract-to-title + soak across four overlay generations. All 72 exercised game PCs lie in static + compiled ranges; the other 35 PCs are covered by the separately generated + BIOS/kernel ranges, yielding 107/107 combined range coverage with zero true + gaps. Live-byte guards safely rejected 21 stale candidates, visuals remained + correct, and the dev history recorded 21 verified immutable snapshots with no + invalid or duplicate references. - Normal PS-X discovery now exports its exact overlapping `F/R` alias recipes. Clean overlay builds therefore preserve decoder-proven indirect entries without promoting every entry to a hard root or borrowing a previous cache manifest. diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 8df675c4..6ba2e715 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -262,6 +262,18 @@ framed scans likewise recover Psy-Q's exact the old root eight bytes late. This closes Tomba 1's historical `0x80110E30` gap in the clean X00 variant without minting overlapping functions. +A fresh Ape validation rebuilt all **47/47** candidates with zero audit or host- +compiler failures. A full title-to-attract-to-title cycle exercised four overlay +generations and 107 dispatch PCs. The overlay cache covers all 72 exercised game +PCs by compiled range; the remaining 35 are BIOS/kernel PCs, and the separately +generated base BIOS covers them all (**100.0% combined code-range recall, zero +true gaps**). Live-byte validation rejected 21 stale candidates without executing +them, while every loaded shard reported zero missing manifests and both loader +indexes stayed within capacity. Screenshots verified the rainbow/falling attract +scene and returned title screen rendered correctly. The dev persistence profile +produced 21/21 signed immutable snapshots with zero invalid or duplicate records; +production uses only the append-only executable-local history. + ## Next to raise coverage (future session) 1. ~~Direct-call frameless-leaf discovery~~ — DONE. Reachable `jal` targets are diff --git a/tools/aot_overlay_spike/coverage_report.py b/tools/aot_overlay_spike/coverage_report.py index 873b0fc6..13fb4e0a 100644 --- a/tools/aot_overlay_spike/coverage_report.py +++ b/tools/aot_overlay_spike/coverage_report.py @@ -446,11 +446,16 @@ def main(): md.append(f'- Covered by compiled static code ranges: ' f'**{r["covered_by_code_range"]}** ' f'(**{r["recall_code_range"]*100:.1f}%** code-range recall)') + md.append(f'- Overlay-only true code-range gaps: ' + f'**{r["missed_code_range"]}**') if combined: md.append(f'- Including base BIOS native code ranges: ' f'**{combined["covered_by_code_range"]}** ' f'(**{combined["recall_code_range"]*100:.1f}%**)') - md.append(f'- **MISSED live: {r["missed"]}**\n') + md.append(f'- Combined true code-range gaps: ' + f'**{combined["missed_code_range"]}**') + md.append(f'- Exact-entry misses (diagnostic; may be interior fragments): ' + f'**{r["missed"]}**\n') if a.prior_report: if a.prior_report == '-': prior = json.load(sys.stdin) @@ -531,11 +536,16 @@ def main(): md.append(f'- Covered by current static code ranges: ' f'**{r["covered_by_code_range"]}** ' f'(**{r["recall_code_range"]*100:.1f}%** code-range recall)') + md.append(f'- Overlay-only true code-range gaps: ' + f'**{r["missed_code_range"]}**') if combined: md.append(f'- Including base BIOS native code ranges: ' f'**{combined["covered_by_code_range"]}** ' f'(**{combined["recall_code_range"]*100:.1f}%**)') - md.append(f'- **MISSED live: {len(miss)}**') + md.append(f'- Combined true code-range gaps: ' + f'**{combined["missed_code_range"]}**') + md.append(f'- Exact-entry misses (diagnostic; may be interior fragments): ' + f'**{len(miss)}**') md.append('- Provenance: caller explicitly asserted that the current static ' 'entry set retains every prior static entry.\n') From 6af54370580da1a351307355391404ec6295aed4 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 09:05:41 -0700 Subject: [PATCH 23/37] tools: publish overlay shards atomically --- docs/AOT_OVERLAY_PLAN.md | 4 ++ .../tests/test_aot_overlay_discovery.py | 39 +++++++++++++ tools/aot_overlay_spike/README.md | 7 +++ tools/compile_overlays.py | 55 +++++++++++++++++-- 4 files changed, 100 insertions(+), 5 deletions(-) diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index d16c9af1..fc541675 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -57,6 +57,10 @@ continue to fail safely to the interpreter. 100% combined BIOS code-range coverage for both vault and live history. - Large archive reads are now linear rather than quadratic sector concatenation. These validations wrote only `%TEMP%` caches; title caches remain untouched. +- DLL publication is now atomic: compilers target a unique sibling temporary, + successful non-empty output replaces the final cache name, and cache reuse + requires its non-empty range manifest. Interrupted compiles can no longer + poison later runs or overwrite a prior good shard during `--force`. - Tomba 1 header exports are authoritative mid-function dispatch entries rather than callable roots. Their conservative retry now uses only the nearest preceding prologue host, hard-stops it at the next prologue, and rejects calls diff --git a/recompiler/tests/test_aot_overlay_discovery.py b/recompiler/tests/test_aot_overlay_discovery.py index bd6e60cc..bc250616 100644 --- a/recompiler/tests/test_aot_overlay_discovery.py +++ b/recompiler/tests/test_aot_overlay_discovery.py @@ -288,6 +288,44 @@ def check_retained_alias_contract(recompiler): assert f"R {LOAD:08X} 2C" in ranges +def check_atomic_dll_publication(): + original = MOD._compile_dll_direct + try: + with tempfile.TemporaryDirectory() as tmp: + final = os.path.join(tmp, "shard.dll") + ranges = os.path.join(tmp, "shard.ranges") + with open(final, "wb") as out: + out.write(b"old-good-shard") + + def fail_compile(_src, staged, _includes, **_kwargs): + with open(staged, "wb") as out: + out.write(b"partial") + return False + + MOD._compile_dll_direct = fail_compile + assert not MOD.compile_dll("ignored.c", final, []) + with open(final, "rb") as built: + assert built.read() == b"old-good-shard" + assert not list(pathlib.Path(tmp).glob(".*.tmp.*")) + assert not MOD.compiled_shard_complete(final) + + def good_compile(_src, staged, _includes, **_kwargs): + with open(staged, "wb") as out: + out.write(b"new-complete-shard") + return True + + MOD._compile_dll_direct = good_compile + assert MOD.compile_dll("ignored.c", final, []) + with open(final, "rb") as built: + assert built.read() == b"new-complete-shard" + with open(ranges, "w", encoding="utf-8") as out: + out.write("F 80010000 DEADBEEF\nR 80010000 10\n") + assert MOD.compiled_shard_complete(final) + assert not list(pathlib.Path(tmp).glob(".*.tmp.*")) + finally: + MOD._compile_dll_direct = original + + def main(): default_recompiler = ROOT / "recompiler" / "build" / "psxrecomp-game.exe" parser = argparse.ArgumentParser() @@ -303,6 +341,7 @@ def main(): check_forward_branch_root() check_recompiler_composite_contract(args.recompiler) check_retained_alias_contract(args.recompiler) + check_atomic_dll_publication() data = bytearray(0x200) diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 6ba2e715..e4088066 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -274,6 +274,13 @@ scene and returned title screen rendered correctly. The dev persistence profile produced 21/21 signed immutable snapshots with zero invalid or duplicate records; production uses only the append-only executable-local history. +Shard publication is interruption-safe. GCC/tcc now writes a unique sibling +temporary DLL and the tool atomically replaces the content-keyed cache name only +after a successful, non-empty output. Cache reuse additionally requires a +non-empty `.ranges` manifest. A killed compiler therefore cannot leave a partial +final-named DLL that a later build mistakes for a valid shard, and a failed +`--force` rebuild cannot destroy the previous good DLL. + ## Next to raise coverage (future session) 1. ~~Direct-call frameless-leaf discovery~~ — DONE. Reachable `jal` targets are diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index 712c75a3..dcddb9b0 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -2078,7 +2078,7 @@ def compile_interior_fragment(interior: int, data: bytes, load_addr: int, struct.pack(' bool: +def _compile_dll_direct(c_path: str, out_dll: str, include_dirs: list[str], + gcc: str = 'gcc', flavor: int = 0, + compiler: str = 'gcc', tcc: str = 'tcc') -> bool: import platform # Absolute paths (interpreter flavor) for EVERY path the native compiler # touches — see native_path's docstring for the two rules and the @@ -2262,6 +2262,50 @@ def compile_dll(c_path: str, out_dll: str, include_dirs: list[str], return True +def compile_dll(c_path: str, out_dll: str, include_dirs: list[str], + gcc: str = 'gcc', flavor: int = 0, + compiler: str = 'gcc', tcc: str = 'tcc') -> bool: + """Publish a shard only after its compiler has completed successfully. + + GCC and tcc write their output incrementally. If the process is interrupted + while targeting the final cache name, the next run sees that partial file + and used to skip it as if it were complete. Compile to a unique sibling and + atomically replace the final DLL instead; an existing good shard therefore + also survives a failed --force rebuild. + """ + final_out = os.path.abspath(out_dll) + out_dir = os.path.dirname(final_out) + os.makedirs(out_dir, exist_ok=True) + suffix = os.path.splitext(final_out)[1] or overlay_ext() + fd, staged = tempfile.mkstemp( + prefix=f'.{os.path.basename(final_out)}.tmp.', suffix=suffix, + dir=out_dir) + os.close(fd) + os.unlink(staged) # let the native compiler create its output normally + try: + if not _compile_dll_direct( + c_path, staged, include_dirs, gcc=gcc, flavor=flavor, + compiler=compiler, tcc=tcc): + return False + if not os.path.isfile(staged) or os.path.getsize(staged) == 0: + print(' COMPILE ERROR: compiler reported success but emitted no DLL') + return False + os.replace(staged, final_out) + return True + finally: + try: + os.unlink(staged) + except FileNotFoundError: + pass + + +def compiled_shard_complete(dll_path: str) -> bool: + """A cached shard is dispatchable only with its non-empty range manifest.""" + ranges = os.path.splitext(dll_path)[0] + '.ranges' + return (os.path.isfile(dll_path) and os.path.getsize(dll_path) > 0 and + os.path.isfile(ranges) and os.path.getsize(ranges) > 0) + + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -2568,7 +2612,8 @@ def main(): stats.add_skip() return - if not args.static and os.path.exists(dll_path) and not args.force: + if (not args.static and compiled_shard_complete(dll_path) and + not args.force): print(' SKIP: DLL already exists (use --force to recompile)\n') stats.add_skip() return From dbe14dfadb16435c48ac7b5422773fbd7c4cf610 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 10:13:18 -0700 Subject: [PATCH 24/37] aot: recover safe frameless and interior overlay coverage --- docs/AOT_OVERLAY_PLAN.md | 14 +++-- recompiler/src/function_analysis.cpp | 61 ++++++++++++++----- recompiler/src/main_psx.cpp | 13 +++- recompiler/tests/reachable_discovery_test.cpp | 40 ++++++++++++ recompiler/tests/test_aot_base_recovery.py | 27 ++++++++ .../tests/test_aot_overlay_discovery.py | 61 +++++++++++++++++++ tools/aot_overlay_spike/README.md | 27 +++++--- tools/aot_overlay_spike/extract_generic.py | 55 ++++++++++++++++- tools/compile_overlays.py | 49 +++++++++++++-- 9 files changed, 307 insertions(+), 40 deletions(-) diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index fc541675..eacb8549 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -62,12 +62,14 @@ continue to fail safely to the interpreter. requires its non-empty range manifest. Interrupted compiles can no longer poison later runs or overwrite a prior good shard during `--force`. - Tomba 1 header exports are authoritative mid-function dispatch entries rather - than callable roots. Their conservative retry now uses only the nearest - preceding prologue host, hard-stops it at the next prologue, and rejects calls - outside those producer ranges. The clean 25-record corpus built 18 unique DLLs - with zero failures (the rest deduplicated); across 24 legacy sessions it covers - 86.2% of played PCs by overlay code range and 95.1% with base BIOS/kernel code, - leaving 22 genuine range misses rather than admitting embedded data as code. + than callable roots. Their conservative retry uses the nearest preceding + prologue host, stable-round call discovery prevents interior callees from + truncating that host, and dense local pointer tables are rejected as code. + Bounded return-to-return scanning also recovers unreferenced frameless leaves. + The clean 25-record corpus produces 17 unique shards with zero audit failures. + Overlay ranges cover 820/823 historical-vault PCs and 410/450 live-history PCs; + every residual is in the separately recompiled BIOS/kernel ranges, yielding + **100.0% combined code-range recall and zero true gaps** for both scoreboards. --- diff --git a/recompiler/src/function_analysis.cpp b/recompiler/src/function_analysis.cpp index 570e5ead..b9011d21 100644 --- a/recompiler/src/function_analysis.cpp +++ b/recompiler/src/function_analysis.cpp @@ -675,8 +675,25 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( // Requiring a stack prologue here discarded legitimate frameless leaf // functions and tiny library wrappers. The target still has to decode // as a valid PS1 instruction and lie inside the verified image bound. + // + // Three adjacent aligned words that all point back into this image are + // stronger evidence for a position-fixed pointer table than the first + // word's accidental MIPS decode. Do not follow a JAL-shaped word into + // that data. The caller remains valid and dispatches the unresolved + // target at runtime, where the ordinary live-byte guard can select a + // separately proven implementation or fall back to the interpreter. if (!word_opt.has_value() || !exact_is_valid_mips_word(*word_opt)) return false; + bool dense_local_pointer_table = true; + for (uint32_t i = 0; i < 3u; i++) { + auto value = exe_.read_word(addr + i * 4u); + if (!value.has_value() || (*value & 3u) != 0u || + !in_exe(*value)) { + dense_local_pointer_table = false; + break; + } + } + if (dense_local_pointer_table) return false; if (producer_ranges.empty()) return true; int source_producer = producer_for(source); int target_producer = producer_for(addr); @@ -860,29 +877,43 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( }; std::set known_entries; - std::queue pending; for (uint32_t entry : entries) { if (!in_exe(entry)) continue; - if (known_entries.insert(entry).second) pending.push(entry); + known_entries.insert(entry); } size_t explicit_count = known_entries.size(); fmt::print("Explicit entries: {}\n", explicit_count); - std::set processed; - while (!pending.empty()) { - uint32_t entry = pending.front(); - pending.pop(); - if (processed.count(entry)) continue; - processed.insert(entry); - - auto next_it = known_entries.upper_bound(entry); - uint32_t hard_cap = (next_it != known_entries.end()) ? *next_it : exe_.end_address(); - ExactWalkResult wr = walk(entry, hard_cap); - - for (uint32_t target : wr.direct_jal_targets) { - if (known_entries.insert(target).second) pending.push(target); + // Discover callees in rounds against one stable entry partition. Processing + // roots sequentially let an early caller insert a JAL target that actually + // lay inside a later explicit root. That new entry hard-capped the later + // root before it was walked (Tomba X01's jump-table host), splitting one + // function and losing every case after the accidental cap. If a candidate + // is already owned by any walk in the current partition, leave it absorbed; + // main_psx's range-ownership pass materializes it as an interior alias. + while (true) { + std::vector round_entries( + known_entries.begin(), known_entries.end()); + std::set owned_addresses; + std::set candidate_targets; + for (size_t i = 0; i < round_entries.size(); i++) { + uint32_t hard_cap = (i + 1 < round_entries.size()) + ? round_entries[i + 1] : exe_.end_address(); + ExactWalkResult wr = walk(round_entries[i], hard_cap); + owned_addresses.insert(wr.visited.begin(), wr.visited.end()); + candidate_targets.insert(wr.direct_jal_targets.begin(), + wr.direct_jal_targets.end()); } + + bool added = false; + for (uint32_t target : candidate_targets) { + if (known_entries.count(target) || owned_addresses.count(target)) + continue; + known_entries.insert(target); + added = true; + } + if (!added) break; } result.call_discovered_count = static_cast(known_entries.size() - explicit_count); diff --git a/recompiler/src/main_psx.cpp b/recompiler/src/main_psx.cpp index 0eb8f949..ac8aa3f5 100644 --- a/recompiler/src/main_psx.cpp +++ b/recompiler/src/main_psx.cpp @@ -564,6 +564,16 @@ int main(int argc, char** argv) { auto w = exe->read_word(a); return w.has_value() ? *w : 0u; }; + auto dense_local_pointer_table = [&](uint32_t a) -> bool { + for (uint32_t i = 0; i < 3u; i++) { + auto value = exe->read_word(a + i * 4u); + if (!value.has_value() || (*value & 3u) != 0u || + *value < exe_lo || *value >= exe->end_address()) { + return false; + } + } + return true; + }; auto callable_boundary = [&](uint32_t a) -> bool { uint32_t w = read_w(a); if (!PSXRecomp::FunctionAnalyzer::is_valid_mips_word(w)) return false; @@ -603,7 +613,8 @@ int main(int argc, char** argv) { fmt::print(" seed 0x{:08X} accepted as dispatch root " "(install-slot class, no boundary evidence)\n", a); roots.insert(a); - } else if (trusted_call_root_seeds.count(a)) { + } else if (trusted_call_root_seeds.count(a) && + !dense_local_pointer_table(a)) { fmt::print(" seed 0x{:08X} accepted as static call root " "(direct/constant-register target)\n", a); roots.insert(a); diff --git a/recompiler/tests/reachable_discovery_test.cpp b/recompiler/tests/reachable_discovery_test.cpp index 6f635f73..cebdff98 100644 --- a/recompiler/tests/reachable_discovery_test.cpp +++ b/recompiler/tests/reachable_discovery_test.cpp @@ -111,6 +111,46 @@ int main() { CHECK(!reachable.count(kLoad + 0x1100), "direct target beyond verified bound fails closed"); + // A JAL-shaped word can target an embedded table whose first pointer also + // happens to decode as a legal MIPS instruction. Three adjacent local + // pointers are stronger data evidence; keep that target interpreted. + auto pointer_image = make_exe_buffer(0x1000); + put32(pointer_image, text + 0x00, jal(kLoad + 0x400)); + put32(pointer_image, text + 0x04, 0x00000000u); + put32(pointer_image, text + 0x08, 0x03E00008u); + put32(pointer_image, text + 0x0C, 0x00000000u); + put32(pointer_image, text + 0x400, kLoad + 0x500); + put32(pointer_image, text + 0x404, kLoad + 0x520); + put32(pointer_image, text + 0x408, kLoad + 0x540); + auto pointer_exe = parse(pointer_image); + PSXRecomp::FunctionAnalyzer pointer_analyzer(pointer_exe); + const auto pointer_starts = starts( + pointer_analyzer.analyze_exact_entries({kLoad})); + CHECK(!pointer_starts.count(kLoad + 0x400), + "reachable analysis rejects a dense local pointer table target"); + + // Discover call targets in stable rounds. An early root's JAL into the + // middle of a later explicit root must remain absorbed by that host rather + // than splitting/hard-capping it before the host is walked. + auto interior_call_image = make_exe_buffer(0x1000); + put32(interior_call_image, text + 0x00, jal(kLoad + 0x108)); + put32(interior_call_image, text + 0x04, 0x00000000u); + put32(interior_call_image, text + 0x08, 0x03E00008u); + put32(interior_call_image, text + 0x0C, 0x00000000u); + put32(interior_call_image, text + 0x100, 0x24020001u); + put32(interior_call_image, text + 0x104, 0x24420001u); + put32(interior_call_image, text + 0x108, 0x24420001u); + put32(interior_call_image, text + 0x10C, 0x03E00008u); + put32(interior_call_image, text + 0x110, 0x00000000u); + auto interior_call_exe = parse(interior_call_image); + PSXRecomp::FunctionAnalyzer interior_call_analyzer(interior_call_exe); + const auto interior_call_starts = starts( + interior_call_analyzer.analyze_exact_entries( + {kLoad, kLoad + 0x100})); + CHECK(interior_call_starts == + std::set({kLoad, kLoad + 0x100}), + "direct call into an owned function stays an interior alias"); + // A synthetic adjacent-producer envelope is not one linked image. The // caller supplies the cross-boundary targets that passed its independent // callable-CFG proof; other valid-looking targets stay with the interpreter. diff --git a/recompiler/tests/test_aot_base_recovery.py b/recompiler/tests/test_aot_base_recovery.py index 3647d015..7c5289f7 100644 --- a/recompiler/tests/test_aot_base_recovery.py +++ b/recompiler/tests/test_aot_base_recovery.py @@ -48,6 +48,19 @@ def main(): struct.pack_into('> 2) & 0x03FFFFFF)) + struct.pack_into('= 3: for value in words[start:end]: - if _callable_boundary(data, value-base): + if (_callable_boundary(data, value-base) and + not _dense_local_pointer_table(data,base,value-base)): targets.add(value) start = end return targets @@ -133,6 +151,32 @@ def pointer_table_targets(data, base): def supplemental_callable_seeds(data, base): return pointer_table_targets(data, base) +def frameless_leaf_entries(data, base): + """Return independently bounded functions immediately following returns. + + A previous ``jr ra`` plus its architectural delay slot establishes the + start boundary (allowing the same six alignment NOPs as exact-entry mode). + The shared bounded-CFG verifier then requires the candidate itself to reach + a return without invalid instructions or sequential runway. This recovers + unreferenced callback leaves without promoting arbitrary return-adjacent + data; dense local-pointer tables fail that verifier explicitly. + """ + entries=set() + for jr_off in range(0,len(data)-12,4): + if _word(data,jr_off) != 0x03E00008: + continue + entry_off=jr_off+8 + padding=0 + while padding<6 and _word(data,entry_off)==0: + entry_off+=4; padding+=1 + if entry_off>len(data)-4: + continue + entry=base+entry_off + if co.plausible_callable_target( + data,base,len(data),entry,base+len(data)): + entries.add(entry) + return entries + def jal_targets(data): """Distinct absolute jal call targets. These are BASE-INDEPENDENT: the target is 0x80000000 | (imm26<<2) regardless of where the file loads, because all @@ -154,6 +198,7 @@ def direct_jal_roots(data, base): lo,hi=base,base+len(data) return sorted(t for t in jal_targets(data) if lo <= t < hi and (t&3)==0 and + not _dense_local_pointer_table(data,base,t-base) and (_callable_boundary(data,t-base) or ((_word(data,t-base) or 0)>>16)==0x27BD)) @@ -833,8 +878,9 @@ def main(): # against prologues they never point at -> wrong base by 2-16KB). base,score=rb prologue_seeds=prologues(data,base) + frameless=frameless_leaf_entries(data,base) supplemental=supplemental_callable_seeds(data,base) - seeds=sorted(set(prologue_seeds)|supplemental) + seeds=sorted(set(prologue_seeds)|frameless|supplemental) # SUPPLEMENT the prologue seeds with the overlay's OWN export/dispatch # table (the header pointers) — the game's authoritative list of live # entry points, sitting on the disc (play-free). These are exactly the @@ -869,6 +915,7 @@ def main(): print(f" [header-table] {p}: {len(data)}B file@0x{base:08X} " f"region@0x{page_base:08X}(+{fill}) " f"({fit_method} score={score}), prologue seeds={len(prologue_seeds)} " + f"+{len(frameless-set(prologue_seeds))} bounded frameless leaves " f"+{len(supplemental-set(prologue_seeds))} pointer-table seeds " f"+{extra} export-table dispatch entries") @@ -882,8 +929,9 @@ def main(): if rb is not None: base,score,second=rb prologue_seeds=prologues(data,base) + frameless=frameless_leaf_entries(data,base) supplemental=supplemental_callable_seeds(data,base) - seeds=sorted(set(prologue_seeds)|supplemental) + seeds=sorted(set(prologue_seeds)|frameless|supplemental) page_base,region=page_aligned_region(base,data) direct=direct_jal_roots(data,base) record=rec(page_base,region,seeds,static_discovery=direct) @@ -897,6 +945,7 @@ def main(): f"region@0x{page_base:08X} " f"(strict jal-fit score={score}, runner-up={second}), " f"prologue seeds={len(prologue_seeds)} " + f"+{len(frameless-set(prologue_seeds))} bounded frameless leaves " f"+{len(supplemental-set(prologue_seeds))} pointer-table seeds") continue diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index dcddb9b0..8bf68016 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -579,9 +579,21 @@ def _callable_legacy_seed(data: bytes, load_addr: int, addr: int) -> bool: return _is_addiu_sp_neg(word) and not _is_control_flow(prev) +def _dense_local_pointer_table(data: bytes, load_addr: int, + addr: int, count: int = 3) -> bool: + """Reject instruction-shaped data at a supposed callable target.""" + hi = load_addr + len(data) + for index in range(count): + value = _word_at(data, load_addr, addr + index * 4) + if value is None or (value & 3) or not (load_addr <= value < hi): + return False + return True + + def _callable_direct_jal_target(data: bytes, load_addr: int, addr: int) -> bool: """True only when a direct JAL target has independent boundary evidence.""" - return _callable_legacy_seed(data, load_addr, addr) + return (not _dense_local_pointer_table(data, load_addr, addr) and + _callable_legacy_seed(data, load_addr, addr)) def _instruction_writes_gpr(word: int, reg: int) -> bool: @@ -642,8 +654,8 @@ def _resolve_constant_transfer(data: bytes, load_addr: int, return None -def _plausible_callable_target(data: bytes, load_addr: int, size: int, - target: int, producer_hi: int) -> bool: +def plausible_callable_target(data: bytes, load_addr: int, size: int, + target: int, producer_hi: int) -> bool: """CFG proof for a cross-producer call target without a classic prologue. Direct calls are sufficient boundary evidence inside one linked producer. @@ -655,6 +667,8 @@ def _plausible_callable_target(data: bytes, load_addr: int, size: int, first = _word_at(data, load_addr, target) if first in (None, 0) or not _is_valid_mips_word(first): return False + if _dense_local_pointer_table(data, load_addr, target): + return False cap = min(load_addr + size, producer_hi, target + 0x4000) work = deque([target]) visited = set() @@ -741,6 +755,9 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, return None def callable_transfer_target(source: int, target: int) -> bool: + if _dense_local_pointer_table(data, load_addr, target): + rejected_cross_producer_calls.add(target) + return False if not producer_ranges: return True source_range = producer_for(source) @@ -759,7 +776,7 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, accepted_cross_producer_calls.add(target) return True if (target_range is not None and - _plausible_callable_target( + plausible_callable_target( data, load_addr, size, target, target_range[1])): accepted_cross_producer_calls.add(target) return True @@ -1024,7 +1041,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, if not region(addr): excluded[addr] = 'UNKNOWN' elif (_callable_legacy_seed(data, load_addr, addr) or - _plausible_callable_target(data, load_addr, size, addr, hi)): + plausible_callable_target(data, load_addr, size, addr, hi)): include(addr, 'STATIC_DISCOVERY_ROOT') else: # Preserve the weaker classification. Overlay mode will perform @@ -1045,6 +1062,20 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, # roots — as roots they would hard-cap (truncate) the sibling walk that # owns them. known = {a for a, r in included.items() if r != 'DISPATCH_INTERIOR'} + # Establish range ownership before call-derived roots can change the + # partition. A direct-call target already visited by an existing root is an + # interior alias, not a sibling root: promoting it would hard-cap the host + # and can hide the remainder of a jump-table function. + initial_owned = set() + initial_known = set(known) + initial_sorted = sorted(initial_known) + for index, entry in enumerate(initial_sorted): + hard_cap = initial_sorted[index + 1] if index + 1 < len(initial_sorted) else hi + walk = _walk_overlay_function( + data, load_addr, size, entry, hard_cap, producer_ranges, + allow_cross_producer_calls) + initial_owned.update(walk['visited']) + pending = deque(sorted(known)) processed = set() all_branch_targets = set() @@ -1064,6 +1095,9 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, all_branch_targets.update(walk['branch_targets']) all_branch_targets.update(walk['jump_table_targets']) for target in sorted(walk['direct_jals']): + if target not in initial_known and target in initial_owned: + included[target] = 'DISPATCH_INTERIOR' + continue # Always re-include: a target may already be present as a # heuristic FUNCTION_POINTER_TARGET. Reachable JAL evidence # upgrades it to a trusted call root. @@ -1072,6 +1106,9 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, known.add(target) pending.append(target) for target in sorted(walk['static_indirect_targets']): + if target not in initial_known and target in initial_owned: + included[target] = 'DISPATCH_INTERIOR' + continue include(target, 'STATIC_INDIRECT_TARGET') if target in included and target not in known: known.add(target) @@ -1094,7 +1131,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, if producer_ranges and source_range != target_range: continue producer_hi = target_range[1] if target_range else hi - if not _plausible_callable_target( + if not plausible_callable_target( data, load_addr, size, target, producer_hi): continue include(target, 'STATIC_BRANCH_ROOT') From 4af125cf07bb312122b44df0f5cc39834e1ac555 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 11:08:16 -0700 Subject: [PATCH 25/37] tools: align TinyCC overlay build contract --- docs/AOT_OVERLAY_PLAN.md | 12 +++++++ .../tests/test_aot_overlay_discovery.py | 35 +++++++++++++++++++ tools/aot_overlay_spike/README.md | 15 ++++++++ tools/compile_overlays.py | 29 +++++++++++++-- 4 files changed, 88 insertions(+), 3 deletions(-) diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index eacb8549..4b229d55 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -70,6 +70,18 @@ continue to fail safely to the interpreter. Overlay ranges cover 820/823 historical-vault PCs and 410/450 live-history PCs; every residual is in the separately recompiled BIOS/kernel ranges, yielding **100.0% combined code-range recall and zero true gaps** for both scoreboards. +- The resulting `cg5_060368b2` checkpoint was re-extracted from disc and tested + against disposable caches on the other producers. Ape rebuilt 47/47 GCC shards + and retained 107/107 combined live-history range coverage. MMX6 rebuilt 31/31; + two unsafe broad-discovery attempts failed their audit/compile gates and fell + back to conservative seeds, retaining 677/677 combined range coverage. Tomba 2 + audited all 53 candidates with zero unsupported/bad targets and GCC-built eight + representative shards (MAIN, four large area variants, OPN/CRD, and BIOS); + its remaining full GCC compile is a low-priority disposable-cache follow-up. +- TCC overlay compilation now receives the same release/debug and cycle-model + defines as GCC. A TCC-only generated-source shim supplies `__builtin_ctz` + without changing the hashed runtime headers. Real T2 BIOS and OPN/CRD shards + compile cleanly through the packaged TinyCC 0.9.27 toolchain. --- diff --git a/recompiler/tests/test_aot_overlay_discovery.py b/recompiler/tests/test_aot_overlay_discovery.py index c0171c46..1dc2fe10 100644 --- a/recompiler/tests/test_aot_overlay_discovery.py +++ b/recompiler/tests/test_aot_overlay_discovery.py @@ -385,6 +385,40 @@ def check_atomic_dll_publication(): MOD._compile_dll_direct = original +def check_tcc_runtime_define_parity(): + """TCC shards must use the same release/cycle contract as GCC shards.""" + seen = [] + old_run = MOD.subprocess.run + old_inc = MOD._bom_free_incdir + + class Result: + returncode = 0 + stdout = '' + stderr = '' + + def fake_run(cmd, **_kwargs): + seen.extend(cmd) + return Result() + + try: + MOD.subprocess.run = fake_run + MOD._bom_free_incdir = lambda path: path + with tempfile.TemporaryDirectory() as td: + source = pathlib.Path(td) / 'overlay.c' + source.write_text('int overlay_test(void) { return 0; }\n') + assert MOD._compile_dll_tcc( + str(source), str(pathlib.Path(td) / 'overlay.dll'), + [td], 3, 'tcc') + assert b'psx_tcc_ctz' in source.read_bytes() + finally: + MOD.subprocess.run = old_run + MOD._bom_free_incdir = old_inc + + assert '-DPSX_NO_DEBUG_TOOLS' in seen + assert '-DPSX_ENABLE_BLOCK_CYCLES=1' in seen + assert '-DPSX_OVERLAY_FLAVOR=3' in seen + + def main(): default_recompiler = ROOT / "recompiler" / "build" / "psxrecomp-game.exe" parser = argparse.ArgumentParser() @@ -403,6 +437,7 @@ def main(): check_recompiler_pointer_table_call_root(args.recompiler) check_retained_alias_contract(args.recompiler) check_atomic_dll_publication() + check_tcc_runtime_define_parity() data = bytearray(0x200) diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 0f27e42a..91530d81 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -284,6 +284,21 @@ non-empty `.ranges` manifest. A killed compiler therefore cannot leave a partial final-named DLL that a later build mistakes for a valid shard, and a failed `--force` rebuild cannot destroy the previous good DLL. +The `cg5_060368b2` discovery checkpoint was then regenerated from each disc and +cross-title checked in disposable caches. Ape GCC-built **47/47** candidates and +retained **107/107 combined live-history code-range coverage**. MMX6 GCC-built +**31/31** candidates and retained **677/677 combined coverage**; broad normal-mode +attempts containing embedded data or an incomplete label graph were rejected and +the corresponding shards safely retried with direct seeds. Tomba 2's full set of +**53/53** candidates passed the generated-C audit with zero unsupported/bad +targets, and GCC built representative MAIN, large-area, OPN/CRD, and BIOS shards; +the remaining full GCC pass is intentionally running at low priority in a +disposable cache. That validation also exposed and fixed two TCC-path parity +bugs: TCC now receives the same `PSX_NO_DEBUG_TOOLS` and block-cycle defines as +GCC, and a generated-source-only shim supplies the unsupported `__builtin_ctz` +without changing hashed runtime headers. The packaged TinyCC 0.9.27 now builds +real T2 BIOS and OPN/CRD shards cleanly. + ## Next to raise coverage (future session) 1. ~~Direct-call frameless-leaf discovery~~ — DONE. Reachable `jal` targets are diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index 8bf68016..31ca8b76 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -2197,6 +2197,19 @@ def _toolchain_env(gcc: str): # BOM-stripped copy of the include dirs (used ONLY for tcc's -I; the codegen hash # still derives from the real headers, so the cache dir matches the runtime). _TCC_BOMFREE_INC = {} # orig include dir -> bom-stripped temp dir (memoized) +_TCC_COMPAT_PREFIX = b'''\ +#ifdef __TINYC__ +/* TinyCC 0.9.27 has no GCC __builtin_ctz. Keep this shim local to the + * disposable generated source so runtime headers and the codegen hash stay + * byte-identical across compilers. */ +static inline unsigned psx_tcc_ctz(unsigned value) { + unsigned bit = 0u; + while (((value >> bit) & 1u) == 0u) ++bit; + return bit; +} +#define __builtin_ctz(value) psx_tcc_ctz((unsigned)(value)) +#endif +''' def _bom_free_incdir(d: str) -> str: d = os.path.abspath(d) @@ -2217,14 +2230,24 @@ def _bom_free_incdir(d: str) -> str: def _compile_dll_tcc(c_path: str, out_dll: str, include_dirs, flavor: int, tcc: str) -> bool: - # strip a UTF-8 BOM off the overlay C itself (tcc 0.9.27 chokes on it) + # Strip a UTF-8 BOM off the overlay C itself (tcc 0.9.27 chokes on it) and + # provide the one GCC builtin used by the shared cycle-model header. with open(c_path, 'rb') as f: data = f.read() if data[:3] == b'\xef\xbb\xbf': - with open(c_path, 'wb') as f: - f.write(data[3:]) + data = data[3:] + if not data.startswith(_TCC_COMPAT_PREFIX): + data = _TCC_COMPAT_PREFIX + data + with open(c_path, 'wb') as f: + f.write(data) cmd = [tcc, '-shared', '-DPSX_OVERLAY_DLL_BUILD', + # Keep TCC semantically identical to the GCC overlay path. Omitting + # these left debug_server_cyc_observe unresolved in release shards + # and disabled guest-cycle charging in every successfully linked + # TCC shard. + '-DPSX_NO_DEBUG_TOOLS', + '-DPSX_ENABLE_BLOCK_CYCLES=1', f'-DPSX_OVERLAY_FLAVOR={int(flavor)}', native_path(c_path), '-o', native_path(out_dll)] for d in include_dirs: From 205438eb98c8d8755076b8b35dd328e402448610 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 11:15:13 -0700 Subject: [PATCH 26/37] tools: clean TinyCC staging sidecars --- docs/AOT_OVERLAY_PLAN.md | 11 ++++++----- recompiler/tests/test_aot_overlay_discovery.py | 4 ++++ tools/aot_overlay_spike/README.md | 9 +++++---- tools/compile_overlays.py | 13 +++++++++---- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index 4b229d55..3544f3e8 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -75,13 +75,14 @@ continue to fail safely to the interpreter. and retained 107/107 combined live-history range coverage. MMX6 rebuilt 31/31; two unsafe broad-discovery attempts failed their audit/compile gates and fell back to conservative seeds, retaining 677/677 combined range coverage. Tomba 2 - audited all 53 candidates with zero unsupported/bad targets and GCC-built eight - representative shards (MAIN, four large area variants, OPN/CRD, and BIOS); - its remaining full GCC compile is a low-priority disposable-cache follow-up. + audited all 53 candidates with zero unsupported/bad targets, TCC-built the full + 53/53 set, and GCC-built 11 representative shards including MAIN, large area + variants, OPN/CRD, and BIOS. - TCC overlay compilation now receives the same release/debug and cycle-model defines as GCC. A TCC-only generated-source shim supplies `__builtin_ctz` - without changing the hashed runtime headers. Real T2 BIOS and OPN/CRD shards - compile cleanly through the packaged TinyCC 0.9.27 toolchain. + without changing the hashed runtime headers. The packaged TinyCC 0.9.27 + toolchain compiles the complete T2 corpus, and atomic publication now removes + its staged `.def` sidecars after both success and failure. --- diff --git a/recompiler/tests/test_aot_overlay_discovery.py b/recompiler/tests/test_aot_overlay_discovery.py index 1dc2fe10..5b24989e 100644 --- a/recompiler/tests/test_aot_overlay_discovery.py +++ b/recompiler/tests/test_aot_overlay_discovery.py @@ -359,6 +359,8 @@ def check_atomic_dll_publication(): def fail_compile(_src, staged, _includes, **_kwargs): with open(staged, "wb") as out: out.write(b"partial") + with open(os.path.splitext(staged)[0] + ".def", "w") as out: + out.write("temporary exports") return False MOD._compile_dll_direct = fail_compile @@ -371,6 +373,8 @@ def check_atomic_dll_publication(): def good_compile(_src, staged, _includes, **_kwargs): with open(staged, "wb") as out: out.write(b"new-complete-shard") + with open(os.path.splitext(staged)[0] + ".def", "w") as out: + out.write("temporary exports") return True MOD._compile_dll_direct = good_compile diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 91530d81..98af74e9 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -292,12 +292,13 @@ attempts containing embedded data or an incomplete label graph were rejected and the corresponding shards safely retried with direct seeds. Tomba 2's full set of **53/53** candidates passed the generated-C audit with zero unsupported/bad targets, and GCC built representative MAIN, large-area, OPN/CRD, and BIOS shards; -the remaining full GCC pass is intentionally running at low priority in a -disposable cache. That validation also exposed and fixed two TCC-path parity +11 representative GCC shards were retained before the full-corpus pass switched +to the packaged compiler. That validation also exposed and fixed TCC-path parity bugs: TCC now receives the same `PSX_NO_DEBUG_TOOLS` and block-cycle defines as GCC, and a generated-source-only shim supplies the unsupported `__builtin_ctz` -without changing hashed runtime headers. The packaged TinyCC 0.9.27 now builds -real T2 BIOS and OPN/CRD shards cleanly. +without changing hashed runtime headers. TinyCC 0.9.27 then built the complete +T2 set **53/53**. Atomic publication also removes TCC's staged `.def` sidecars, +so successful and interrupted compiles leave only final DLL/manifest pairs. ## Next to raise coverage (future session) diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index 31ca8b76..c9a484a8 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -2353,10 +2353,15 @@ def compile_dll(c_path: str, out_dll: str, include_dirs: list[str], os.replace(staged, final_out) return True finally: - try: - os.unlink(staged) - except FileNotFoundError: - pass + # TinyCC emits an export-definition sidecar beside its requested DLL. + # It inherits the unique staged basename and must not accumulate in the + # content cache after either success or failure. + staged_sidecars = [os.path.splitext(staged)[0] + '.def'] + for temporary in [staged, *staged_sidecars]: + try: + os.unlink(temporary) + except FileNotFoundError: + pass def compiled_shard_complete(dll_path: str) -> bool: From aac755b0aecf9d3a858d2def8d8685da49b96bb8 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 13:14:57 -0700 Subject: [PATCH 27/37] tools: accelerate cumulative coverage scoring --- tools/aot_overlay_spike/README.md | 14 +- tools/aot_overlay_spike/coverage_report.py | 199 +++++++++++++-- .../aot_overlay_spike/test_coverage_report.py | 231 +++++++++++++++++- 3 files changed, 417 insertions(+), 27 deletions(-) diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 98af74e9..f1a2a7db 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -79,8 +79,18 @@ record lacks its exact immutable snapshot. For a live-recall scoreboard that survives individual launches, pass the same history to `coverage_report.py --addendum ...`. It unions only dispatch/function -entries from every verified snapshot, so the report no longer depends on which -latest capture happened to be present when it ran. +entries from verified snapshots, so the report no longer depends on which latest +capture happened to be present when it ran. Runtime snapshots are cumulative +within a session. For well-formed v2 records the reporter verifies the newest +snapshot, parses the older entry lists, and skips their expensive FNV passes only +after proving every older set is contained by the verified head. The report calls +these records out explicitly as superseded but not FNV-verified; only the verified +head contributes entries. Every candidate must still carry a syntactically valid +FNV signature, and unverified parsing has per-file and aggregate size bounds. +Non-cumulative +content and records with missing, duplicate, or nonmonotonic session metadata are +conservatively processed independently with FNV verification; invalid records are +rejected. For existing corpora that predate the addendum, `--captures` is repeatable; all named snapshots are unioned into one live-history denominator. diff --git a/tools/aot_overlay_spike/coverage_report.py b/tools/aot_overlay_spike/coverage_report.py index 13fb4e0a..6755dc71 100644 --- a/tools/aot_overlay_spike/coverage_report.py +++ b/tools/aot_overlay_spike/coverage_report.py @@ -30,6 +30,10 @@ import argparse, bisect, os, sys, json, re, glob, zlib from collections import defaultdict MASK = 0x1FFFFFFF +FNV64_OFFSET = 1469598103934665603 +FNV64_PRIME = 1099511628211 +MAX_UNVERIFIED_SNAPSHOT_BYTES = 32 * 1024 * 1024 +MAX_UNVERIFIED_TOTAL_BYTES = 2 * 1024 * 1024 * 1024 def norm(a): return a & MASK @@ -172,35 +176,85 @@ def parse_captures_executed(path): ent.add(norm(int(a, 16))) return ent +def _fnv64_bytes(data, value=FNV64_OFFSET): + for byte in data: + value ^= byte + value = (value * FNV64_PRIME) & 0xFFFFFFFFFFFFFFFF + return value + def _fnv64_file(path): - value = 1469598103934665603 + value = FNV64_OFFSET with open(path, 'rb') as source: while True: chunk = source.read(65536) if not chunk: break - for byte in chunk: - value ^= byte - value = (value * 1099511628211) & 0xFFFFFFFFFFFFFFFF + value = _fnv64_bytes(chunk, value) return value def parse_addendum_entries(path): - """Union dispatch/function entries from verified v1/v2 history records.""" + """Union dispatch/function entries from verified v1/v2 history records. + + Runtime v2 snapshots are cumulative within one session: the capture tables + only grow, and every history record references a full immutable snapshot. + For a well-formed session, verify the newest snapshot and cheaply prove that + every older entry set is a subset before skipping their expensive FNV pass. + Every candidate must still have a syntactically valid signature, and the + unverified reads are size-bounded. If content is non-cumulative, missing, or + unreadable, fall back to verifying and unioning every valid record as before. + + Records without trustworthy session/sequence metadata remain independent, + preserving the conservative behavior for hand-written and legacy history. + """ entries = set() seen_refs = set() audit = {'v1_records': 0, 'v2_records': 0, 'invalid_records': 0, - 'duplicate_refs': 0, 'verified_snapshots': 0} + 'duplicate_refs': 0, 'verified_snapshots': 0, + 'unverified_superseded_records': 0, + 'unverified_superseded_bytes': 0} + session_records = defaultdict(list) + independent_records = [] + all_v2_records = [] + verified_probe = {} + unverified_bytes_read = 0 - def add_captures(captures): + def capture_entries(captures): + found = set() if not isinstance(captures, list): - return + return found for cap in captures: if not isinstance(cap, dict): continue for key in ('dispatch_entry_pcs', 'function_entry_pcs'): for addr in cap.get(key, []): - entries.add(norm(int(addr, 16) if isinstance(addr, str) - else int(addr))) + found.add(norm(int(addr, 16) if isinstance(addr, str) + else int(addr))) + return found + + def add_captures(captures): + entries.update(capture_entries(captures)) + + def snapshot_fields(record): + snapshot = record.get('snapshot') + expected = str(record.get('fnv64', '')).upper() + if not isinstance(snapshot, str) or not snapshot: + return None, expected, None + if not os.path.isabs(snapshot): + snapshot = os.path.join(os.path.dirname(path), snapshot) + snapshot = os.path.abspath(snapshot) + return snapshot, expected, (os.path.normcase(snapshot), expected) + + def read_snapshot_bytes(snapshot, max_bytes=None): + with open(snapshot, 'rb') as source: + if max_bytes is not None and os.fstat(source.fileno()).st_size > max_bytes: + raise ValueError('snapshot exceeds unverified parse bound') + data = source.read() if max_bytes is None else source.read(max_bytes + 1) + if max_bytes is not None and len(data) > max_bytes: + raise ValueError('snapshot exceeds unverified parse bound') + return data + + def entries_from_bytes(data): + return capture_entries(json.loads(data)) with open(path, encoding='utf-8', errors='replace') as history: for lineno, line in enumerate(history, 1): @@ -224,30 +278,120 @@ def parse_addendum_entries(path): audit['invalid_records'] += 1 continue audit['v2_records'] += 1 - snapshot = record.get('snapshot') - expected = str(record.get('fnv64', '')).upper() - if not isinstance(snapshot, str) or not snapshot: + all_v2_records.append((lineno, record)) + game = record.get('game') + session = record.get('session') + sequence = record.get('sequence') + if (isinstance(game, str) and game and + isinstance(session, str) and session and + isinstance(sequence, int) and not isinstance(sequence, bool) and + sequence > 0): + session_records[(game, session)].append( + (sequence, lineno, record)) + else: + independent_records.append((lineno, record)) + + # Preserve the original duplicate-reference audit even when cumulative + # session records later avoid per-file verification. + referenced = set() + for _, record in all_v2_records: + _, _, ref = snapshot_fields(record) + if ref is not None: + if ref in referenced: + audit['duplicate_refs'] += 1 + else: + referenced.add(ref) + + # A valid runtime session has unique, monotonically increasing sequence + # numbers in append order. Anything else is treated independently rather + # than assuming the producer's cumulative-snapshot contract. + candidate_groups = [] + for records in session_records.values(): + append_order = [sequence for sequence, _, _ in records] + if (len(set(append_order)) != len(append_order) or + append_order != sorted(append_order)): + independent_records.extend((lineno, record) + for _, lineno, record in records) + else: + candidate_groups.append(list(reversed(records))) + candidate_groups.extend([[(0, lineno, record)] + for lineno, record in independent_records]) + + for candidates in candidate_groups: + # Metadata is not itself proof that a hand-written v2 producer is + # cumulative. Parse (but do not FNV-hash) the older entry lists and + # skip them only when the newest verified set demonstrably contains + # every one. Otherwise the original per-record verifier below runs. + collapse_metadata_valid = all( + re.fullmatch(r'[0-9A-F]{16}', snapshot_fields(record)[1]) + for _, _, record in candidates) + if len(candidates) > 1 and collapse_metadata_valid: + _, _, newest_record = candidates[0] + newest_path, newest_expected, newest_ref = snapshot_fields(newest_record) + if (newest_path is not None and newest_ref not in seen_refs and + os.path.isfile(newest_path) and newest_expected): + try: + # Hash and parse one byte buffer. Reopening between those + # operations would permit a replacement-file TOCTOU. + newest_bytes = read_snapshot_bytes(newest_path) + if newest_expected != '%016X' % _fnv64_bytes(newest_bytes): + raise ValueError('snapshot signature mismatch') + newest_entries = entries_from_bytes(newest_bytes) + verified_probe[newest_ref] = newest_entries + cumulative = True + unverified_limit = min( + MAX_UNVERIFIED_SNAPSHOT_BYTES, + max(1024 * 1024, len(newest_bytes) * 2)) + group_unverified_bytes = 0 + for _, _, older_record in candidates[1:]: + older_path, _, _ = snapshot_fields(older_record) + if older_path is None or not os.path.isfile(older_path): + cumulative = False + break + remaining = (MAX_UNVERIFIED_TOTAL_BYTES - + unverified_bytes_read) + if remaining <= 0: + cumulative = False + break + older_bytes = read_snapshot_bytes( + older_path, min(unverified_limit, remaining)) + unverified_bytes_read += len(older_bytes) + group_unverified_bytes += len(older_bytes) + if not entries_from_bytes(older_bytes).issubset(newest_entries): + cumulative = False + break + except Exception: + cumulative = False + if cumulative: + seen_refs.add(newest_ref) + entries.update(newest_entries) + audit['verified_snapshots'] += 1 + audit['unverified_superseded_records'] += len(candidates) - 1 + audit['unverified_superseded_bytes'] += group_unverified_bytes + continue + for _, lineno, record in candidates: + snapshot, expected, ref = snapshot_fields(record) + if snapshot is None: audit['invalid_records'] += 1 continue - if not os.path.isabs(snapshot): - snapshot = os.path.join(os.path.dirname(path), snapshot) - snapshot = os.path.abspath(snapshot) - ref = (os.path.normcase(snapshot), expected) if ref in seen_refs: - audit['duplicate_refs'] += 1 continue seen_refs.add(ref) - if (not os.path.isfile(snapshot) or not expected or - expected != '%016X' % _fnv64_file(snapshot)): + if not os.path.isfile(snapshot) or not expected: print(f' warn: ignoring missing/mismatched snapshot on line ' f'{lineno}: {snapshot}') audit['invalid_records'] += 1 continue try: - with open(snapshot, encoding='utf-8') as source: - add_captures(json.load(source)) + snapshot_entries = verified_probe.pop(ref, None) + if snapshot_entries is None: + snapshot_bytes = read_snapshot_bytes(snapshot) + if expected != '%016X' % _fnv64_bytes(snapshot_bytes): + raise ValueError('snapshot signature mismatch') + snapshot_entries = entries_from_bytes(snapshot_bytes) + entries.update(snapshot_entries) except Exception as exc: - print(f' warn: ignoring unreadable snapshot on line {lineno} ' + print(f' warn: ignoring unreadable/mismatched snapshot on line {lineno} ' f'({exc})') audit['invalid_records'] += 1 continue @@ -437,9 +581,16 @@ def main(): md.append(f'- Sources: **{" + ".join(live_sources)}**') if a.addendum and 'live_addendum_audit' in report: aa = report['live_addendum_audit'] - md.append(f'- Verified immutable snapshots: ' + md.append(f'- FNV-verified immutable snapshots: ' f'**{aa["verified_snapshots"]}**; invalid records: ' f'**{aa["invalid_records"]}**') + if aa.get('unverified_superseded_records'): + unverified_mib = (aa.get('unverified_superseded_bytes', 0) / + (1024 * 1024)) + md.append('- Superseded session snapshots **not FNV-verified** ' + '(their parsed entry sets were subsets of a verified ' + f'head): **{aa["unverified_superseded_records"]}** ' + f'({unverified_mib:.1f} MiB parsed under bounds)') md.append(f'- Dispatch entries exercised: **{r["needed"]}**') md.append(f'- Discovered by static: **{r["covered_here"]}** ' f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') diff --git a/tools/aot_overlay_spike/test_coverage_report.py b/tools/aot_overlay_spike/test_coverage_report.py index f6016235..484d0b3b 100644 --- a/tools/aot_overlay_spike/test_coverage_report.py +++ b/tools/aot_overlay_spike/test_coverage_report.py @@ -4,6 +4,7 @@ import json import tempfile import unittest import zlib +from unittest import mock import coverage_report @@ -86,7 +87,235 @@ const PsxNativeStub psx_bios_native_stubs[1] = { self.assertEqual(entries, {0x100, 0x200}) self.assertEqual(audit, { 'v1_records': 1, 'v2_records': 3, 'invalid_records': 2, - 'duplicate_refs': 1, 'verified_snapshots': 1}) + 'duplicate_refs': 1, 'verified_snapshots': 1, + 'unverified_superseded_records': 0, + 'unverified_superseded_bytes': 0}) + + def test_addendum_uses_newest_cumulative_snapshot_per_session(self): + with tempfile.TemporaryDirectory() as tmp: + snapshots = [] + captures = [ + [{'dispatch_entry_pcs': ['0x80000100']}], + [{'dispatch_entry_pcs': ['0x80000100', '0x80000200']}], + ] + for index, body in enumerate(captures, 1): + snapshot = os.path.join(tmp, f'immutable-{index}.json') + with open(snapshot, 'w', encoding='utf-8') as out: + json.dump(body, out) + snapshots.append(snapshot) + addendum = os.path.join(tmp, 'history.jsonl') + with open(addendum, 'w', encoding='utf-8', newline='\n') as out: + for sequence, snapshot in enumerate(snapshots, 1): + out.write(json.dumps({ + 'schema': 'psxrecomp overlay capture addendum v2', + 'game': 'TEST', 'session': 'session-1', + 'sequence': sequence, 'snapshot': snapshot, + 'fnv64': '%016X' % coverage_report._fnv64_file(snapshot), + }) + '\n') + entries, audit = coverage_report.parse_addendum_entries(addendum) + self.assertEqual(entries, {0x100, 0x200}) + self.assertEqual(audit['v2_records'], 2) + self.assertEqual(audit['verified_snapshots'], 1) + self.assertEqual(audit['unverified_superseded_records'], 1) + + def test_addendum_falls_back_when_newest_snapshot_is_invalid(self): + with tempfile.TemporaryDirectory() as tmp: + older = os.path.join(tmp, 'older.json') + newer = os.path.join(tmp, 'newer.json') + with open(older, 'w', encoding='utf-8') as out: + json.dump([{'function_entry_pcs': ['0x80000100']}], out) + with open(newer, 'w', encoding='utf-8') as out: + json.dump([{'function_entry_pcs': ['0x80000200']}], out) + addendum = os.path.join(tmp, 'history.jsonl') + records = [ + {'schema': 'psxrecomp overlay capture addendum v2', + 'game': 'TEST', 'session': 'session-1', 'sequence': 1, + 'snapshot': older, + 'fnv64': '%016X' % coverage_report._fnv64_file(older)}, + {'schema': 'psxrecomp overlay capture addendum v2', + 'game': 'TEST', 'session': 'session-1', 'sequence': 2, + 'snapshot': newer, 'fnv64': '0000000000000000'}, + ] + with open(addendum, 'w', encoding='utf-8', newline='\n') as out: + for record in records: + out.write(json.dumps(record) + '\n') + entries, audit = coverage_report.parse_addendum_entries(addendum) + self.assertEqual(entries, {0x100}) + self.assertEqual(audit['invalid_records'], 1) + self.assertEqual(audit['verified_snapshots'], 1) + self.assertEqual(audit['unverified_superseded_records'], 0) + + def test_addendum_does_not_collapse_nonmonotonic_session_records(self): + with tempfile.TemporaryDirectory() as tmp: + records = [] + for sequence, address in ((2, '0x80000200'), (1, '0x80000100')): + snapshot = os.path.join(tmp, f'immutable-{sequence}.json') + with open(snapshot, 'w', encoding='utf-8') as out: + json.dump([{'dispatch_entry_pcs': [address]}], out) + records.append({ + 'schema': 'psxrecomp overlay capture addendum v2', + 'game': 'TEST', 'session': 'session-1', + 'sequence': sequence, 'snapshot': snapshot, + 'fnv64': '%016X' % coverage_report._fnv64_file(snapshot), + }) + addendum = os.path.join(tmp, 'history.jsonl') + with open(addendum, 'w', encoding='utf-8', newline='\n') as out: + for record in records: + out.write(json.dumps(record) + '\n') + entries, audit = coverage_report.parse_addendum_entries(addendum) + self.assertEqual(entries, {0x100, 0x200}) + self.assertEqual(audit['verified_snapshots'], 2) + self.assertEqual(audit['unverified_superseded_records'], 0) + + def test_addendum_does_not_collapse_non_cumulative_session_content(self): + with tempfile.TemporaryDirectory() as tmp: + records = [] + for sequence, address in ((1, '0x80000100'), (2, '0x80000200')): + snapshot = os.path.join(tmp, f'immutable-{sequence}.json') + with open(snapshot, 'w', encoding='utf-8') as out: + json.dump([{'dispatch_entry_pcs': [address]}], out) + records.append({ + 'schema': 'psxrecomp overlay capture addendum v2', + 'game': 'TEST', 'session': 'session-1', + 'sequence': sequence, 'snapshot': snapshot, + 'fnv64': '%016X' % coverage_report._fnv64_file(snapshot), + }) + addendum = os.path.join(tmp, 'history.jsonl') + with open(addendum, 'w', encoding='utf-8', newline='\n') as out: + for record in records: + out.write(json.dumps(record) + '\n') + entries, audit = coverage_report.parse_addendum_entries(addendum) + self.assertEqual(entries, {0x100, 0x200}) + self.assertEqual(audit['verified_snapshots'], 2) + self.assertEqual(audit['unverified_superseded_records'], 0) + + def test_tampered_superseded_hash_cannot_change_score(self): + with tempfile.TemporaryDirectory() as tmp: + older = os.path.join(tmp, 'older.json') + newer = os.path.join(tmp, 'newer.json') + with open(older, 'w', encoding='utf-8') as out: + json.dump([{'dispatch_entry_pcs': ['0x80000100']}], out) + stale_signature = '%016X' % coverage_report._fnv64_file(older) + # Change the bytes while preserving an entry set contained by the + # verified head. The old file must not contribute to the score. + with open(older, 'a', encoding='utf-8') as out: + out.write(' ') + with open(newer, 'w', encoding='utf-8') as out: + json.dump([{'dispatch_entry_pcs': [ + '0x80000100', '0x80000200']}], out) + records = [ + {'schema': 'psxrecomp overlay capture addendum v2', + 'game': 'TEST', 'session': 'session-1', 'sequence': 1, + 'snapshot': older, 'fnv64': stale_signature}, + {'schema': 'psxrecomp overlay capture addendum v2', + 'game': 'TEST', 'session': 'session-1', 'sequence': 2, + 'snapshot': newer, + 'fnv64': '%016X' % coverage_report._fnv64_file(newer)}, + ] + addendum = os.path.join(tmp, 'history.jsonl') + with open(addendum, 'w', encoding='utf-8', newline='\n') as out: + for record in records: + out.write(json.dumps(record) + '\n') + entries, audit = coverage_report.parse_addendum_entries(addendum) + self.assertEqual(entries, {0x100, 0x200}) + self.assertEqual(audit['verified_snapshots'], 1) + self.assertEqual(audit['invalid_records'], 0) + self.assertEqual(audit['unverified_superseded_records'], 1) + + def test_bad_older_fnv_metadata_forces_invalid_record_audit(self): + for label, bad_fnv in (('missing', None), ('garbage', 'not-a-hash')): + with self.subTest(label=label), tempfile.TemporaryDirectory() as tmp: + older = os.path.join(tmp, 'older.json') + newer = os.path.join(tmp, 'newer.json') + with open(older, 'w', encoding='utf-8') as out: + json.dump([{'dispatch_entry_pcs': ['0x80000100']}], out) + with open(newer, 'w', encoding='utf-8') as out: + json.dump([{'dispatch_entry_pcs': [ + '0x80000100', '0x80000200']}], out) + older_record = { + 'schema': 'psxrecomp overlay capture addendum v2', + 'game': 'TEST', 'session': 'session-1', 'sequence': 1, + 'snapshot': older, + } + if bad_fnv is not None: + older_record['fnv64'] = bad_fnv + records = [ + older_record, + {'schema': 'psxrecomp overlay capture addendum v2', + 'game': 'TEST', 'session': 'session-1', 'sequence': 2, + 'snapshot': newer, + 'fnv64': '%016X' % coverage_report._fnv64_file(newer)}, + ] + addendum = os.path.join(tmp, 'history.jsonl') + with open(addendum, 'w', encoding='utf-8', newline='\n') as out: + for record in records: + out.write(json.dumps(record) + '\n') + entries, audit = coverage_report.parse_addendum_entries(addendum) + self.assertEqual(entries, {0x100, 0x200}) + self.assertEqual(audit['invalid_records'], 1) + self.assertEqual(audit['verified_snapshots'], 1) + self.assertEqual(audit['unverified_superseded_records'], 0) + + def test_oversized_unverified_snapshot_forces_full_verification(self): + with tempfile.TemporaryDirectory() as tmp: + older = os.path.join(tmp, 'older.json') + newer = os.path.join(tmp, 'newer.json') + with open(older, 'w', encoding='utf-8') as out: + json.dump([{'dispatch_entry_pcs': ['0x80000100']}], out) + out.write(' ' * 64) + with open(newer, 'w', encoding='utf-8') as out: + json.dump([{'dispatch_entry_pcs': [ + '0x80000100', '0x80000200']}], out) + records = [ + {'schema': 'psxrecomp overlay capture addendum v2', + 'game': 'TEST', 'session': 'session-1', 'sequence': 1, + 'snapshot': older, + 'fnv64': '%016X' % coverage_report._fnv64_file(older)}, + {'schema': 'psxrecomp overlay capture addendum v2', + 'game': 'TEST', 'session': 'session-1', 'sequence': 2, + 'snapshot': newer, + 'fnv64': '%016X' % coverage_report._fnv64_file(newer)}, + ] + addendum = os.path.join(tmp, 'history.jsonl') + with open(addendum, 'w', encoding='utf-8', newline='\n') as out: + for record in records: + out.write(json.dumps(record) + '\n') + with mock.patch.object( + coverage_report, 'MAX_UNVERIFIED_SNAPSHOT_BYTES', 32): + entries, audit = coverage_report.parse_addendum_entries(addendum) + self.assertEqual(entries, {0x100, 0x200}) + self.assertEqual(audit['verified_snapshots'], 2) + self.assertEqual(audit['unverified_superseded_records'], 0) + + def test_aggregate_unverified_size_bound_forces_full_verification(self): + with tempfile.TemporaryDirectory() as tmp: + snapshots = [] + bodies = [ + [{'dispatch_entry_pcs': ['0x80000100']}], + [{'dispatch_entry_pcs': ['0x80000100', '0x80000200']}], + [{'dispatch_entry_pcs': [ + '0x80000100', '0x80000200', '0x80000300']}], + ] + for sequence, body in enumerate(bodies, 1): + snapshot = os.path.join(tmp, f'immutable-{sequence}.json') + with open(snapshot, 'w', encoding='utf-8') as out: + json.dump(body, out) + snapshots.append(snapshot) + addendum = os.path.join(tmp, 'history.jsonl') + with open(addendum, 'w', encoding='utf-8', newline='\n') as out: + for sequence, snapshot in enumerate(snapshots, 1): + out.write(json.dumps({ + 'schema': 'psxrecomp overlay capture addendum v2', + 'game': 'TEST', 'session': 'session-1', + 'sequence': sequence, 'snapshot': snapshot, + 'fnv64': '%016X' % coverage_report._fnv64_file(snapshot), + }) + '\n') + with mock.patch.object( + coverage_report, 'MAX_UNVERIFIED_TOTAL_BYTES', 1): + entries, audit = coverage_report.parse_addendum_entries(addendum) + self.assertEqual(entries, {0x100, 0x200, 0x300}) + self.assertEqual(audit['verified_snapshots'], 3) + self.assertEqual(audit['unverified_superseded_records'], 0) if __name__ == '__main__': From 91c9aea91f2d866c9bff5ec640eeb55186125e7e Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 14:08:45 -0700 Subject: [PATCH 28/37] aot: make discovery and shard publication fail closed --- recompiler/include/function_analysis.h | 17 + recompiler/src/function_analysis.cpp | 259 ++++- recompiler/src/main_psx.cpp | 101 +- recompiler/tests/reachable_discovery_test.cpp | 306 ++++++ .../tests/test_aot_overlay_discovery.py | 739 +++++++++++++- runtime/src/overlay_loader.c | 145 +-- tools/compile_overlays.py | 899 +++++++++++++----- 7 files changed, 2068 insertions(+), 398 deletions(-) diff --git a/recompiler/include/function_analysis.h b/recompiler/include/function_analysis.h index d5a914db..a32e2df8 100644 --- a/recompiler/include/function_analysis.h +++ b/recompiler/include/function_analysis.h @@ -31,6 +31,14 @@ struct Function { std::vector alias_group_entries; }; +struct AbsorbedEntry { + uint32_t addr; + uint32_t host_start; + uint32_t host_end; + uint32_t source_addr; + bool resolved_indirect; +}; + struct FunctionAnalysisResult { std::vector functions; int total_instructions; @@ -41,6 +49,15 @@ struct FunctionAnalysisResult { int bios_thunk_count = 0; // Packed A0/B0/C0 BIOS dispatch thunks int state_continuation_count = 0; // Split entries after calls to SaveState-style helpers int pointer_table_entry_count = 0; // Function entries found from executable pointer tables + // Statically proven direct/constant-register transfer targets that the + // FINAL exact-entry partition reaches inside another function. These are + // safe overlapping aliases: the source edge and host reachability were + // both observed by the analyzer (not inferred from a raw byte envelope). + std::vector absorbed_entries; + // Instruction PCs reached by the FINAL exact-entry partition. Consumers + // use this to reject hostless `interior` seeds that merely fall inside a + // function's coarse [start,end) envelope or an unreachable data hole. + std::set exact_reachable_pcs; }; class FunctionAnalyzer { diff --git a/recompiler/src/function_analysis.cpp b/recompiler/src/function_analysis.cpp index b9011d21..59758edd 100644 --- a/recompiler/src/function_analysis.cpp +++ b/recompiler/src/function_analysis.cpp @@ -475,6 +475,8 @@ namespace { enum class ExactCfKind { Normal, Branch, + BranchNever, + BranchNeverLikely, Jump, Jal, JrRa, @@ -490,10 +492,14 @@ struct ExactCf { struct ExactWalkResult { std::set visited; std::set direct_jal_targets; + // target -> reachable (source PC, was resolved jalr/jr) evidence + std::map>> transfer_sources; std::set jump_table_targets; uint32_t jr_ra_count = 0; }; +static ExactCf exact_classify_cf(uint32_t pc, uint32_t instr); + static bool exact_is_jr_ra_word(uint32_t instr) { return instr == 0x03E00008u; } @@ -592,6 +598,44 @@ static bool exact_resolve_constant_transfer(const PS1Executable& exe, continue; } if (opcode == 0x0Fu && rt == target_reg) { + // A definition in a control-transfer delay slot is not a local + // reaching definition for the linear suffix. A call can clobber + // the register before returning at pc+4; a branch/jump can enter a + // different path entirely. Without path/interprocedural proof, + // reject every such candidate rather than combine raw words. + if (pc >= entry + 4u) { + auto predecessor = exe.read_word(pc - 4u); + if (predecessor.has_value()) { + uint32_t predecessor_op = (*predecessor >> 26) & 0x3Fu; + if (FunctionAnalyzer::is_branch_or_jump(*predecessor) || + (predecessor_op >= 0x14u && predecessor_op <= 0x17u)) { + return false; + } + } + } + + // A linear backward scan is only a reaching-definition proof when + // no direct control-flow edge can enter the suffix after this LUI. + // For example, `j low; lui rN,hi; low: addiu rN,rN,lo; jalr rN` + // skips the LUI at runtime. Without this boundary check the raw + // words still look like a constant pair and manufacture a target. + bool crossed_inbound_boundary = false; + for (uint32_t source = entry; source < transfer_pc; source += 4u) { + auto source_word = exe.read_word(source); + if (!source_word.has_value()) continue; + ExactCf source_cf = exact_classify_cf(source, *source_word); + bool has_direct_target = + source_cf.kind == ExactCfKind::Branch || + source_cf.kind == ExactCfKind::Jump || + source_cf.kind == ExactCfKind::Jal; + if (has_direct_target && source_cf.target > pc && + source_cf.target <= transfer_pc) { + crossed_inbound_boundary = true; + break; + } + } + if (crossed_inbound_boundary) return false; + uint32_t upper = (instr & 0xFFFFu) << 16; if (!have_low) { target_out = upper; @@ -633,12 +677,83 @@ static ExactCf exact_classify_cf(uint32_t pc, uint32_t instr) { cf.target = exact_jump_target(pc, instr); return cf; } + uint32_t rt = (instr >> 16) & 0x1Fu; + + // `beq rN,rN,target` is an unconditional branch. Treating its + // syntactic fallthrough as reachable lets raw data after the delay slot + // manufacture call/ownership evidence. + if ((opcode == 0x04u || opcode == 0x14u) && rs == rt) { + cf.kind = ExactCfKind::Jump; + cf.target = exact_branch_target(pc, instr); + return cf; + } + // Conversely, `bne rN,rN,target` can never take its encoded target. Keep + // its delay slot and PC+8 path, but never walk target-shaped data. + if (opcode == 0x05u && rs == rt) { + cf.kind = ExactCfKind::BranchNever; + return cf; + } + // Branch-likely annuls its delay slot when the condition is false. + if (opcode == 0x15u && rs == rt) { + cf.kind = ExactCfKind::BranchNeverLikely; + return cf; + } + // Conditions against the architectural zero register are also exact. + if (opcode == 0x06u && rs == 0u) { // blez $zero + cf.kind = ExactCfKind::Jump; + cf.target = exact_branch_target(pc, instr); + return cf; + } + if (opcode == 0x07u && rs == 0u) { // bgtz $zero + cf.kind = ExactCfKind::BranchNever; + return cf; + } + if (opcode == 0x16u && rs == 0u) { // blezl $zero + cf.kind = ExactCfKind::Jump; + cf.target = exact_branch_target(pc, instr); + return cf; + } + if (opcode == 0x17u && rs == 0u) { // bgtzl $zero + cf.kind = ExactCfKind::BranchNeverLikely; + return cf; + } + if (opcode == 0x01u && rs == 0u) { + // BLTZ/BLTZAL (and likely forms) are false for zero; BGEZ forms are + // true. The link-true forms are BAL/BALL aliases, so model their + // target and return continuation like a call rather than a tail jump. + if (rt == 0x00u || rt == 0x10u) { + cf.kind = ExactCfKind::BranchNever; + return cf; + } + if (rt == 0x02u || rt == 0x12u) { + cf.kind = ExactCfKind::BranchNeverLikely; + return cf; + } + if (rt == 0x01u || rt == 0x03u) { + cf.kind = ExactCfKind::Jump; + cf.target = exact_branch_target(pc, instr); + return cf; + } + if (rt == 0x11u || rt == 0x13u) { // bal / ball aliases + cf.kind = ExactCfKind::Jal; + cf.target = exact_branch_target(pc, instr); + return cf; + } + } if (opcode == 0x01u || (opcode >= 0x04u && opcode <= 0x07u) || (opcode >= 0x14u && opcode <= 0x17u)) { cf.kind = ExactCfKind::Branch; cf.target = exact_branch_target(pc, instr); return cf; } + // COP1/COP2 condition branches are ordinary conditional branches. COP1 + // is not present on PS1, but classifying both keeps raw control-flow and + // delay-slot boundaries conservative for composite inputs. + if ((opcode == 0x11u || opcode == 0x12u) && rs == 0x08u) { + cf.kind = ExactCfKind::Branch; + cf.target = exact_branch_target(pc, instr); + return cf; + } return cf; } @@ -697,12 +812,16 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( if (producer_ranges.empty()) return true; int source_producer = producer_for(source); int target_producer = producer_for(addr); - if (source_producer >= 0 && source_producer == target_producer) + // A composite's uncovered padding is not a producer. In particular, + // two gap addresses both map to -1 and must not be treated as sharing + // ownership, nor may an allow-list bless a target with no producer. + if (source_producer < 0 || target_producer < 0) return false; + if (source_producer == target_producer) return true; return cross_call_allow.count(addr) != 0; }; - auto find_jump_table_targets = [&](uint32_t entry, uint32_t hard_cap, + [[maybe_unused]] auto find_jump_table_targets = [&](uint32_t entry, uint32_t hard_cap, uint32_t jr_pc, uint32_t jr_rs) { std::vector targets; uint32_t lw_base = 0xFFu; @@ -789,9 +908,13 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( ExactWalkResult wr; std::queue work; work.push(entry); + const int entry_producer = producer_for(entry); auto in_function = [&](uint32_t addr) { - return in_exe(addr) && addr >= entry && addr < hard_cap; + return in_exe(addr) && addr >= entry && addr < hard_cap && + (producer_ranges.empty() || + (entry_producer >= 0 && + producer_for(addr) == entry_producer)); }; while (!work.empty()) { @@ -816,6 +939,13 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( if (in_function(pc + 8u)) work.push(pc + 8u); if (in_function(cf.target)) work.push(cf.target); break; + case ExactCfKind::BranchNever: + if (in_function(delay)) wr.visited.insert(delay); + if (in_function(pc + 8u)) work.push(pc + 8u); + break; + case ExactCfKind::BranchNeverLikely: + if (in_function(pc + 8u)) work.push(pc + 8u); + break; case ExactCfKind::Jump: if (in_function(delay)) wr.visited.insert(delay); if (in_function(cf.target)) work.push(cf.target); @@ -826,6 +956,8 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( if (in_exe(cf.target) && callable_direct_jal_target(pc, cf.target)) { wr.direct_jal_targets.insert(cf.target); + wr.transfer_sources[cf.target].insert( + std::make_pair(pc, false)); } break; case ExactCfKind::Jalr: @@ -839,6 +971,8 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( in_exe(target) && callable_direct_jal_target(pc, target)) { wr.direct_jal_targets.insert(target); + wr.transfer_sources[target].insert( + std::make_pair(pc, true)); } } break; @@ -861,13 +995,14 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( // An absolute JR out of the current function is a // statically proven tail-call entry. wr.direct_jal_targets.insert(target); + wr.transfer_sources[target].insert( + std::make_pair(pc, true)); } } else { - for (uint32_t jt : find_jump_table_targets(entry, hard_cap, - pc, jr_rs)) { - wr.jump_table_targets.insert(jt); - if (in_function(jt)) work.push(jt); - } + // Fail closed: the former 40-word raw backscan could + // combine unrelated loads/constants into a synthetic + // jump table. Unresolved JR cases stay interpreted until + // register/control dependence is proven. } } break; @@ -879,10 +1014,14 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( std::set known_entries; for (uint32_t entry : entries) { if (!in_exe(entry)) continue; + if (!producer_ranges.empty() && producer_for(entry) < 0) continue; known_entries.insert(entry); } size_t explicit_count = known_entries.size(); + const std::set explicit_entries = known_entries; + std::set derived_entries; + std::map>> derived_evidence; fmt::print("Explicit entries: {}\n", explicit_count); // Discover callees in rounds against one stable entry partition. Processing @@ -890,30 +1029,72 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( // lay inside a later explicit root. That new entry hard-capped the later // root before it was walked (Tomba X01's jump-table host), splitting one // function and losing every case after the accidental cap. If a candidate - // is already owned by any walk in the current partition, leave it absorbed; - // main_psx's range-ownership pass materializes it as an interior alias. + // is already owned by any walk in the final partition, leave it absorbed; + // the analyzer exports that proven host mapping for main_psx to materialize. + uint32_t discovery_round = 0; + std::set, std::set>> seen_states; while (true) { + auto state = std::make_pair(known_entries, derived_entries); + if (++discovery_round > 64u || !seen_states.insert(state).second) { + fmt::print("WARNING: exact-entry ownership did not converge; " + "falling back to explicit roots only\n"); + known_entries = explicit_entries; + derived_entries.clear(); + derived_evidence.clear(); + break; + } + const std::set previous_entries = known_entries; + const std::set previous_derived = derived_entries; std::vector round_entries( known_entries.begin(), known_entries.end()); - std::set owned_addresses; std::set candidate_targets; + std::map>> candidate_evidence; for (size_t i = 0; i < round_entries.size(); i++) { uint32_t hard_cap = (i + 1 < round_entries.size()) ? round_entries[i + 1] : exe_.end_address(); ExactWalkResult wr = walk(round_entries[i], hard_cap); - owned_addresses.insert(wr.visited.begin(), wr.visited.end()); candidate_targets.insert(wr.direct_jal_targets.begin(), wr.direct_jal_targets.end()); + for (const auto& [target, evidence] : wr.transfer_sources) + candidate_evidence[target].insert( + evidence.begin(), evidence.end()); } - bool added = false; - for (uint32_t target : candidate_targets) { - if (known_entries.count(target) || owned_addresses.count(target)) + derived_entries.clear(); + derived_evidence.clear(); + for (uint32_t target : candidate_targets) + if (!explicit_entries.count(target)) { + derived_entries.insert(target); + auto evidence = candidate_evidence.find(target); + if (evidence != candidate_evidence.end()) + derived_evidence.emplace(target, evidence->second); + } + + // Rebuild ownership FROM SCRATCH whenever the candidate universe grows. + // A target B absorbed by A in one round may need to become a root after + // a later target X lands between A and B and caps A. A monotonic + // "absorbed forever" set therefore loses reachable code. + known_entries = explicit_entries; + known_entries.insert(derived_entries.begin(), derived_entries.end()); + for (uint32_t target : derived_entries) { + auto target_it = known_entries.find(target); + if (target_it == known_entries.end() || + explicit_entries.count(target) || + target_it == known_entries.begin()) { continue; - known_entries.insert(target); - added = true; + } + auto owner_it = std::prev(target_it); + auto next_it = std::next(target_it); + uint32_t hard_cap = next_it != known_entries.end() + ? *next_it : exe_.end_address(); + ExactWalkResult owner_walk = walk(*owner_it, hard_cap); + if (owner_walk.visited.count(target)) { + known_entries.erase(target_it); + } } - if (!added) break; + + if (known_entries == previous_entries && + derived_entries == previous_derived) break; } result.call_discovered_count = static_cast(known_entries.size() - explicit_count); @@ -953,9 +1134,44 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( result.jr_ra_count += static_cast(wr.jr_ra_count); result.total_instructions += static_cast(wr.visited.size()); + result.exact_reachable_pcs.insert(wr.visited.begin(), wr.visited.end()); result.functions.push_back(func); } + // Export only analyzer-proven aliases with a host in the FINAL partition. + // Re-walking final roots makes stale ownership impossible: if a later root + // now caps the old host before B, B is either a real root or absent here. + std::set final_starts(known_entries.begin(), known_entries.end()); + std::vector> final_walks; + for (size_t i = 0; i < starts_vec.size(); i++) { + uint32_t host_start = starts_vec[i]; + uint32_t hard_cap = (i + 1 < starts_vec.size()) + ? starts_vec[i + 1] : exe_.end_address(); + ExactWalkResult wr = walk(host_start, hard_cap); + if (wr.visited.empty()) continue; + uint32_t host_end = *wr.visited.rbegin() + 4u; + Function host{}; + host.start_addr = host_start; + host.end_addr = host_end; + final_walks.push_back({host, std::move(wr)}); + } + for (uint32_t target : derived_entries) { + if (final_starts.count(target)) continue; + for (const auto& [host, wr] : final_walks) { + if (!wr.visited.count(target)) continue; + auto evidence = derived_evidence.find(target); + if (evidence == derived_evidence.end()) break; + auto source = std::find_if( + evidence->second.begin(), evidence->second.end(), + [&](const auto& item) { return wr.visited.count(item.first); }); + if (source == evidence->second.end()) break; + result.absorbed_entries.push_back( + {target, host.start_addr, host.end_addr, + source->first, source->second}); + break; + } + } + return result; } @@ -1437,6 +1653,13 @@ FunctionAnalysisResult FunctionAnalyzer::analyze() { if (in_fn(pc + 8u)) work.push(pc + 8u); if (in_fn(cf.target)) work.push(cf.target); break; + case ExactCfKind::BranchNever: + if (in_fn(delay)) visited.insert(delay); + if (in_fn(pc + 8u)) work.push(pc + 8u); + break; + case ExactCfKind::BranchNeverLikely: + if (in_fn(pc + 8u)) work.push(pc + 8u); + break; case ExactCfKind::Jump: if (in_fn(delay)) visited.insert(delay); if (in_fn(cf.target)) work.push(cf.target); diff --git a/recompiler/src/main_psx.cpp b/recompiler/src/main_psx.cpp index ac8aa3f5..f2dc0fb3 100644 --- a/recompiler/src/main_psx.cpp +++ b/recompiler/src/main_psx.cpp @@ -430,7 +430,6 @@ int main(int argc, char** argv) { * valid seeds. */ std::vector file_seeds; std::vector interior_seeds; - std::vector retained_alias_seeds; std::set trusted_root_seeds; std::set trusted_call_root_seeds; std::vector> producer_ranges; @@ -471,21 +470,8 @@ int main(int argc, char** argv) { continue; } if (line.rfind("retained_alias", 0) == 0) { - std::istringstream in(line); - std::string tag, addr_text, lo_text, hi_text; - in >> tag >> addr_text >> lo_text >> hi_text; - uint32_t addr = static_cast( - std::strtoul(addr_text.c_str(), nullptr, 16)); - uint32_t lo = static_cast( - std::strtoul(lo_text.c_str(), nullptr, 16)); - uint32_t hi = static_cast( - std::strtoul(hi_text.c_str(), nullptr, 16)); - if (addr_text.empty() || lo_text.empty() || hi_text.empty() || - lo < seed_lo || addr < lo || addr >= hi || hi > seed_hi) { - fmt::print(stderr, "ERROR: invalid retained_alias: {}\n", line); - return 1; - } - retained_alias_seeds.push_back({addr, lo, hi}); + fmt::print(" ignoring legacy retained_alias; old host " + "ranges are never reused: {}\n", line); continue; } bool interior = false; @@ -538,12 +524,11 @@ int main(int argc, char** argv) { } fmt::print("Loaded {} extra function addresses ({} interior, " "{} dispatch-root, {} call-root, {} producer ranges, " - "{} cross-call allows, {} retained aliases) from {}\n", + "{} cross-call allows) from {}\n", file_seeds.size() + interior_seeds.size(), interior_seeds.size(), trusted_root_seeds.size(), trusted_call_root_seeds.size(), producer_ranges.size(), cross_call_allow.size(), - retained_alias_seeds.size(), extra_funcs_path); } else { fmt::print("WARNING: Cannot open extra-funcs file: {}\n", extra_funcs_path); @@ -653,7 +638,7 @@ int main(int argc, char** argv) { for (uint32_t a : interior) { const PSXRecomp::Function* host = containing(a); if (host && a == host->start_addr) continue; // became a real entry - if (host) { + if (host && analysis_result.exact_reachable_pcs.count(a)) { if (alias_seen.insert(a).second) alias_entries.push_back({a, host->start_addr, host->end_addr}); } else { @@ -662,73 +647,19 @@ int main(int argc, char** argv) { } } - // Range-ownership completeness (B): a DIRECT CALL (jal) whose target - // lands strictly INSIDE an already-discovered function is a separate - // routine the host's discovered range absorbed (e.g. via a break/trap - // fall-through in discovery — the crt0 `jal main; break` that swallowed - // the next function, the Tomba 2 FMV-freeze root cause). Register every - // such target as an ALIAS entry so the host's entry-switch routes a - // dispatch of that PC to the correct interior block (running the absorbed - // routine natively) instead of failing closed to the interpreter. This is - // derived purely from static jal evidence in the discovered code, so it - // does not depend on the PC having been captured/executed. jal encodes a - // code target, so this never mints code from data. - size_t jal_alias_added = 0; - { - std::set known_starts; - for (const auto& f : analysis_result.functions) - known_starts.insert(f.start_addr); - std::set jal_targets; - for (const auto& f : analysis_result.functions) { - for (uint32_t a = f.start_addr; a + 4 <= f.end_addr; a += 4) { - uint32_t w = read_w(a); - if ((w >> 26) == 0x03u) { // jal - uint32_t tgt = (a & 0xF0000000u) | ((w & 0x03FFFFFFu) << 2); - jal_targets.insert(tgt); - } - } - } - for (uint32_t t : jal_targets) { - if (known_starts.count(t)) continue; // already a real entry - if (!alias_seen.count(t)) { - const PSXRecomp::Function* host = containing(t); - if (host && t != host->start_addr) { - alias_seen.insert(t); - alias_entries.push_back({t, host->start_addr, host->end_addr}); - jal_alias_added++; - } - } - } - } - if (jal_alias_added) - fmt::print(" +{} in-function jal-target alias entries " - "(range-ownership completeness)\n", jal_alias_added); - - // An existing manifest for these exact image bytes is durable proof - // that its overlapping alias groups compiled successfully before. - // Preserve those groups without feeding their entries back as walk - // roots: roots would hard-cap newly discovered siblings, while a - // retained overlapping body lets old indirect entries and new static - // call roots coexist. Ordinary current functions win at an identical - // entry; retained aliases replace only an ordinary alias at that PC. - if (!retained_alias_seeds.empty()) { - std::set current_starts; - for (const auto& f : analysis_result.functions) - current_starts.insert(f.start_addr); - size_t retained_added = 0; - for (const auto& retained : retained_alias_seeds) { - if (current_starts.count(retained.addr)) continue; - alias_entries.erase( - std::remove_if(alias_entries.begin(), alias_entries.end(), - [&](const AliasEntry& ae) { return ae.addr == retained.addr; }), - alias_entries.end()); - alias_entries.push_back(retained); - retained_added++; - } - if (retained_added) - fmt::print(" +{} retained prior-manifest alias entries " - "(monotonic identical-image coverage)\n", retained_added); + // Range-ownership completeness uses only analyzer-proven transfers from + // reachable instructions. The analyzer also revalidates each target's + // host against the final root partition and producer boundaries. + size_t proven_alias_added = 0; + for (const auto& absorbed : analysis_result.absorbed_entries) { + if (!alias_seen.insert(absorbed.addr).second) continue; + alias_entries.push_back( + {absorbed.addr, absorbed.host_start, absorbed.host_end}); + proven_alias_added++; } + if (proven_alias_added) + fmt::print(" +{} analyzer-proven absorbed alias entries " + "(final range ownership)\n", proven_alias_added); materialize_alias_groups(analysis_result, alias_entries); fmt::print("Exact-entry alias entries emitted: {}\n\n", alias_entries.size()); diff --git a/recompiler/tests/reachable_discovery_test.cpp b/recompiler/tests/reachable_discovery_test.cpp index cebdff98..805fd93b 100644 --- a/recompiler/tests/reachable_discovery_test.cpp +++ b/recompiler/tests/reachable_discovery_test.cpp @@ -151,6 +151,269 @@ int main() { std::set({kLoad, kLoad + 0x100}), "direct call into an owned function stays an interior alias"); + // Ownership must also settle among roots discovered in the same round. + // The initial root calls both A and an address B inside A. Neither address + // is owned by the initial partition, but walking A without B as a cap + // reaches B, so B is an alias rather than a sibling function. + auto same_round_image = make_exe_buffer(0x1000); + put32(same_round_image, text + 0x00, jal(kLoad + 0x100)); + put32(same_round_image, text + 0x04, 0x00000000u); + put32(same_round_image, text + 0x08, jal(kLoad + 0x110)); + put32(same_round_image, text + 0x0C, 0x00000000u); + put32(same_round_image, text + 0x10, 0x03E00008u); + put32(same_round_image, text + 0x14, 0x00000000u); + put32(same_round_image, text + 0x100, jal(kLoad + 0x110)); + put32(same_round_image, text + 0x104, 0x00000000u); + put32(same_round_image, text + 0x108, 0x24420001u); + put32(same_round_image, text + 0x10C, 0x24420001u); + put32(same_round_image, text + 0x110, 0x03E00008u); + put32(same_round_image, text + 0x114, 0x00000000u); + auto same_round_exe = parse(same_round_image); + PSXRecomp::FunctionAnalyzer same_round_analyzer(same_round_exe); + const auto same_round_result = + same_round_analyzer.analyze_exact_entries({kLoad}); + const auto same_round_starts = starts(same_round_result); + CHECK(same_round_starts == + std::set({kLoad, kLoad + 0x100}), + "same-round call target inside a discovered host stays absorbed"); + bool same_round_hosted = false; + for (const auto& absorbed : same_round_result.absorbed_entries) { + same_round_hosted |= (absorbed.addr == kLoad + 0x110 && + absorbed.host_start == kLoad + 0x100 && + absorbed.addr < absorbed.host_end && + absorbed.source_addr == kLoad + 0x100 && + !absorbed.resolved_indirect); + } + CHECK(same_round_hosted, + "every analyzer-proven absorbed target names its final CFG host"); + + // Resolved constant-register calls carry the same reachable provenance as + // direct JALs and can therefore become final hosted aliases too. + auto indirect_alias_image = make_exe_buffer(0x1000); + put32(indirect_alias_image, text + 0x00, jal(kLoad + 0x100)); + put32(indirect_alias_image, text + 0x04, 0x00000000u); + put32(indirect_alias_image, text + 0x08, 0x03E00008u); + put32(indirect_alias_image, text + 0x0C, 0x00000000u); + put32(indirect_alias_image, text + 0x100, 0x3C088001u); + put32(indirect_alias_image, text + 0x104, 0x25080110u); + put32(indirect_alias_image, text + 0x108, 0x0100F809u); + put32(indirect_alias_image, text + 0x10C, 0x00000000u); + put32(indirect_alias_image, text + 0x110, 0x03E00008u); + put32(indirect_alias_image, text + 0x114, 0x00000000u); + auto indirect_alias_exe = parse(indirect_alias_image); + PSXRecomp::FunctionAnalyzer indirect_alias_analyzer(indirect_alias_exe); + const auto indirect_alias_result = + indirect_alias_analyzer.analyze_exact_entries({kLoad}); + bool indirect_alias_hosted = false; + for (const auto& absorbed : indirect_alias_result.absorbed_entries) { + indirect_alias_hosted |= (absorbed.addr == kLoad + 0x110 && + absorbed.host_start == kLoad + 0x100 && + absorbed.source_addr == kLoad + 0x108 && + absorbed.resolved_indirect); + } + CHECK(indirect_alias_hosted, + "resolved indirect call target exports final alias provenance"); + + // The ordering can span rounds too: B is found first, then B discovers an + // earlier A whose reachable body contains B. Once A exists, ownership must + // be recomputed without retaining B as a stale hard cap. + auto later_round_image = make_exe_buffer(0x1000); + put32(later_round_image, text + 0x00, jal(kLoad + 0x120)); + put32(later_round_image, text + 0x04, 0x00000000u); + put32(later_round_image, text + 0x08, 0x03E00008u); + put32(later_round_image, text + 0x0C, 0x00000000u); + put32(later_round_image, text + 0x100, 0x10000007u); // b 0x80010120 + put32(later_round_image, text + 0x104, 0x00000000u); + put32(later_round_image, text + 0x120, jal(kLoad + 0x100)); + put32(later_round_image, text + 0x124, 0x00000000u); + put32(later_round_image, text + 0x128, 0x03E00008u); + put32(later_round_image, text + 0x12C, 0x00000000u); + auto later_round_exe = parse(later_round_image); + PSXRecomp::FunctionAnalyzer later_round_analyzer(later_round_exe); + const auto later_round_starts = starts( + later_round_analyzer.analyze_exact_entries({kLoad})); + CHECK(later_round_starts == + std::set({kLoad, kLoad + 0x100}), + "later-discovered host retracts its stale interior hard cap"); + + // R discovers A and B; A initially owns B. Walking that expanded A then + // discovers X between them. X caps A before B and does not itself reach B, + // so final ownership must reconsider B and restore it as a real root. + auto invalidated_owner_image = make_exe_buffer(0x1000); + put32(invalidated_owner_image, text + 0x00, jal(kLoad + 0x100)); + put32(invalidated_owner_image, text + 0x04, 0x00000000u); + put32(invalidated_owner_image, text + 0x08, jal(kLoad + 0x140)); + put32(invalidated_owner_image, text + 0x0C, 0x00000000u); + put32(invalidated_owner_image, text + 0x10, 0x03E00008u); + put32(invalidated_owner_image, text + 0x14, 0x00000000u); + put32(invalidated_owner_image, text + 0x100, + 0x08000000u | (((kLoad + 0x130) >> 2) & 0x03FFFFFFu)); + put32(invalidated_owner_image, text + 0x104, 0x00000000u); + put32(invalidated_owner_image, text + 0x120, 0x03E00008u); // X + put32(invalidated_owner_image, text + 0x124, 0x00000000u); + put32(invalidated_owner_image, text + 0x130, 0x24420001u); + put32(invalidated_owner_image, text + 0x134, 0x24420001u); + put32(invalidated_owner_image, text + 0x138, 0x24420001u); + put32(invalidated_owner_image, text + 0x13C, 0x24420001u); + put32(invalidated_owner_image, text + 0x140, jal(kLoad + 0x120)); // B -> X + put32(invalidated_owner_image, text + 0x144, 0x00000000u); + put32(invalidated_owner_image, text + 0x148, 0x03E00008u); + put32(invalidated_owner_image, text + 0x14C, 0x00000000u); + auto invalidated_owner_exe = parse(invalidated_owner_image); + PSXRecomp::FunctionAnalyzer invalidated_owner_analyzer( + invalidated_owner_exe); + const auto invalidated_owner_result = + invalidated_owner_analyzer.analyze_exact_entries({kLoad}); + const auto invalidated_owner_starts = starts(invalidated_owner_result); + CHECK(invalidated_owner_starts.count(kLoad + 0x100) && + invalidated_owner_starts.count(kLoad + 0x120) && + invalidated_owner_starts.count(kLoad + 0x140), + "R/A/X/B ownership invalidation restores B as a final root"); + bool stale_b_alias = false; + for (const auto& absorbed : invalidated_owner_result.absorbed_entries) + stale_b_alias |= absorbed.addr == kLoad + 0x140; + CHECK(!stale_b_alias, + "R/A/X/B final ownership does not retain stale B alias"); + + // Producer boundaries are hard CFG ownership boundaries. A call into the + // next producer is rejected without approval and, when approved, becomes + // its own root rather than being absorbed by fallthrough from producer A. + auto producer_owner_image = make_exe_buffer(0x1000); + put32(producer_owner_image, text + 0x00, jal(kLoad + 0x100)); + put32(producer_owner_image, text + 0x04, 0x00000000u); + put32(producer_owner_image, text + 0x08, jal(kLoad + 0x108)); + put32(producer_owner_image, text + 0x0C, 0x00000000u); + put32(producer_owner_image, text + 0x10, 0x03E00008u); + put32(producer_owner_image, text + 0x14, 0x00000000u); + put32(producer_owner_image, text + 0x100, 0x24420001u); + put32(producer_owner_image, text + 0x104, 0x24420001u); + put32(producer_owner_image, text + 0x108, 0x03E00008u); + put32(producer_owner_image, text + 0x10C, 0x00000000u); + auto producer_owner_exe = parse(producer_owner_image); + const std::vector> ownership_ranges = { + {kLoad, kLoad + 0x108}, {kLoad + 0x108, kLoad + 0x1000}}; + PSXRecomp::FunctionAnalyzer rejected_owner_analyzer(producer_owner_exe); + const auto rejected_owner = rejected_owner_analyzer.analyze_exact_entries( + {kLoad}, ownership_ranges, {}); + CHECK(!starts(rejected_owner).count(kLoad + 0x108) && + rejected_owner.absorbed_entries.empty(), + "rejected cross-producer call is neither root nor alias"); + PSXRecomp::FunctionAnalyzer allowed_owner_analyzer(producer_owner_exe); + const auto allowed_owner = allowed_owner_analyzer.analyze_exact_entries( + {kLoad}, ownership_ranges, {kLoad + 0x108}); + CHECK(starts(allowed_owner).count(kLoad + 0x108), + "approved cross-producer call remains a separate root"); + bool cross_absorbed = false; + for (const auto& absorbed : allowed_owner.absorbed_entries) + cross_absorbed |= absorbed.addr == kLoad + 0x108; + CHECK(!cross_absorbed, + "approved cross-producer target is never absorbed across boundary"); + + // An uncovered composite gap has no producer identity. Two gap addresses + // both returning producer -1 must not become a synthetic shared producer. + auto producer_gap_image = make_exe_buffer(0x1000); + put32(producer_gap_image, text + 0x00, 0x03E00008u); + put32(producer_gap_image, text + 0x04, 0x00000000u); + put32(producer_gap_image, text + 0x60, 0x24020001u); + put32(producer_gap_image, text + 0x64, 0x03E00008u); + put32(producer_gap_image, text + 0x68, 0x00000000u); + const auto producer_gap_exe = parse(producer_gap_image); + PSXRecomp::FunctionAnalyzer producer_gap_analyzer(producer_gap_exe); + const std::vector> gap_ranges = { + {kLoad, kLoad + 0x40}, {kLoad + 0x80, kLoad + 0x100}}; + const auto producer_gap_result = producer_gap_analyzer.analyze_exact_entries( + {kLoad, kLoad + 0x60}, gap_ranges, {}); + CHECK(starts(producer_gap_result) == std::set({kLoad}) && + !producer_gap_result.exact_reachable_pcs.count(kLoad + 0x60), + "explicit roots in uncovered producer gaps fail closed"); + + // R reaches S, S calls B, and B calls earlier X. X then caps R before S, + // invalidating B's only source edge. Recomputing evidence oscillates, so the + // analyzer must fail closed to explicit R instead of retaining B forever. + auto stale_source_image = make_exe_buffer(0x1000); + put32(stale_source_image, text + 0x00, + 0x08000000u | (((kLoad + 0x100) >> 2) & 0x03FFFFFFu)); + put32(stale_source_image, text + 0x04, 0x00000000u); + put32(stale_source_image, text + 0x80, 0x03E00008u); // X + put32(stale_source_image, text + 0x84, 0x00000000u); + put32(stale_source_image, text + 0x100, jal(kLoad + 0x140)); // S -> B + put32(stale_source_image, text + 0x104, 0x00000000u); + put32(stale_source_image, text + 0x108, 0x03E00008u); + put32(stale_source_image, text + 0x10C, 0x00000000u); + put32(stale_source_image, text + 0x140, jal(kLoad + 0x80)); // B -> X + put32(stale_source_image, text + 0x144, 0x00000000u); + put32(stale_source_image, text + 0x148, 0x03E00008u); + put32(stale_source_image, text + 0x14C, 0x00000000u); + auto stale_source_exe = parse(stale_source_image); + PSXRecomp::FunctionAnalyzer stale_source_analyzer(stale_source_exe); + const auto stale_source_result = + stale_source_analyzer.analyze_exact_entries({kLoad}); + CHECK(starts(stale_source_result) == std::set({kLoad}) && + stale_source_result.absorbed_entries.empty(), + "cyclic stale call evidence fails closed to explicit roots"); + + // beq rN,rN is unconditional: PC+8 is not reachable and a hidden JAL + // there cannot manufacture a function or absorbed alias. + auto always_branch_image = make_exe_buffer(0x1000); + put32(always_branch_image, text + 0x00, + (0x04u << 26) | (8u << 21) | (8u << 16) | 0x1Fu); // -> +0x80 + put32(always_branch_image, text + 0x04, 0x00000000u); + put32(always_branch_image, text + 0x08, jal(kLoad + 0x40)); + put32(always_branch_image, text + 0x0C, 0x00000000u); + put32(always_branch_image, text + 0x40, 0x03E00008u); + put32(always_branch_image, text + 0x44, 0x00000000u); + put32(always_branch_image, text + 0x80, 0x03E00008u); + put32(always_branch_image, text + 0x84, 0x00000000u); + auto always_branch_exe = parse(always_branch_image); + PSXRecomp::FunctionAnalyzer always_branch_analyzer(always_branch_exe); + const auto always_branch_result = + always_branch_analyzer.analyze_exact_entries({kLoad}); + CHECK(starts(always_branch_result) == std::set({kLoad}) && + always_branch_result.absorbed_entries.empty(), + "always-taken branch excludes unreachable fallthrough call"); + + // bne rN,rN is never taken: its encoded target cannot contribute a hidden + // call edge, while its ordinary delay slot and PC+8 remain reachable. + auto never_branch_image = make_exe_buffer(0x1000); + put32(never_branch_image, text + 0x00, + (0x05u << 26) | (8u << 21) | (8u << 16) | 0x1Fu); // false -> +0x80 + put32(never_branch_image, text + 0x04, 0x00000000u); + put32(never_branch_image, text + 0x08, 0x03E00008u); + put32(never_branch_image, text + 0x0C, 0x00000000u); + put32(never_branch_image, text + 0x40, 0x03E00008u); + put32(never_branch_image, text + 0x44, 0x00000000u); + put32(never_branch_image, text + 0x80, jal(kLoad + 0x40)); + put32(never_branch_image, text + 0x84, 0x00000000u); + put32(never_branch_image, text + 0x88, 0x03E00008u); + put32(never_branch_image, text + 0x8C, 0x00000000u); + auto never_branch_exe = parse(never_branch_image); + PSXRecomp::FunctionAnalyzer never_branch_analyzer(never_branch_exe); + const auto never_branch_result = + never_branch_analyzer.analyze_exact_entries({kLoad}); + CHECK(starts(never_branch_result) == std::set({kLoad}) && + never_branch_result.absorbed_entries.empty(), + "never-taken branch excludes unreachable target call"); + + // bgezal $zero is the BAL alias: the target is call evidence and the + // return continuation at PC+8 remains reachable. + auto bal_image = make_exe_buffer(0x1000); + put32(bal_image, text + 0x00, + (0x01u << 26) | (0x11u << 16) | 0x1Fu); // bal +0x80 + put32(bal_image, text + 0x04, 0x00000000u); + put32(bal_image, text + 0x08, jal(kLoad + 0x40)); + put32(bal_image, text + 0x0C, 0x00000000u); + put32(bal_image, text + 0x10, 0x03E00008u); + put32(bal_image, text + 0x14, 0x00000000u); + put32(bal_image, text + 0x40, 0x03E00008u); + put32(bal_image, text + 0x44, 0x00000000u); + put32(bal_image, text + 0x80, 0x03E00008u); + put32(bal_image, text + 0x84, 0x00000000u); + auto bal_exe = parse(bal_image); + PSXRecomp::FunctionAnalyzer bal_analyzer(bal_exe); + const auto bal_starts = starts(bal_analyzer.analyze_exact_entries({kLoad})); + CHECK(bal_starts.count(kLoad + 0x40) && bal_starts.count(kLoad + 0x80), + "BAL preserves call target and return-continuation discovery"); + // A synthetic adjacent-producer envelope is not one linked image. The // caller supplies the cross-boundary targets that passed its independent // callable-CFG proof; other valid-looking targets stay with the interpreter. @@ -182,6 +445,49 @@ int main() { CHECK(indirect.count(kLoad + 0x400) && indirect.count(kLoad + 0x500), "constant-register JR discovers its frameless tail-call target"); + // A jump enters at the low-half definition and skips the preceding LUI. + // A raw backward scan would combine them; reaching-definition proof must + // reject that synthetic indirect target. + auto split_constant_image = make_exe_buffer(0x1000, kLoad + 0x500); + put32(split_constant_image, text + 0x400, 0x03E00008u); + put32(split_constant_image, text + 0x404, 0x00000000u); + put32(split_constant_image, text + 0x500, + 0x08000000u | (((kLoad + 0x518) >> 2) & 0x03FFFFFFu)); + put32(split_constant_image, text + 0x504, 0x00000000u); + put32(split_constant_image, text + 0x514, 0x3C088001u); + put32(split_constant_image, text + 0x518, 0x25080400u); + put32(split_constant_image, text + 0x51C, 0x01000008u); + put32(split_constant_image, text + 0x520, 0x00000000u); + auto split_constant_exe = parse(split_constant_image); + PSXRecomp::FunctionAnalyzer split_constant_analyzer(split_constant_exe); + const auto split_constant = starts( + split_constant_analyzer.analyze_exact_entries({kLoad + 0x500})); + CHECK(!split_constant.count(kLoad + 0x400), + "indirect resolver rejects a definition skipped by an inbound edge"); + + // The LUI executes in a JAL delay slot, then the callee may clobber t0 + // before returning to the low half. It is not a reaching definition for + // the later JALR even though the raw words are adjacent. + auto call_delay_image = make_exe_buffer(0x1000, kLoad + 0x500); + put32(call_delay_image, text + 0x300, 0x03E00008u); + put32(call_delay_image, text + 0x304, 0x00000000u); + put32(call_delay_image, text + 0x400, 0x03E00008u); + put32(call_delay_image, text + 0x404, 0x00000000u); + put32(call_delay_image, text + 0x500, jal(kLoad + 0x300)); + put32(call_delay_image, text + 0x504, 0x3C088001u); // call delay LUI + put32(call_delay_image, text + 0x508, 0x25080400u); + put32(call_delay_image, text + 0x50C, 0x0100F809u); + put32(call_delay_image, text + 0x510, 0x00000000u); + put32(call_delay_image, text + 0x514, 0x03E00008u); + put32(call_delay_image, text + 0x518, 0x00000000u); + auto call_delay_exe = parse(call_delay_image); + PSXRecomp::FunctionAnalyzer call_delay_analyzer(call_delay_exe); + const auto call_delay = starts( + call_delay_analyzer.analyze_exact_entries({kLoad + 0x500})); + CHECK(call_delay.count(kLoad + 0x300) && + !call_delay.count(kLoad + 0x400), + "indirect resolver rejects a LUI in a call delay slot"); + // A packed BIOS thunk is also a backward-scan boundary. The frameless // leaf after its delay slot must not be swallowed into the thunk run. auto thunk_image = make_exe_buffer(0x1000); diff --git a/recompiler/tests/test_aot_overlay_discovery.py b/recompiler/tests/test_aot_overlay_discovery.py index 5b24989e..e9f93179 100644 --- a/recompiler/tests/test_aot_overlay_discovery.py +++ b/recompiler/tests/test_aot_overlay_discovery.py @@ -8,6 +8,7 @@ import struct import subprocess import sys import tempfile +import time ROOT = pathlib.Path(__file__).resolve().parents[2] @@ -128,6 +129,445 @@ def check_owned_direct_call_is_interior(): assert f'interior 0x{target:08X}' in seeds +def check_discovered_host_owns_same_round_call_target(): + data = bytearray(0x200) + host = LOAD + 0x100 + target = host + 0x08 + put(data, 0x00, jal(host)) + put(data, 0x04, 0x00000000) + put(data, 0x08, jal(target)) + put(data, 0x0C, 0x00000000) + put(data, 0x10, 0x03E00008) + put(data, 0x14, 0x00000000) + put(data, 0x100, 0x24020001) + put(data, 0x104, 0x24420001) + put(data, 0x108, 0x24420001) + put(data, 0x10C, 0x03E00008) + put(data, 0x110, 0x00000000) + cap = { + 'schema': 'psxrecomp overlay capture v2', + 'function_entry_pcs': [f'0x{LOAD:08X}'], + } + seeds, audit = MOD.classify_overlay_seeds( + cap, bytes(data), LOAD, len(data), 0, {}) + assert audit['included_reasons'][host] == 'DIRECT_JAL_TARGET' + assert audit['included_reasons'][target] == 'DISPATCH_INTERIOR' + assert f'call_root 0x{host:08X}' in seeds + assert f'interior 0x{target:08X}' in seeds + + +def check_later_discovered_host_retracts_interior_root(): + data = bytearray(0x200) + host = LOAD + 0x100 + target = LOAD + 0x120 + put(data, 0x00, jal(target)) + put(data, 0x04, 0x00000000) + put(data, 0x08, 0x03E00008) + put(data, 0x0C, 0x00000000) + put(data, 0x100, 0x10000007) # b target + put(data, 0x104, 0x00000000) + put(data, 0x120, jal(host)) + put(data, 0x124, 0x00000000) + put(data, 0x128, 0x03E00008) + put(data, 0x12C, 0x00000000) + cap = { + 'schema': 'psxrecomp overlay capture v2', + 'function_entry_pcs': [f'0x{LOAD:08X}'], + } + seeds, audit = MOD.classify_overlay_seeds( + cap, bytes(data), LOAD, len(data), 0, {}) + assert audit['included_reasons'][host] == 'DIRECT_JAL_TARGET' + assert audit['included_reasons'][target] == 'DISPATCH_INTERIOR' + assert f'call_root 0x{host:08X}' in seeds + assert f'interior 0x{target:08X}' in seeds + + +def check_recompiler_discovered_host_ownership(recompiler): + data = bytearray(0x200) + host = LOAD + 0x100 + target = host + 0x08 + put(data, 0x00, jal(host)) + put(data, 0x04, 0x00000000) + put(data, 0x08, jal(target)) + put(data, 0x0C, 0x00000000) + put(data, 0x10, 0x03E00008) + put(data, 0x14, 0x00000000) + put(data, 0x100, 0x24020001) + put(data, 0x104, 0x24420001) + put(data, 0x108, 0x24420001) + put(data, 0x10C, 0x03E00008) + put(data, 0x110, 0x00000000) + + with tempfile.TemporaryDirectory() as tmp: + psx = os.path.join(tmp, 'discovered_ownership.psx') + seeds = os.path.join(tmp, 'seeds.txt') + out = os.path.join(tmp, 'out') + with open(psx, 'wb') as f: + f.write(make_psxexe(LOAD, data)) + with open(seeds, 'w', encoding='utf-8') as f: + f.write(f'0x{LOAD:08X}\n') + f.write(f'interior 0x{target:08X}\n') + result = subprocess.run( + [recompiler, psx, '--seeds', seeds, '--out-dir', out, '--overlay'], + capture_output=True, text=True) + assert result.returncode == 0, result.stderr or result.stdout + full = ''.join(path.read_text() + for path in pathlib.Path(out).glob('*_full*.c')) + ranges = next(pathlib.Path(out).glob('*_full.ranges')).read_text() + + assert f'void func_{host:08X}' in full + assert f'void func_{target:08X}' in full + assert f'F {target:08X}' in ranges + assert f'R {host:08X}' in ranges + + +def assert_all_interiors_have_final_host(data, seeds, producer_ranges=(), + allow_cross=True): + roots = [] + interiors = set() + for seed in seeds: + if seed.startswith('interior '): + interiors.add(int(seed.split()[-1], 16)) + elif seed.startswith(('producer_range ', 'cross_call_allow ', + 'retained_alias ')): + continue + else: + roots.append(int(seed.split()[-1], 16)) + covered = set() + roots.sort() + for index, entry in enumerate(roots): + hard_cap = roots[index + 1] if index + 1 < len(roots) else LOAD + len(data) + covered.update(MOD._walk_overlay_function( + bytes(data), LOAD, len(data), entry, hard_cap, producer_ranges, + allow_cross)['visited']) + assert interiors <= covered, (interiors - covered) + + +def check_ownership_invalidation_and_producer_boundaries(): + # R discovers A+B; expanded A discovers X between them. X caps A before B, + # so the final partition must reconsider B as a root, not retain a stale + # A-owned interior classification. + data = bytearray(0x200) + host = LOAD + 0x100 + middle = LOAD + 0x120 + target = LOAD + 0x140 + put(data, 0x00, jal(host)) + put(data, 0x04, 0x00000000) + put(data, 0x08, jal(target)) + put(data, 0x0C, 0x00000000) + put(data, 0x10, 0x03E00008) + put(data, 0x14, 0x00000000) + put(data, 0x100, 0x08000000 | (((LOAD + 0x130) >> 2) & 0x03FFFFFF)) + put(data, 0x104, 0x00000000) + put(data, 0x120, 0x03E00008) + put(data, 0x124, 0x00000000) + put(data, 0x130, 0x24420001) + put(data, 0x134, 0x24420001) + put(data, 0x138, 0x24420001) + put(data, 0x13C, 0x24420001) + put(data, 0x140, jal(middle)) + put(data, 0x144, 0x00000000) + put(data, 0x148, 0x03E00008) + put(data, 0x14C, 0x00000000) + cap = { + 'schema': 'psxrecomp overlay capture v2', + 'function_entry_pcs': [f'0x{LOAD:08X}'], + } + seeds, audit = MOD.classify_overlay_seeds( + cap, bytes(data), LOAD, len(data), 0, {}) + root_addrs = sorted(int(seed.split()[-1], 16) for seed in seeds + if not seed.startswith(('interior ', 'producer_range ', + 'cross_call_allow ', + 'retained_alias '))) + final_predecessor = max(addr for addr in root_addrs if addr < target) + assert final_predecessor != host + assert (audit['included_reasons'][target] != 'DISPATCH_INTERIOR' or + final_predecessor == LOAD + 0x130) + assert_all_interiors_have_final_host(data, seeds) + + # A and B straddle producer boundaries. The unapproved cross-producer call + # is rejected entirely; approval makes B a separate root, never an alias + # owned by A's apparent fallthrough. + cross = bytearray(0x200) + a = LOAD + 0x100 + b = LOAD + 0x108 + put(cross, 0x00, jal(a)) + put(cross, 0x04, 0x00000000) + put(cross, 0x08, jal(b)) + put(cross, 0x0C, 0x00000000) + put(cross, 0x10, 0x03E00008) + put(cross, 0x14, 0x00000000) + put(cross, 0x100, 0x24420001) + put(cross, 0x104, 0x24420001) + put(cross, 0x108, 0x03E00008) + put(cross, 0x10C, 0x00000000) + raw_ranges = [ + {'start': f'0x{LOAD:08X}', 'end': f'0x{b:08X}'}, + {'start': f'0x{b:08X}', 'end': f'0x{LOAD + len(cross):08X}'}, + ] + strict_cap = dict(cap, producer_ranges=raw_ranges, + strict_producer_ranges=True) + strict_seeds, strict_audit = MOD.classify_overlay_seeds( + strict_cap, bytes(cross), LOAD, len(cross), 0, {}) + assert b not in strict_audit['included_reasons'] + assert b in strict_audit['rejected_cross_producer_calls'] + assert not any(seed.endswith(f'0x{b:08X}') and + not seed.startswith('producer_range ') + for seed in strict_seeds) + parsed_ranges = [(LOAD, b), (b, LOAD + len(cross))] + assert_all_interiors_have_final_host( + cross, strict_seeds, parsed_ranges, False) + + allowed_cap = dict(cap, producer_ranges=raw_ranges, + strict_producer_ranges=False) + allowed_seeds, allowed_audit = MOD.classify_overlay_seeds( + allowed_cap, bytes(cross), LOAD, len(cross), 0, {}) + assert allowed_audit['included_reasons'][b] == 'DIRECT_JAL_TARGET' + assert f'call_root 0x{b:08X}' in allowed_seeds + assert f'interior 0x{b:08X}' not in allowed_seeds + assert_all_interiors_have_final_host( + cross, allowed_seeds, parsed_ranges, True) + + crossing_alias_cap = dict( + strict_cap, + static_alias_ranges=[{ + 'entry': f'0x{a:08X}', + 'start': f'0x{a:08X}', + 'end': f'0x{b + 4:08X}', + }]) + try: + MOD.classify_overlay_seeds( + crossing_alias_cap, bytes(cross), LOAD, len(cross), 0, {}) + assert False, 'cross-producer supplied alias was accepted' + except RuntimeError as exc: + assert 'crosses producer boundary' in str(exc) + + # A seed in composite padding has no producer. It must be rejected rather + # than letting producer_for(None) equate disconnected uncovered gaps. + gap = LOAD + 0x60 + gap_data = bytearray(0x100) + put(gap_data, 0x00, 0x03E00008) + put(gap_data, 0x04, 0x00000000) + put(gap_data, 0x60, 0x24020001) + put(gap_data, 0x64, 0x03E00008) + put(gap_data, 0x68, 0x00000000) + gap_cap = { + 'schema': 'psxrecomp overlay capture v2', + 'function_entry_pcs': [f'0x{LOAD:08X}', f'0x{gap:08X}'], + 'producer_ranges': [ + {'start': f'0x{LOAD:08X}', 'end': f'0x{LOAD + 0x40:08X}'}, + {'start': f'0x{LOAD + 0x80:08X}', + 'end': f'0x{LOAD + len(gap_data):08X}'}, + ], + 'strict_producer_ranges': True, + } + gap_seeds, gap_audit = MOD.classify_overlay_seeds( + gap_cap, bytes(gap_data), LOAD, len(gap_data), 0, {}) + assert gap not in gap_audit['included_reasons'] + assert not any(seed.endswith(f'0x{gap:08X}') and + not seed.startswith('producer_range ') + for seed in gap_seeds) + + orphan = LOAD + 0x180 + put(cross, 0x180, 0x24420001) + orphan_cap = dict(cap, dispatch_entry_pcs=[f'0x{orphan:08X}']) + orphan_seeds, orphan_audit = MOD.classify_overlay_seeds( + orphan_cap, bytes(cross), LOAD, len(cross), 0, {}) + assert orphan not in orphan_audit['included_reasons'] + assert f'interior 0x{orphan:08X}' not in orphan_seeds + assert_all_interiors_have_final_host(cross, orphan_seeds) + + +def check_recompiler_unreachable_jal_not_alias(recompiler): + # The host's address envelope contains an unreachable JAL-shaped word whose + # non-dense target looks like code. Only a raw envelope rescan sees it; the + # reachable analyzer must not export it as an alias. + data = bytearray(0x100) + target = LOAD + 0x40 + put(data, 0x00, 0x08000000 | (((LOAD + 0x80) >> 2) & 0x03FFFFFF)) + put(data, 0x04, 0x00000000) + put(data, 0x20, jal(target)) + put(data, 0x40, 0x24420001) + put(data, 0x44, 0x03E00008) + put(data, 0x48, 0x00000000) + put(data, 0x80, 0x03E00008) + put(data, 0x84, 0x00000000) + with tempfile.TemporaryDirectory() as tmp: + psx = os.path.join(tmp, 'unreachable_jal.psx') + seeds = os.path.join(tmp, 'seeds.txt') + out = os.path.join(tmp, 'out') + with open(psx, 'wb') as f: + f.write(make_psxexe(LOAD, data)) + with open(seeds, 'w', encoding='utf-8') as f: + f.write(f'0x{LOAD:08X}\n') + f.write(f'interior 0x{target:08X}\n') + result = subprocess.run( + [recompiler, psx, '--seeds', seeds, '--out-dir', out, '--overlay'], + capture_output=True, text=True) + assert result.returncode == 0, result.stderr or result.stdout + full = ''.join(path.read_text() + for path in pathlib.Path(out).glob('*_full*.c')) + assert f'void func_{target:08X}' not in full + + +def check_recomputed_evidence_and_unconditional_branch(): + # R reaches S, S calls B, and B calls earlier X. Adding X caps R before S, + # invalidating the only source edge for B. This oscillating partition must + # fail closed to explicit R rather than retain monotonic B evidence. + data = bytearray(0x200) + x = LOAD + 0x80 + s = LOAD + 0x100 + b = LOAD + 0x140 + put(data, 0x00, 0x08000000 | ((s >> 2) & 0x03FFFFFF)) + put(data, 0x04, 0x00000000) + put(data, 0x80, 0x03E00008) + put(data, 0x84, 0x00000000) + put(data, 0x100, jal(b)) + put(data, 0x104, 0x00000000) + put(data, 0x108, 0x03E00008) + put(data, 0x10C, 0x00000000) + put(data, 0x140, jal(x)) + put(data, 0x144, 0x00000000) + put(data, 0x148, 0x03E00008) + put(data, 0x14C, 0x00000000) + cap = {'schema': 'psxrecomp overlay capture v2', + 'function_entry_pcs': [f'0x{LOAD:08X}']} + seeds, audit = MOD.classify_overlay_seeds( + cap, bytes(data), LOAD, len(data), 0, {}) + assert b not in audit['included_reasons'] + assert not any(seed.endswith(f'0x{b:08X}') for seed in seeds) + + # Always-taken beq rN,rN has no PC+8 fallthrough. A JAL-shaped word + # there is unreachable and cannot manufacture a root or alias. + branch = bytearray(0x100) + hidden = LOAD + 0x40 + put(branch, 0x00, (0x04 << 26) | (8 << 21) | (8 << 16) | 0x1F) + put(branch, 0x04, 0x00000000) + put(branch, 0x08, jal(hidden)) + put(branch, 0x0C, 0x00000000) + put(branch, 0x40, 0x03E00008) + put(branch, 0x44, 0x00000000) + put(branch, 0x80, 0x03E00008) + put(branch, 0x84, 0x00000000) + branch_seeds, branch_audit = MOD.classify_overlay_seeds( + cap, bytes(branch), LOAD, len(branch), 0, {}) + assert hidden not in branch_audit['included_reasons'] + assert not any(seed.endswith(f'0x{hidden:08X}') for seed in branch_seeds) + + # Never-taken bne rN,rN must not walk its encoded target. + never = bytearray(0x100) + put(never, 0x00, (0x05 << 26) | (8 << 21) | (8 << 16) | 0x1F) + put(never, 0x04, 0x00000000) + put(never, 0x08, 0x03E00008) + put(never, 0x0C, 0x00000000) + put(never, 0x40, 0x03E00008) + put(never, 0x44, 0x00000000) + put(never, 0x80, jal(hidden)) + put(never, 0x84, 0x00000000) + put(never, 0x88, 0x03E00008) + put(never, 0x8C, 0x00000000) + never_seeds, never_audit = MOD.classify_overlay_seeds( + cap, bytes(never), LOAD, len(never), 0, {}) + assert hidden not in never_audit['included_reasons'] + assert not any(seed.endswith(f'0x{hidden:08X}') for seed in never_seeds) + + # All zero-provable branch families use exact direction. Likely-false + # variants additionally annul their delay slot. + target = LOAD + 0x80 + assert MOD._classify_cf(LOAD, (0x06 << 26) | 0x1F) == ('j', target) + assert MOD._classify_cf(LOAD, (0x07 << 26) | 0x1F) == ('branch_never', 0) + assert MOD._classify_cf(LOAD, (0x01 << 26) | (0x00 << 16) | 0x1F) == ( + 'branch_never', 0) # bltz zero + assert MOD._classify_cf(LOAD, (0x01 << 26) | (0x01 << 16) | 0x1F) == ( + 'j', target) # bgez zero + assert MOD._classify_cf(LOAD, (0x01 << 26) | (0x11 << 16) | 0x1F) == ( + 'jal', target) # bal keeps both target and return continuation + assert MOD._classify_cf(LOAD, (0x15 << 26) | (8 << 21) | + (8 << 16) | 0x1F) == ('branch_never_likely', 0) + assert MOD._classify_cf(LOAD, (0x17 << 26) | 0x1F) == ( + 'branch_never_likely', 0) + + # Unresolved JR-table inference is disabled until dependence is proven. + assert MOD._find_jump_table_targets( + bytes(branch), LOAD, len(branch), LOAD, LOAD + len(branch), + LOAD + 0x20, 8) == set() + + # A jump enters at the low-half definition, skipping the preceding LUI. + # The backward resolver must not combine definitions across that inbound + # basic-block boundary. + split = bytearray(0x600) + put(split, 0x400, 0x03E00008) + put(split, 0x404, 0x00000000) + put(split, 0x500, 0x08000000 | (((LOAD + 0x518) >> 2) & 0x03FFFFFF)) + put(split, 0x504, 0x00000000) + put(split, 0x514, 0x3C088001) + put(split, 0x518, 0x25080400) + put(split, 0x51C, 0x01000008) + put(split, 0x520, 0x00000000) + split_walk = MOD._walk_overlay_function( + bytes(split), LOAD, len(split), LOAD + 0x500, LOAD + len(split)) + assert split_walk['static_indirect_targets'] == set() + + # A LUI in a JAL delay slot can be clobbered by the callee before control + # returns to the low half. Raw adjacency is not reaching-definition proof. + call_delay = bytearray(0x600) + put(call_delay, 0x300, 0x03E00008) + put(call_delay, 0x304, 0x00000000) + put(call_delay, 0x400, 0x03E00008) + put(call_delay, 0x404, 0x00000000) + put(call_delay, 0x500, jal(LOAD + 0x300)) + put(call_delay, 0x504, 0x3C088001) + put(call_delay, 0x508, 0x25080400) + put(call_delay, 0x50C, 0x0100F809) + put(call_delay, 0x510, 0x00000000) + put(call_delay, 0x514, 0x03E00008) + put(call_delay, 0x518, 0x00000000) + call_delay_walk = MOD._walk_overlay_function( + bytes(call_delay), LOAD, len(call_delay), LOAD + 0x500, + LOAD + len(call_delay)) + assert call_delay_walk['direct_jals'] == {LOAD + 0x300} + assert call_delay_walk['static_indirect_targets'] == set() + + +def check_t2_shaped_retained_partition_conflict(recompiler): + # Shape of the causal T2 regression: current ordinary host BA0 ends +0x114, + # new ordinary roots exist at +0x17C/+0x180, while an old retained alias + # claims the original host through +0x2A8. The old range must be ignored. + data = bytearray(0x300) + old_alias = LOAD + 0x114 + split_a = LOAD + 0x17C + split_b = LOAD + 0x180 + put(data, 0x10C, 0x03E00008) + put(data, 0x110, 0x00000000) + put(data, 0x17C, 0x24420001) + put(data, 0x180, 0x03E00008) + put(data, 0x184, 0x00000000) + with tempfile.TemporaryDirectory() as tmp: + psx = os.path.join(tmp, 't2_partition_conflict.psx') + seed_path = os.path.join(tmp, 'seeds.txt') + out = os.path.join(tmp, 'out') + with open(psx, 'wb') as f: + f.write(make_psxexe(LOAD, data)) + with open(seed_path, 'w', encoding='utf-8') as f: + f.write(f'0x{LOAD:08X}\n') + f.write(f'call_root 0x{split_a:08X}\n') + f.write(f'call_root 0x{split_b:08X}\n') + f.write(f'retained_alias 0x{old_alias:08X} ' + f'0x{LOAD:08X} 0x{LOAD + 0x2A8:08X}\n') + result = subprocess.run( + [recompiler, psx, '--seeds', seed_path, '--out-dir', out, + '--overlay'], capture_output=True, text=True) + assert result.returncode == 0, result.stderr or result.stdout + ranges = next(pathlib.Path(out).glob('*_full.ranges')).read_text() + full = ''.join(path.read_text() + for path in pathlib.Path(out).glob('*_full*.c')) + assert 'ignoring legacy retained_alias' in result.stdout + assert f'F {old_alias:08X}' not in ranges + assert f'void func_{old_alias:08X}' not in full + assert f'F {split_a:08X}' in ranges + assert f'F {split_b:08X}' in ranges + + def check_static_alias_recipe(): data = bytearray(0x40) put(data, 0x00, 0x03E00008) @@ -146,8 +586,9 @@ def check_static_alias_recipe(): } seeds, audit = MOD.classify_overlay_seeds( cap, bytes(data), LOAD, len(data), 0, {}) - assert (f'retained_alias 0x{LOAD + 0x20:08X} ' - f'0x{LOAD:08X} 0x{LOAD + 0x2C:08X}') in seeds + assert not any(seed.startswith('retained_alias ') for seed in seeds) + assert f'interior 0x{LOAD + 0x20:08X}' not in seeds + assert LOAD + 0x20 not in audit['included_reasons'] assert audit['static_alias_ranges'] == { (LOAD + 0x20, LOAD, LOAD + 0x2C)} @@ -315,9 +756,41 @@ def check_recompiler_pointer_table_call_root(recompiler): assert f"void func_{target:08X}" not in full +def check_recompiler_pointer_table_alias(recompiler): + # A function envelope may span unreachable data between a jump and its + # target. A JAL-shaped word in that hole must not turn a dense pointer + # table inside the envelope into an alias function. + data = bytearray(0x100) + target = LOAD + 0x40 + put(data, 0x00, 0x08000000 | (((LOAD + 0x80) >> 2) & 0x03FFFFFF)) + put(data, 0x04, 0x00000000) + put(data, 0x20, jal(target)) + for index, offset in enumerate((0x70, 0x80, 0x90)): + put(data, 0x40 + index * 4, LOAD + offset) + put(data, 0x80, 0x03E00008) + put(data, 0x84, 0x00000000) + + with tempfile.TemporaryDirectory() as tmp: + psx = os.path.join(tmp, "pointer_table_alias.psx") + seeds = os.path.join(tmp, "seeds.txt") + out = os.path.join(tmp, "out") + with open(psx, "wb") as f: + f.write(make_psxexe(LOAD, data)) + with open(seeds, "w", encoding="utf-8") as f: + f.write(f"0x{LOAD:08X}\n") + result = subprocess.run( + [recompiler, psx, "--seeds", seeds, "--out-dir", out, "--overlay"], + capture_output=True, text=True) + assert result.returncode == 0, result.stderr or result.stdout + full = ''.join(path.read_text() + for path in pathlib.Path(out).glob("*_full*.c")) + assert "+1 in-function jal-target alias entries" not in result.stdout + assert f"void func_{target:08X}" not in full + + def check_retained_alias_contract(recompiler): - # A previously compiled indirect entry retains its overlapping host body - # without becoming a root that caps the newly discovered ordinary walk. + # Legacy host ranges are ignored. A prior alias can be reintroduced only as + # an entry candidate and must acquire a host from current exact reachability. data = bytearray(0x80) put(data, 0x00, 0x03E00008) put(data, 0x04, 0x00000000) @@ -341,20 +814,22 @@ def check_retained_alias_contract(recompiler): capture_output=True, text=True) assert result.returncode == 0, result.stderr or result.stdout full = ''.join(path.read_text() for path in pathlib.Path(out).glob("*_full*.c")) - assert f"void func_{LOAD + 0x20:08X}" in full + assert f"void func_{LOAD + 0x20:08X}" not in full ranges = next(pathlib.Path(out).glob("*_full.ranges")).read_text() - assert f"F {LOAD + 0x20:08X}" in ranges - assert f"R {LOAD:08X} 2C" in ranges + assert f"F {LOAD + 0x20:08X}" not in ranges def check_atomic_dll_publication(): original = MOD._compile_dll_direct + original_replace = MOD.os.replace try: with tempfile.TemporaryDirectory() as tmp: final = os.path.join(tmp, "shard.dll") ranges = os.path.join(tmp, "shard.ranges") with open(final, "wb") as out: out.write(b"old-good-shard") + with open(ranges, "w", encoding="utf-8") as out: + out.write("OLD-RANGES\n") def fail_compile(_src, staged, _includes, **_kwargs): with open(staged, "wb") as out: @@ -367,8 +842,9 @@ def check_atomic_dll_publication(): assert not MOD.compile_dll("ignored.c", final, []) with open(final, "rb") as built: assert built.read() == b"old-good-shard" + assert pathlib.Path(ranges).read_text() == "OLD-RANGES\n" assert not list(pathlib.Path(tmp).glob(".*.tmp.*")) - assert not MOD.compiled_shard_complete(final) + assert MOD.compiled_shard_complete(final) def good_compile(_src, staged, _includes, **_kwargs): with open(staged, "wb") as out: @@ -378,14 +854,247 @@ def check_atomic_dll_publication(): return True MOD._compile_dll_direct = good_compile - assert MOD.compile_dll("ignored.c", final, []) + func_ids = [(0x80010000, 0xDEADBEEF, + ((0x80010000, 0x10),))] + pair_id = MOD.overlay_pair_id("new source", func_ids) + assert pair_id != MOD.overlay_pair_id("changed source", func_ids) + bound_source = MOD.add_overlay_pair_export("new source", pair_id) + assert f"overlay_pair_id(void) {{ return UINT64_C(0x{pair_id:016X}); }}" in bound_source + assert MOD.compile_dll("ignored.c", final, [], + func_ids=func_ids, pair_id=pair_id) with open(final, "rb") as built: assert built.read() == b"new-complete-shard" - with open(ranges, "w", encoding="utf-8") as out: - out.write("F 80010000 DEADBEEF\nR 80010000 10\n") + manifest = pathlib.Path(ranges).read_text() + assert f"P {pair_id:016X}\n" in manifest + assert "F 80010000 DEADBEEF\nR 80010000 10\n" in manifest assert MOD.compiled_shard_complete(final) assert not list(pathlib.Path(tmp).glob(".*.tmp.*")) + + # Kill a separate writer process AFTER each durable rename. This tests + # real kernel lock release and recovery, including the hardest point: + # the final DLL rename committed but journal cleanup never ran. + module_path = str(ROOT / "tools" / "compile_overlays.py") + crash_script = r''' +import importlib.util, os, sys +spec = importlib.util.spec_from_file_location("co_child", sys.argv[1]) +m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m) +real = m.os.replace +count = 0 +def crash_after(src, dst): + global count + result = real(src, dst) + count += 1 + if count == int(sys.argv[5]): os._exit(91) + return result +m.os.replace = crash_after +m.publish_shard_pair(sys.argv[2], sys.argv[3], sys.argv[4]) +''' + for fail_after in range(1, 6): + with tempfile.TemporaryDirectory() as tmp: + final = os.path.join(tmp, "shard.dll") + ranges = os.path.join(tmp, "shard.ranges") + staged = os.path.join(tmp, ".new.dll") + staged_ranges = os.path.join(tmp, ".new.ranges") + pathlib.Path(final).write_bytes(b"OLD-DLL") + pathlib.Path(ranges).write_text("OLD-RANGES\n") + pathlib.Path(staged).write_bytes(b"NEW-DLL") + pathlib.Path(staged_ranges).write_text("NEW-RANGES\n") + child = subprocess.run( + [sys.executable, "-c", crash_script, module_path, + staged, staged_ranges, final, str(fail_after)]) + assert child.returncode == 91 + + MOD.recover_shard_pair(final) + expected = ((b"NEW-DLL", "NEW-RANGES\n") if fail_after == 5 + else (b"OLD-DLL", "OLD-RANGES\n")) + assert pathlib.Path(final).read_bytes() == expected[0] + assert pathlib.Path(ranges).read_text() == expected[1] + assert not list(pathlib.Path(tmp).glob("*.pair-txn.json")) + + # Recovery is restartable if an individual rollback rename faults. + with tempfile.TemporaryDirectory() as tmp: + final = os.path.join(tmp, "shard.dll") + ranges = os.path.join(tmp, "shard.ranges") + staged = os.path.join(tmp, ".new.dll") + staged_ranges = os.path.join(tmp, ".new.ranges") + pathlib.Path(final).write_bytes(b"OLD-DLL") + pathlib.Path(ranges).write_text("OLD-RANGES\n") + pathlib.Path(staged).write_bytes(b"NEW-DLL") + pathlib.Path(staged_ranges).write_text("NEW-RANGES\n") + child = subprocess.run( + [sys.executable, "-c", crash_script, module_path, + staged, staged_ranges, final, "3"]) + assert child.returncode == 91 + real_replace = MOD.os.replace + failed_once = False + + def fail_one_restore(src, dst): + nonlocal failed_once + if not failed_once and str(src).endswith('.old-ranges'): + failed_once = True + raise OSError("injected recovery rename fault") + return real_replace(src, dst) + + MOD.os.replace = fail_one_restore + try: + try: + MOD.recover_shard_pair(final) + raise AssertionError("recovery fault was not surfaced") + except OSError as exc: + assert "injected recovery" in str(exc) + finally: + MOD.os.replace = real_replace + MOD.recover_shard_pair(final) + assert pathlib.Path(final).read_bytes() == b"OLD-DLL" + assert pathlib.Path(ranges).read_text() == "OLD-RANGES\n" + + # Real cross-process writers must serialize on the permanent OS lock. + publish_script = r''' +import importlib.util, sys +spec = importlib.util.spec_from_file_location("co_child", sys.argv[1]) +m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m) +m.publish_shard_pair(sys.argv[2], sys.argv[3], sys.argv[4]) +''' + with tempfile.TemporaryDirectory() as tmp: + final = os.path.join(tmp, "shard.dll") + ranges = os.path.join(tmp, "shard.ranges") + pathlib.Path(final).write_bytes(b"DLL-old") + pathlib.Path(ranges).write_text("RANGES-old") + writers = [] + for index in range(8): + staged = os.path.join(tmp, f"new-{index}.dll") + staged_ranges = os.path.join(tmp, f"new-{index}.ranges") + pathlib.Path(staged).write_bytes(f"DLL-{index}".encode()) + pathlib.Path(staged_ranges).write_text(f"RANGES-{index}") + writers.append(subprocess.Popen( + [sys.executable, "-c", publish_script, module_path, + staged, staged_ranges, final])) + assert all(writer.wait(timeout=30) == 0 for writer in writers) + dll_generation = pathlib.Path(final).read_text().removeprefix("DLL-") + range_generation = pathlib.Path(ranges).read_text().removeprefix("RANGES-") + assert dll_generation == range_generation + assert not list(pathlib.Path(tmp).glob("*.pair-txn.json")) + + # Readers must not observe the deliberate manifest-only publication + # window. Pause a real publisher after rename #4 while it owns the OS + # lock; completeness and coverage readers must block, then consume the + # committed pair rather than the transient half-state. + pause_script = r''' +import importlib.util, os, pathlib, sys, time +spec = importlib.util.spec_from_file_location("co_child", sys.argv[1]) +m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m) +real = m.os.replace +count = 0 +def pause_after_manifest(src, dst): + global count + result = real(src, dst); count += 1 + if count == 4: + pathlib.Path(sys.argv[5]).write_text("paused") + time.sleep(1.0) + return result +m.os.replace = pause_after_manifest +m.publish_shard_pair(sys.argv[2], sys.argv[3], sys.argv[4]) +''' + for reader in ('complete', 'coverage'): + with tempfile.TemporaryDirectory() as tmp: + final = os.path.join(tmp, "00010000_DEADBEEF.dll") + ranges = os.path.splitext(final)[0] + '.ranges' + staged = os.path.join(tmp, ".new.dll") + staged_ranges = os.path.join(tmp, ".new.ranges") + paused = os.path.join(tmp, "paused") + pathlib.Path(final).write_bytes(b"OLD-DLL") + pathlib.Path(ranges).write_text( + "F 80010004 AAAAAAAA\nR 80010004 4\n") + pathlib.Path(staged).write_bytes(b"NEW-DLL") + pathlib.Path(staged_ranges).write_text( + "F 80010000 DEADBEEF\nR 80010000 10\n") + writer = subprocess.Popen( + [sys.executable, "-c", pause_script, module_path, + staged, staged_ranges, final, paused]) + deadline = time.monotonic() + 10 + while not os.path.exists(paused) and time.monotonic() < deadline: + time.sleep(0.01) + assert os.path.exists(paused), "publisher did not reach pause point" + started = time.monotonic() + if reader == 'complete': + assert MOD.compiled_shard_complete(final) + else: + assert MOD.load_region_coverage(tmp, 0x10000) == { + (0x80010000, 0xDEADBEEF)} + elapsed = time.monotonic() - started + assert elapsed >= 0.6, f'{reader} escaped pair lock in {elapsed:.3f}s' + assert writer.wait(timeout=10) == 0 + + if os.name == 'nt': + # Windows keeps a loaded image mapped even after its canonical name + # is renamed. Old-backup deletion may therefore return WinError 5; + # a committed new pair must survive and cleanup must be deferred. + import _ctypes + import ctypes + import shutil + gcc = r'C:\msys64\mingw64\bin\gcc.exe' + assert os.path.isfile(gcc), "real loaded-DLL regression needs MinGW gcc" + with tempfile.TemporaryDirectory() as tmp: + # The actual host compiler must export the same 64-bit identity + # serialized in P; this catches width/decorated-name mistakes + # that a source-string assertion cannot. + pair_id = 0xFEDCBA9876543210 + pair_source = pathlib.Path(tmp) / "pair.c" + pair_dll = str(pathlib.Path(tmp) / "pair.dll") + pair_source.write_text( + '#include \n' + + '__declspec(dllexport) int overlay_abi(void) { return 14; }\n' + + MOD.add_overlay_pair_export('', pair_id)) + subprocess.run([gcc, '-shared', str(pair_source), '-o', pair_dll], + check=True, capture_output=True, text=True) + pathlib.Path(pair_dll).with_suffix('.ranges').write_text( + MOD.overlay_ranges_text([], pair_id)) + pair_lib = ctypes.WinDLL(pair_dll) + pair_fn = pair_lib.overlay_pair_id + pair_fn.restype = ctypes.c_uint64 + assert pair_fn() == pair_id + manifest = MOD.overlay_ranges_text([], pair_id) + assert f'P {pair_fn():016X}\n' in manifest + pair_handle = pair_lib._handle + pair_lib._handle = 0 + _ctypes.FreeLibrary(pair_handle) + assert MOD.compiled_shard_complete(pair_dll, 14) + assert not MOD.compiled_shard_complete( + pair_dll, 14 | (1 << 16)) # same cg dir, stale flavor + + source = pathlib.Path(tmp) / "loaded.c" + final = str(pathlib.Path(tmp) / "shard.dll") + ranges = str(pathlib.Path(tmp) / "shard.ranges") + staged = str(pathlib.Path(tmp) / ".new.dll") + staged_ranges = str(pathlib.Path(tmp) / ".new.ranges") + source.write_text("__declspec(dllexport) int loaded(void) { return 7; }\n") + subprocess.run([gcc, '-shared', str(source), '-o', final], + check=True, capture_output=True, text=True) + pathlib.Path(ranges).write_text("OLD-RANGES\n") + shutil.copyfile(final, staged) + pathlib.Path(staged_ranges).write_text("NEW-RANGES\n") + loaded = ctypes.WinDLL(final) + MOD.publish_shard_pair(staged, staged_ranges, final) + assert pathlib.Path(ranges).read_text() == "NEW-RANGES\n" + assert not os.path.exists(final + '.pair-txn.json') + deferred = list(pathlib.Path(tmp).glob( + 'shard.dll.pair-txn.*.old-dll')) + assert deferred, "loaded old DLL backup should require deferred cleanup" + handle = loaded._handle + loaded._handle = 0 + _ctypes.FreeLibrary(handle) + MOD.recover_shard_pair(final) + assert not [path for path in deferred if path.exists()] + + with tempfile.TemporaryDirectory() as tmp: + ranges = pathlib.Path(tmp) / "00010000_DEADBEEF.ranges" + ranges.write_text("F 80010000 DEADBEEF\nR 80010000 10\n") + assert MOD.load_region_coverage(tmp, 0x10000) == set() + ranges.with_suffix(MOD.overlay_ext()).write_bytes(b"DLL") + assert MOD.load_region_coverage(tmp, 0x10000) == { + (0x80010000, 0xDEADBEEF)} finally: + MOD.os.replace = original_replace MOD._compile_dll_direct = original @@ -434,11 +1143,19 @@ def main(): check_composite_call_boundaries() check_static_discovery_provenance() check_owned_direct_call_is_interior() + check_discovered_host_owns_same_round_call_target() + check_later_discovered_host_retracts_interior_root() + check_recompiler_discovered_host_ownership(args.recompiler) + check_ownership_invalidation_and_producer_boundaries() + check_recompiler_unreachable_jal_not_alias(args.recompiler) + check_recomputed_evidence_and_unconditional_branch() + check_t2_shaped_retained_partition_conflict(args.recompiler) check_static_alias_recipe() check_optional_enrichment_fallback() check_forward_branch_root() check_recompiler_composite_contract(args.recompiler) check_recompiler_pointer_table_call_root(args.recompiler) + check_recompiler_pointer_table_alias(args.recompiler) check_retained_alias_contract(args.recompiler) check_atomic_dll_publication() check_tcc_runtime_define_parity() diff --git a/runtime/src/overlay_loader.c b/runtime/src/overlay_loader.c index cf50dd51..3655758e 100644 --- a/runtime/src/overlay_loader.c +++ b/runtime/src/overlay_loader.c @@ -310,6 +310,9 @@ uint32_t overlay_loader_get_inprogress(void) { return s_native_inprogress; } /* Same-state differential — defined fully below; declared here for dispatch. */ static int s_diff_mode = 0; +/* Optional diagnostic filter: when non-zero, shadow-diff only this candidate + * entry while every other validated candidate follows its normal live route. */ +static uint32_t s_diff_addr = 0; static int s_in_shadow = 0; /* Candidate whose shadow NATIVE pass is currently executing (NULL outside it). * Set ONLY around run_shadow_diff's native pass — never during the interp pass, @@ -497,6 +500,7 @@ void overlay_loader_static_match_stats(uint64_t *rehashes, /* ---- Per-DLL code-range manifest --------------------------------------- */ /* Minimal line format emitted by tools/compile_overlays.py beside each DLL: + * P optional DLL/manifest publication identity * F start a function (entry = virtual export addr) * R a code byte-range (virtual addr, byte length) * The recompiler's per-function instruction walk yields exactly the executed @@ -511,8 +515,12 @@ typedef struct { int n; } ManFn; -static ManFn *parse_manifest(const char *path, int *out_n) { +static ManFn *parse_manifest(const char *path, int *out_n, + uint64_t *out_pair_id, + int *out_has_pair_id) { *out_n = 0; + if (out_pair_id) *out_pair_id = 0; + if (out_has_pair_id) *out_has_pair_id = 0; FILE *f = fopen(path, "r"); if (!f) return NULL; int cap = 1024, n = 0; @@ -521,7 +529,13 @@ static ManFn *parse_manifest(const char *path, int *out_n) { char line[160]; ManFn *cur = NULL; while (fgets(line, sizeof(line), f)) { - if (line[0] == 'F') { + if (line[0] == 'P') { + unsigned long long parsed = 0; + if (sscanf(line + 1, "%llx", &parsed) == 1) { + if (out_pair_id) *out_pair_id = (uint64_t)parsed; + if (out_has_pair_id) *out_has_pair_id = 1; + } + } else if (line[0] == 'F') { uint32_t e = 0, crc = 0; int got = sscanf(line + 1, "%x %x", &e, &crc); if (got >= 1) { @@ -817,7 +831,7 @@ static void rebuild_lazy_manifest_index(void) { snprintf(path + n - OVERLAY_SHARED_EXT_LEN, sizeof(path) - (n - OVERLAY_SHARED_EXT_LEN), ".ranges"); int man_n = 0; - ManFn *man = parse_manifest(path, &man_n); + ManFn *man = parse_manifest(path, &man_n, NULL, NULL); if (!man) continue; s_cache_idx[ci].func_count = man_n; s_cache_idx[ci].indexed_func_count = 0; @@ -1084,22 +1098,6 @@ static void cache_idx_remove_path(const char *path) { } } -static void remove_posix_dll_and_manifest(const char *dll_path) { -#ifndef _WIN32 - remove(dll_path); - char ranges[800]; - int n = snprintf(ranges, sizeof(ranges), "%s", dll_path); - if (n >= OVERLAY_SHARED_EXT_LEN && - (size_t)n + 7 - OVERLAY_SHARED_EXT_LEN < sizeof(ranges) && - strcmp(ranges + n - OVERLAY_SHARED_EXT_LEN, OVERLAY_SHARED_EXT) == 0) { - memcpy(ranges + n - OVERLAY_SHARED_EXT_LEN, ".ranges", 8); - remove(ranges); - } -#else - (void)dll_path; -#endif -} - #ifndef _WIN32 typedef struct PosixAbiSweep { int purged; @@ -1128,7 +1126,8 @@ static int posix_abi_sweep_file(const PsxOverlayCacheFile *file, void *opaque) { else loader_log("ABI preflight rejecting unloadable %s: %s", file->path, error); - remove_posix_dll_and_manifest(file->path); + /* Do not delete by canonical pathname after inspecting a loaded handle: + * an atomic publisher may already have replaced that path with a new file. */ cache_idx_remove_path(file->path); sweep->purged++; return 0; @@ -1143,8 +1142,9 @@ static int posix_abi_sweep_file(const PsxOverlayCacheFile *file, void *opaque) { * presented as a "(Not Responding)" black window for minutes (MMX6, 136-DLL * cache, the v10 migration, 2026-07-03 regression gate). * - * Instead, sweep ONCE at scan time: load each indexed DLL, check overlay_abi, - * delete mismatches (DLL + .ranges) and drop them from the index. A marker + * Instead, sweep at scan time: load each indexed DLL, check overlay_abi, and + * drop mismatches from the in-memory index. Never delete by canonical path + * after loading: a concurrent atomic publisher may have replaced that name. A marker * file (.abi_.ok) per cache dir records a completed sweep, so healthy * caches skip the sweep entirely on later boots — steady-state cost is one * stat per dir. Autocompile only ever writes current-ABI DLLs, so the marker @@ -1177,15 +1177,9 @@ static void abi_preflight_sweep(const char *dir) { FreeLibrary(h); if (abi == PSX_OVERLAY_ABI_TAG) { kept++; continue; } } - /* Unloadable or wrong ABI: purge DLL + its .ranges + index entry. */ - DeleteFileA(full); - char ranges[912]; - size_t n = strlen(full); - if (n > 4 && n + 4 < sizeof ranges) { - memcpy(ranges, full, n - 4); - memcpy(ranges + n - 4, ".ranges", 8); - DeleteFileA(ranges); - } + /* Drop only the index entry. Deleting `full` here can delete a new + * shard atomically swapped in after LoadLibrary returned the old + * handle. The compiler owns on-disk replacement. */ purged++; for (int i = 0; i < s_cache_idx_count; i++) { if (strcmp(s_cache_idx[i].path, full) == 0) { @@ -1197,12 +1191,15 @@ static void abi_preflight_sweep(const char *dir) { FindClose(hf); } if (purged) - loader_log("abi preflight: purged %d stale DLL(s), kept %d in %s", + loader_log("abi preflight: rejected %d stale DLL(s), kept %d in %s", purged, kept, dir); - /* Mark the sweep complete (even if nothing purged) so later boots skip it. */ - HANDLE m = CreateFileA(marker, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, - FILE_ATTRIBUTE_NORMAL, NULL); - if (m != INVALID_HANDLE_VALUE) CloseHandle(m); + /* Rejected files remain until the compiler replaces them, so do not create + * a marker that would skip their in-memory rejection next boot. */ + if (!purged) { + HANDLE m = CreateFileA(marker, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, NULL); + if (m != INVALID_HANDLE_VALUE) CloseHandle(m); + } #else char marker[900]; snprintf(marker, sizeof marker, "%s/.abi_%08x.ok", dir, @@ -1212,10 +1209,12 @@ static void abi_preflight_sweep(const char *dir) { PosixAbiSweep sweep = {0}; psx_overlay_posix_scan_cache_dir(dir, posix_abi_sweep_file, &sweep); if (sweep.purged) - loader_log("abi preflight: purged %d stale DLL(s), kept %d in %s", + loader_log("abi preflight: rejected %d stale DLL(s), kept %d in %s", sweep.purged, sweep.kept, dir); - FILE *m = fopen(marker, "wb"); - if (m) fclose(m); + if (!sweep.purged) { + FILE *m = fopen(marker, "wb"); + if (m) fclose(m); + } #endif } @@ -1603,28 +1602,39 @@ static void init_callbacks(void) { /* ---- DLL loading and export enumeration -------------------------------- */ #ifdef _WIN32 -static int load_overlay_dll(const char *dll_path, ManFn *man, int man_n, int dll) { +static int load_overlay_dll(const char *dll_path, ManFn *man, int man_n, int dll, + uint64_t manifest_pair_id, + int manifest_has_pair_id) { HMODULE h = LoadLibraryA(dll_path); if (!h) { loader_log("LoadLibrary(%s) failed: %lu", dll_path, GetLastError()); return 0; } - /* ABI gate: reject any DLL whose contract ABI doesn't match this - * runtime (pre-versioning DLLs lack the export entirely). Delete the - * stale file so the autocompile path regenerates it with the current - * emitter. */ + /* ABI gate: reject any DLL whose contract ABI doesn't match this runtime + * (pre-versioning DLLs lack the export entirely). The compiler, not this + * loaded handle, owns safe on-disk replacement. */ typedef int (*AbiFn)(void); AbiFn abi_fn = (AbiFn)GetProcAddress(h, "overlay_abi"); int abi = abi_fn ? abi_fn() : 0; /* Tag = ABI version (low 16) | codegen flavor (high 16). Mismatch on either * (wrong ABI, or a different-flavor cache e.g. widescreen vs base) is - * rejected + deleted so autocompile regenerates it for THIS build. */ + * rejected so autocompile can replace it for THIS build. */ if (abi != PSX_OVERLAY_ABI_TAG) { loader_log("ABI/flavor mismatch in %s: dll=0x%X runtime=0x%X — rejecting " - "and deleting stale cache entry", dll_path, abi, + "without pathname deletion", dll_path, abi, PSX_OVERLAY_ABI_TAG); FreeLibrary(h); - DeleteFileA(dll_path); + return 0; + } + typedef uint64_t (*PairIdFn)(void); + PairIdFn pair_fn = (PairIdFn)GetProcAddress(h, "overlay_pair_id"); + int dll_has_pair_id = pair_fn != NULL; + uint64_t dll_pair_id = pair_fn ? pair_fn() : 0; + if (dll_has_pair_id != manifest_has_pair_id || + (dll_has_pair_id && dll_pair_id != manifest_pair_id)) { + loader_log("DLL/manifest pair mismatch in %s -- rejecting without " + "deleting (publication may be in progress)", dll_path); + FreeLibrary(h); return 0; } typedef void (*InitFn)(const OverlayCallbacks *); @@ -1683,7 +1693,9 @@ static int load_overlay_dll(const char *dll_path, ManFn *man, int man_n, int dll return registered; } #else -static int load_overlay_dll(const char *dll_path, ManFn *man, int man_n, int dll) { +static int load_overlay_dll(const char *dll_path, ManFn *man, int man_n, int dll, + uint64_t manifest_pair_id, + int manifest_has_pair_id) { char error[256] = {0}; void *h = psx_overlay_posix_library_open(dll_path, error, sizeof(error)); if (!h) { loader_log("dlopen(%s) failed: %s", dll_path, error); return 0; } @@ -1693,10 +1705,21 @@ static int load_overlay_dll(const char *dll_path, ManFn *man, int man_n, int dll int abi = abi_fn ? abi_fn() : 0; if (abi != PSX_OVERLAY_ABI_TAG) { loader_log("ABI/flavor mismatch in %s: dll=0x%X runtime=0x%X — rejecting " - "and deleting stale cache entry", dll_path, abi, + "without pathname deletion", dll_path, abi, PSX_OVERLAY_ABI_TAG); psx_overlay_posix_library_close(h); - remove_posix_dll_and_manifest(dll_path); + return 0; + } + typedef uint64_t (*PairIdFn)(void); + PairIdFn pair_fn = (PairIdFn)psx_overlay_posix_library_symbol( + h, "overlay_pair_id"); + int dll_has_pair_id = pair_fn != NULL; + uint64_t dll_pair_id = pair_fn ? pair_fn() : 0; + if (dll_has_pair_id != manifest_has_pair_id || + (dll_has_pair_id && dll_pair_id != manifest_pair_id)) { + loader_log("DLL/manifest pair mismatch in %s -- rejecting without " + "deleting (publication may be in progress)", dll_path); + psx_overlay_posix_library_close(h); return 0; } typedef void (*InitFn)(const OverlayCallbacks *); @@ -1864,6 +1887,12 @@ void overlay_loader_init(const char *cache_dir, const char *game_id) { s_diff_mode = 1; loader_log("PSX_OVERLAY_DIFF set: native/interp shadow diff ON from boot"); } + const char *diff_addr = getenv("PSX_OVERLAY_DIFF_ADDR"); + if (diff_addr && *diff_addr) { + s_diff_addr = (uint32_t)strtoul(diff_addr, NULL, 0); + loader_log("PSX_OVERLAY_DIFF_ADDR set: shadow diff restricted to 0x%08X", + s_diff_addr); + } } s_active = 1; } @@ -1942,13 +1971,19 @@ static int load_one_dll(const char *dll_path) { sizeof(ranges_path) - (plen - OVERLAY_SHARED_EXT_LEN), ".ranges"); int man_n = 0; - ManFn *man = parse_manifest(ranges_path, &man_n); + uint64_t manifest_pair_id = 0; + int manifest_has_pair_id = 0; + ManFn *man = parse_manifest(ranges_path, &man_n, + &manifest_pair_id, + &manifest_has_pair_id); if (!man || man_n == 0) { loader_log("no/empty manifest %s — DLL left to interpreter", ranges_path); free(man); return 0; } - int registered = load_overlay_dll(dll_path, man, man_n, s_ndlls); + int registered = load_overlay_dll(dll_path, man, man_n, s_ndlls, + manifest_pair_id, + manifest_has_pair_id); #ifdef _WIN32 QueryPerformanceCounter(&q1); QueryPerformanceFrequency(&qf); @@ -2359,7 +2394,8 @@ retry_candidates: * too — the blocklist blocks LIVE native, not the sandbox). */ int in_own_shadow = (s_in_shadow && (const void *)c == s_shadow_cand); if (!in_own_shadow) { - int want_diff = s_diff_mode; + int want_diff = s_diff_mode && + (!s_diff_addr || phys == (s_diff_addr & 0x1FFFFFFFu)); if (want_diff && addr < 0x10000u) want_diff = 0; if (want_diff && c->diff_passes < OVERLAY_DIFF_BUDGET) { if (_probe) s_cps_probe_outcome = 5; @@ -2465,7 +2501,8 @@ retry_candidates: * gcc/DLL candidates are the trusted tier (validated at dev time) and * run native directly; they are diffed only in explicit dev diff mode * (PSX_OVERLAY_DIFF / overlay_diff cmd). */ - int want_diff = s_diff_mode; + int want_diff = s_diff_mode && + (!s_diff_addr || phys == (s_diff_addr & 0x1FFFFFFFu)); /* Kernel-window candidates (call gates 0xA0/0xB0/0xC0 and the RAM * kernel) are NOT shadow-diffable: the gates tail-jump via RAM * tables into kernel SERVICES whose behavior depends on scheduler/ diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index c9a484a8..7e0dc481 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -23,8 +23,12 @@ Each DLL exports: import argparse import base64 import binascii +from bisect import bisect_left from collections import Counter, deque from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager +import glob +import hashlib import os import re import struct @@ -32,6 +36,8 @@ import subprocess import sys import tempfile import threading +import time +import uuid class _ThreadLocalStdout: @@ -82,6 +88,17 @@ def codegen_ver(runtime_include: str) -> int: return int(m.group(1)) +def overlay_abi_tag(runtime_include: str, flavor: int) -> int: + """Return the exact ABI/flavor export expected from a cached shard.""" + hdr = os.path.join(runtime_include, 'overlay_api.h') + with open(hdr) as source: + match = re.search( + r'#define\s+PSX_OVERLAY_ABI_VERSION\s+(\d+)', source.read()) + if not match: + raise SystemExit(f'PSX_OVERLAY_ABI_VERSION not found in {hdr}') + return (int(match.group(1)) & 0xFFFF) | ((int(flavor) & 0xFFFF) << 16) + + def codegen_hash(runtime_include: str) -> int: """Parse PSX_OVERLAY_CODEGEN_HASH from the build-generated overlay_codegen_hash.h (next to overlay_api.h). Folded into the cache path as cg_ so ANY @@ -427,8 +444,10 @@ def _is_control_flow(word) -> bool: return False op = (word >> 26) & 0x3F fn = word & 0x3F - return op in (0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x14, 0x15, 0x16, 0x17) or (op == 0 and fn in (0x08, 0x09)) + return (op in (0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x14, 0x15, 0x16, 0x17) or + (op == 0 and fn in (0x08, 0x09)) or + (op in (0x11, 0x12) and ((word >> 21) & 0x1F) == 0x08)) def _is_valid_mips_word(word) -> bool: @@ -485,6 +504,30 @@ def _classify_cf(pc: int, word: int) -> tuple[str, int]: return ('j', _jump_target(pc, word)) if op == 0x03: return ('jal', _jump_target(pc, word)) + rt = (word >> 16) & 0x1F + if op in (0x04, 0x14) and rs == rt: + return ('j', _branch_target(pc, word)) + if op == 0x05 and rs == rt: + return ('branch_never', 0) + if op == 0x15 and rs == rt: + return ('branch_never_likely', 0) + if op in (0x06, 0x16) and rs == 0: + return ('j', _branch_target(pc, word)) + if op == 0x07 and rs == 0: + return ('branch_never', 0) + if op == 0x17 and rs == 0: + return ('branch_never_likely', 0) + if op == 0x01 and rs == 0: + if rt in (0x00, 0x10): # bltz / bltzal + return ('branch_never', 0) + if rt in (0x02, 0x12): # bltzl / bltzall + return ('branch_never_likely', 0) + if rt in (0x01, 0x03): # bgez / bgezl + return ('j', _branch_target(pc, word)) + if rt in (0x11, 0x13): # bal / ball aliases: target + continuation + return ('jal', _branch_target(pc, word)) + if op in (0x11, 0x12) and rs == 0x08: + return ('branch', _branch_target(pc, word)) if op in (0x01, 0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17): return ('branch', _branch_target(pc, word)) return ('normal', 0) @@ -493,76 +536,12 @@ def _classify_cf(pc: int, word: int) -> tuple[str, int]: def _find_jump_table_targets(data: bytes, load_addr: int, size: int, entry: int, hard_cap: int, jr_pc: int, jr_rs: int) -> set[int]: - """Recognize the compiler's local jump-table idiom feeding `jr reg`.""" - lw_base = None - lw_offset = 0 - addu_cand = [None, None] - lui_val = None - addiu_val = [0, 0] - found_addiu = [False, False] - table_count = 0 + """Fail closed until JR-table register/control dependence is proven. - for back in range(1, 41): - scan_addr = jr_pc - back * 4 - if scan_addr < entry: - break - word = _word_at(data, load_addr, scan_addr) - if word is None: - break - op = (word >> 26) & 0x3F - rs = (word >> 21) & 0x1F - rt = (word >> 16) & 0x1F - rd = (word >> 11) & 0x1F - fn = word & 0x3F - - if op == 0x23 and rt == jr_rs and lw_base is None: - lw_base = rs - lw_offset = word & 0xFFFF - if lw_offset & 0x8000: - lw_offset -= 0x10000 - continue - if op == 0x00 and fn == 0x21 and lw_base is not None and rd == lw_base \ - and addu_cand[0] is None: - addu_cand = [rs, rt] - continue - if op == 0x09 and addu_cand[0] is not None: - for i, cand in enumerate(addu_cand): - if cand is not None and not found_addiu[i] and rs == cand and rt == cand: - imm = word & 0xFFFF - if imm & 0x8000: - imm -= 0x10000 - addiu_val[i] = imm - found_addiu[i] = True - break - continue - if op == 0x0F and addu_cand[0] is not None and lui_val is None: - for i, cand in enumerate(addu_cand): - if cand is not None and rt == cand: - lui_val = (word & 0xFFFF) << 16 - addiu_val[0] = addiu_val[i] if found_addiu[i] else 0 - found_addiu[0] = found_addiu[i] - break - continue - if op == 0x0B and table_count == 0: - table_count = word & 0xFFFF - continue - if lui_val is not None and table_count: - break - - if lui_val is None or table_count == 0 or table_count >= 512: - return set() - - table_base = lui_val + (addiu_val[0] if found_addiu[0] else 0) + lw_offset - lo = load_addr - hi = load_addr + size - targets = set() - for i in range(table_count): - target = _word_at(data, load_addr, table_base + i * 4) - if target is None: - continue - if lo <= target < hi and entry <= target < hard_cap and (target & 3) == 0: - targets.add(target) - return targets + The former 40-word raw backscan could combine unrelated loads, constants, + and bounds checks across basic blocks into a synthetic table. + """ + return set() def _callable_legacy_seed(data: bytes, load_addr: int, addr: int) -> bool: @@ -643,6 +622,31 @@ def _resolve_constant_transfer(data: bytes, load_addr: int, low_is_addiu = op == 0x09 continue if op == 0x0F and rt == target_reg: + # A LUI in a control-transfer delay slot is not a local reaching + # definition for the following suffix. A callee may clobber the + # register before returning at LUI+4, while a branch/jump can leave + # the linear path entirely. Fail closed without path/call proof. + predecessor = _word_at(data, load_addr, pc - 4) + if pc >= entry + 4 and _is_control_flow(predecessor): + return None + + # Reject a syntactic constant pair when a direct control-flow edge + # can enter after this LUI. Example: `j low; lui; low: addiu; + # jalr` never executes the LUI, despite looking like a pair to a + # raw backward scan. + crossed_inbound_boundary = False + for source in range(entry, transfer_pc, 4): + source_word = _word_at(data, load_addr, source) + if source_word is None: + continue + source_kind, source_target = _classify_cf(source, source_word) + if (source_kind in ('branch', 'j', 'jal') and + pc < source_target <= transfer_pc): + crossed_inbound_boundary = True + break + if crossed_inbound_boundary: + return None + upper = (word & 0xFFFF) << 16 if low is None: return upper @@ -711,6 +715,11 @@ def plausible_callable_target(data: bytes, load_addr: int, size: int, if inside(branch_target): work.append(branch_target) # An out-of-probe branch target is a shared external code edge. + elif kind == 'branch_never': + add_delay(pc) + work.append(pc + 8) + elif kind == 'branch_never_likely': + work.append(pc + 8) elif kind in ('jal', 'jalr'): add_delay(pc) work.append(pc + 8) @@ -735,9 +744,6 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, lo = load_addr hi = load_addr + size - def in_function(addr: int) -> bool: - return lo <= addr < hi and entry <= addr < hard_cap and (addr & 3) == 0 - work = deque([entry]) visited = set() direct_jals = set() @@ -754,6 +760,15 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, return range_lo, range_hi return None + entry_producer = producer_for(entry) + + def in_function(addr: int) -> bool: + return (lo <= addr < hi and entry <= addr < hard_cap and + (addr & 3) == 0 and + (not producer_ranges or + (entry_producer is not None and + producer_for(addr) == entry_producer))) + def callable_transfer_target(source: int, target: int) -> bool: if _dense_local_pointer_table(data, load_addr, target): rejected_cross_producer_calls.add(target) @@ -762,6 +777,9 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, return True source_range = producer_for(source) target_range = producer_for(target) + if source_range is None or target_range is None: + rejected_cross_producer_calls.add(target) + return False if source_range is not None and source_range == target_range: return True if not allow_cross_producer_calls: @@ -811,6 +829,16 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, # Exact-entry partitioning hard-caps the current walk there, # but the target remains mechanically proven code reachability. forward_branch_targets.add(target) + elif kind == 'branch_never': + if in_function(delay): + visited.add(delay) + if in_function(pc + 8): + work.append(pc + 8) + branch_targets.add(pc + 8) + elif kind == 'branch_never_likely': + if in_function(pc + 8): + work.append(pc + 8) + branch_targets.add(pc + 8) elif kind == 'j': if in_function(delay): visited.add(delay) @@ -921,6 +949,12 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, for i in range(1, len(producer_ranges))): raise RuntimeError('producer_ranges overlap') + def capture_producer_for(addr: int): + for producer_lo, producer_hi in producer_ranges: + if producer_lo <= addr < producer_hi: + return producer_lo, producer_hi + return None + static_alias_ranges = set() allow_cross_producer_calls = not bool(cap.get('strict_producer_ranges')) for raw_alias in cap.get('static_alias_ranges', []) or []: @@ -940,6 +974,15 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, f'static alias 0x{entry:08X} -> ' f'0x{range_lo:08X}..0x{range_hi:08X} outside/misaligned for ' f'capture 0x{lo:08X}..0x{hi:08X}') + if producer_ranges: + owners = [index for index, (producer_lo, producer_hi) in + enumerate(producer_ranges) + if (producer_lo <= range_lo < producer_hi and + producer_lo <= entry < producer_hi and + producer_lo <= range_hi - 1 < producer_hi)] + if not owners: + raise RuntimeError( + f'static alias 0x{entry:08X} crosses producer boundary') static_alias_ranges.add((entry, range_lo, range_hi)) schema_keys = {'executed_pcs', 'observed_pcs', 'dispatch_entry_pcs', @@ -995,6 +1038,12 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, if not region(addr): excluded[addr] = 'UNKNOWN' return + # Composite padding has bytes but no producer identity. A root there + # would otherwise get entry_producer=None and could walk through every + # disconnected zero-fill gap as though those gaps shared ownership. + if producer_ranges and capture_producer_for(addr) is None: + excluded[addr] = 'UNKNOWN' + return # Boundary gate: a dispatched-to PC is proof of *code reachability*, not # of a *callable function boundary*. A jr-driven jump-table case label # or a mid-function pointer target can be dispatched-to yet have no @@ -1058,86 +1107,141 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, for addr in legacy_callable_seeds: include(addr, 'FUNCTION_POINTER_TARGET') + # Current-capture alias recipes contribute only an interior ENTRY candidate; + # their historical host range is not trusted. Final exact reachability below + # must discover a current host or the entry is dropped to the interpreter. + for addr, _range_lo, _range_hi in static_alias_ranges: + if region(addr): + included[addr] = 'DISPATCH_INTERIOR' + # Walk roots: callable entries only. DISPATCH_INTERIOR addresses are NOT # roots — as roots they would hard-cap (truncate) the sibling walk that # owns them. known = {a for a, r in included.items() if r != 'DISPATCH_INTERIOR'} - # Establish range ownership before call-derived roots can change the - # partition. A direct-call target already visited by an existing root is an - # interior alias, not a sibling root: promoting it would hard-cap the host - # and can hide the remainder of a jump-table function. - initial_owned = set() initial_known = set(known) - initial_sorted = sorted(initial_known) - for index, entry in enumerate(initial_sorted): - hard_cap = initial_sorted[index + 1] if index + 1 < len(initial_sorted) else hi - walk = _walk_overlay_function( - data, load_addr, size, entry, hard_cap, producer_ranges, - allow_cross_producer_calls) - initial_owned.update(walk['visited']) - - pending = deque(sorted(known)) - processed = set() + derived_reasons = {} + promoted_roots = set() + suppressed_derived = set() + suppression_passes = 0 all_branch_targets = set() kernel_window = (load_addr & 0x1FFFFFFF) < 0x10000 while True: - while pending: - entry = pending.popleft() - if entry in processed: - continue - processed.add(entry) + # Recompute call/branch evidence from the CURRENT partition. Evidence is + # not monotonic: a new hard cap can make an old source PC unreachable. + # Dropping stale candidates here prevents an old speculative walk from + # resurrecting a target in the final shard. + discovery_round = 0 + seen_states = {} + state_history = [] + while True: + discovery_round += 1 + state = (tuple(sorted(known)), tuple(sorted(derived_reasons))) + cycle_start = seen_states.get(state) + if discovery_round > 64 or cycle_start is not None: + unstable = set() + if cycle_start is not None: + cycle_states = state_history[cycle_start:] + known_sets = [set(item[0]) for item in cycle_states] + derived_sets = [set(item[1]) for item in cycle_states] + if known_sets: + unstable |= set.union(*known_sets) - set.intersection(*known_sets) + if derived_sets: + unstable |= (set.union(*derived_sets) - + set.intersection(*derived_sets)) + unstable_text = ','.join(f'{addr:08X}' + for addr in sorted(unstable)[:12]) + unstable -= initial_known | promoted_roots + if unstable and suppression_passes < 64: + suppression_passes += 1 + suppressed_derived.update(unstable) + print(' WARNING: ownership discovery cycle; suppressing ' + f'{len(unstable)} unstable derived roots' + + (f' ({unstable_text})' if unstable_text else '')) + known = initial_known | promoted_roots + derived_reasons = {} + discovery_round = 0 + seen_states = {} + state_history = [] + continue + print(' WARNING: ownership discovery did not converge; ' + 'falling back to explicit/dispatch roots' + + (f' (cycle={len(state_history) - cycle_start}, ' + f'unstable={unstable_text})' + if cycle_start is not None else ' (round limit)')) + known = initial_known | promoted_roots + derived_reasons = {} + break + seen_states[state] = len(state_history) + state_history.append(state) + next_reasons = {} + round_branch_targets = set() sorted_known = sorted(known) - hard_cap = next((x for x in sorted_known if x > entry), hi) - walk = _walk_overlay_function( - data, load_addr, size, entry, hard_cap, producer_ranges, - allow_cross_producer_calls) - all_branch_targets.update(walk['branch_targets']) - all_branch_targets.update(walk['jump_table_targets']) - for target in sorted(walk['direct_jals']): - if target not in initial_known and target in initial_owned: - included[target] = 'DISPATCH_INTERIOR' + for index, entry in enumerate(sorted_known): + hard_cap = (sorted_known[index + 1] + if index + 1 < len(sorted_known) else hi) + walk = _walk_overlay_function( + data, load_addr, size, entry, hard_cap, producer_ranges, + allow_cross_producer_calls) + round_branch_targets.update(walk['branch_targets']) + round_branch_targets.update(walk['jump_table_targets']) + for target in walk['direct_jals']: + if (target not in initial_known and + target not in suppressed_derived): + next_reasons[target] = 'DIRECT_JAL_TARGET' + for target in walk['static_indirect_targets']: + if (target not in initial_known and + target not in suppressed_derived): + next_reasons.setdefault(target, 'STATIC_INDIRECT_TARGET') + for target in walk['forward_branch_targets']: + source_reason = (included.get(entry) or + derived_reasons.get(entry) or + ('DISPATCH_ROOT' + if entry in promoted_roots else None)) + strong_source = source_reason in ( + 'DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET', + 'STATIC_BRANCH_ROOT', 'STATIC_DISCOVERY_ROOT', + 'TOML_DECLARED_ENTRY', 'DISPATCH_ROOT') + if not strong_source and not _callable_legacy_seed( + data, load_addr, entry): + continue + source_range = next( + ((rlo, rhi) for rlo, rhi in producer_ranges + if rlo <= entry < rhi), None) + target_range = next( + ((rlo, rhi) for rlo, rhi in producer_ranges + if rlo <= target < rhi), None) + if producer_ranges and source_range != target_range: + continue + producer_hi = target_range[1] if target_range else hi + if plausible_callable_target( + data, load_addr, size, target, producer_hi): + if (target not in initial_known and + target not in suppressed_derived): + next_reasons.setdefault(target, 'STATIC_BRANCH_ROOT') + + normalized = initial_known | promoted_roots | set(next_reasons) + roots = sorted(normalized) + for target in sorted(next_reasons): + if target not in normalized: continue - # Always re-include: a target may already be present as a - # heuristic FUNCTION_POINTER_TARGET. Reachable JAL evidence - # upgrades it to a trusted call root. - include(target, 'DIRECT_JAL_TARGET') - if target in included and target not in known: - known.add(target) - pending.append(target) - for target in sorted(walk['static_indirect_targets']): - if target not in initial_known and target in initial_owned: - included[target] = 'DISPATCH_INTERIOR' + index = bisect_left(roots, target) + if index == len(roots) or roots[index] != target or index == 0: continue - include(target, 'STATIC_INDIRECT_TARGET') - if target in included and target not in known: - known.add(target) - pending.append(target) - for target in sorted(walk['forward_branch_targets']): - source_reason = included.get(entry) - strong_source = source_reason in ( - 'DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET', - 'STATIC_BRANCH_ROOT', 'STATIC_DISCOVERY_ROOT', - 'TOML_DECLARED_ENTRY', 'DISPATCH_ROOT') - if not strong_source and not _callable_legacy_seed( - data, load_addr, entry): - continue - source_range = next( - ((lo, hi) for lo, hi in producer_ranges - if lo <= entry < hi), None) - target_range = next( - ((lo, hi) for lo, hi in producer_ranges - if lo <= target < hi), None) - if producer_ranges and source_range != target_range: - continue - producer_hi = target_range[1] if target_range else hi - if not plausible_callable_target( - data, load_addr, size, target, producer_hi): - continue - include(target, 'STATIC_BRANCH_ROOT') - if target in included and target not in known: - known.add(target) - pending.append(target) + owner = roots[index - 1] + hard_cap = roots[index + 1] if index + 1 < len(roots) else hi + owner_walk = _walk_overlay_function( + data, load_addr, size, owner, hard_cap, producer_ranges, + allow_cross_producer_calls) + if target in owner_walk['visited']: + normalized.remove(target) + roots.pop(index) + + if normalized == known and next_reasons == derived_reasons: + all_branch_targets = round_branch_targets + break + known = normalized + derived_reasons = next_reasons if not kernel_window: break @@ -1165,14 +1269,15 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, # that split one real function and hard-cap each other. a = promoted[0] included[a] = 'DISPATCH_ROOT' + promoted_roots.add(a) known.add(a) - pending.append(a) # Re-walk with the final function set so the branch-target exclusion count # matches the actual compilation boundaries. all_branch_targets.clear() rejected_cross_producer_calls = set() accepted_cross_producer_calls = set() + final_covered = set() sorted_known = sorted(known) for i, entry in enumerate(sorted_known): hard_cap = sorted_known[i + 1] if i + 1 < len(sorted_known) else hi @@ -1182,11 +1287,30 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, all_branch_targets.update(walk['branch_targets']) all_branch_targets.update(walk['jump_table_targets']) all_branch_targets.update(walk['forward_branch_targets']) + final_covered.update(walk['visited']) rejected_cross_producer_calls.update( walk['rejected_cross_producer_calls']) accepted_cross_producer_calls.update( walk['accepted_cross_producer_calls']) + # A DISPATCH_INTERIOR is emitted only when the final partition has a real + # CFG host for that exact PC. Reconsider every derived target after all + # later roots have settled; stale ownership falls back to a root (when it + # remains call-proven) or to the interpreter, never a hostless alias. + for addr, reason in sorted(derived_reasons.items()): + if addr in known: + include(addr, reason) + elif addr in final_covered: + included[addr] = 'DISPATCH_INTERIOR' + else: + included.pop(addr, None) + excluded[addr] = 'OBSERVED_PC_ONLY' + for addr, reason in list(included.items()): + if reason == 'DISPATCH_INTERIOR' and addr not in final_covered: + del included[addr] + excluded[addr] = ('OBSERVED_PC_ONLY' + if addr in executed_pcs else 'UNKNOWN') + candidates = {a for a in (executed_pcs | dispatch_entry_pcs | captured_function_entries | static_discovery_entries | legacy_seeds | toml_entries) @@ -1244,12 +1368,6 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, seeds.append(f'producer_range 0x{range_lo:08X} 0x{range_hi:08X}') for addr in sorted(accepted_cross_producer_calls): seeds.append(f'cross_call_allow 0x{addr:08X}') - retained_aliases = static_alias_ranges | { - (int(addr), int(range_lo), int(range_hi)) - for addr, range_lo, range_hi in cap.get('_prior_aliases', [])} - for addr, range_lo, range_hi in sorted(retained_aliases): - seeds.append( - f'retained_alias 0x{addr:08X} 0x{range_lo:08X} 0x{range_hi:08X}') seeds.extend(seed_line(addr) for addr in sorted(included)) return seeds, audit @@ -1805,23 +1923,60 @@ def parse_overlay_func_ids(src_path: str, data: bytes, load_addr: int, continue ev = (entry & 0x1FFFFFFF) | 0x80000000 out.append((ev, crc & 0xFFFFFFFF, ranges)) + return out -def write_overlay_ranges_from(func_ids: list, out_path: str) -> int: +def overlay_ranges_text(func_ids: list, pair_id: int | None = None) -> str: + """Serialize one loader manifest. + + ``P`` is an optional DLL/manifest publication identity. Old runtimes + ignore the unfamiliar record, while pair-aware runtimes require a matching + ``overlay_pair_id`` export whenever it is present. + """ + out_lines = ['# psxrecomp overlay code-range manifest v2 (entry+code_crc)\n'] + if pair_id is not None: + out_lines.append(f'P {pair_id & 0xFFFFFFFFFFFFFFFF:016X}\n') + for ev, crc, ranges in func_ids: + out_lines.append(f'F {ev:08X} {crc & 0xFFFFFFFF:08X}\n') + for lo, length in ranges: + out_lines.append( + f'R {(lo & 0x1FFFFFFF) | 0x80000000:08X} {length:X}\n') + return ''.join(out_lines) + + +def overlay_pair_id(src: str, func_ids: list) -> int: + """Bind a newly compiled DLL to the exact C and range manifest it uses.""" + digest = hashlib.sha256() + digest.update(b'psxrecomp overlay pair v1\0') + digest.update(src.encode('utf-8')) + digest.update(b'\0') + digest.update(overlay_ranges_text(func_ids).encode('ascii')) + return int.from_bytes(digest.digest()[:8], 'big') + + +def add_overlay_pair_export(src: str, pair_id: int) -> str: + """Add the optional v2-pair binding export without changing the shard ABI.""" + return src + f'''\n +#ifdef _WIN32 +__declspec(dllexport) +#else +__attribute__((visibility("default"))) +#endif +uint64_t overlay_pair_id(void) {{ return UINT64_C(0x{pair_id:016X}); }} +''' + + +def write_overlay_ranges_from(func_ids: list, out_path: str, + pair_id: int | None = None) -> int: """Write the {phys}_{key}.ranges manifest (v2) from a func-id list produced by parse_overlay_func_ids. Returns the number of functions written. Manifest v2 line format: F one per function R one per coalesced code range""" - out_lines = ['# psxrecomp overlay code-range manifest v2 (entry+code_crc)\n'] - for ev, crc, ranges in func_ids: - out_lines.append(f'F {ev:08X} {crc & 0xFFFFFFFF:08X}\n') - for lo, length in ranges: - out_lines.append(f'R {(lo & 0x1FFFFFFF) | 0x80000000:08X} {length:X}\n') with open(out_path, 'w') as f: - f.writelines(out_lines) + f.write(overlay_ranges_text(func_ids, pair_id)) return len(func_ids) @@ -1833,7 +1988,8 @@ def write_overlay_ranges(src_path: str, out_path: str, parse_overlay_func_ids(src_path, data, load_addr, size), out_path) -def load_region_coverage(cache_dir: str, phys_addr: int) -> set: +def load_region_coverage(cache_dir: str, phys_addr: int, + expected_abi: int | None = None) -> set: """Set of (ev, code_crc) function identities already provided by built DLLs for this region_start. The loader content-matches per function across ALL DLLs sharing a region_start, so a function is "covered" as soon as ANY @@ -1849,15 +2005,30 @@ def load_region_coverage(cache_dir: str, phys_addr: int) -> set: for name in names: if not (name.startswith(prefix) and name.endswith('.ranges')): continue + ranges_path = os.path.join(cache_dir, name) + dll_path = os.path.splitext(ranges_path)[0] + overlay_ext() try: - with open(os.path.join(cache_dir, name)) as f: - for ln in f: - p = ln.split() - if len(p) >= 3 and p[0] == 'F': - try: - covered.add((int(p[1], 16), int(p[2], 16))) - except ValueError: - pass + # Keep recovery, DLL validation, and the entire manifest snapshot + # under one OS lock. Otherwise a publisher can remove/replace one + # half between the existence check and F-record parsing. + with _shard_pair_lock(dll_path): + _recover_shard_pair_locked(dll_path) + if (not os.path.isfile(dll_path) or + os.path.getsize(dll_path) == 0 or + not os.path.isfile(ranges_path) or + os.path.getsize(ranges_path) == 0): + continue + if (expected_abi is not None and + not _dll_abi_matches(dll_path, expected_abi)): + continue + with open(ranges_path) as f: + for ln in f: + p = ln.split() + if len(p) >= 3 and p[0] == 'F': + try: + covered.add((int(p[1], 16), int(p[2], 16))) + except ValueError: + pass except OSError: pass return covered @@ -1874,7 +2045,8 @@ def _addr_in_func_ids(addr: int, func_ids: list) -> bool: return False -def load_region_entry_set(cache_dir: str, phys_addr: int) -> set: +def load_region_entry_set(cache_dir: str, phys_addr: int, + expected_abi: int | None = None) -> set: """Set of phys-normalized F-line ENTRY addresses provided by ALL built DLLs (region + fragment) for this region_start. This — not range coverage — is the dispatchability test: native code is enterable ONLY at F entries, so a @@ -1892,15 +2064,27 @@ def load_region_entry_set(cache_dir: str, phys_addr: int) -> set: for name in names: if not (name.startswith(prefix) and name.endswith('.ranges')): continue + ranges_path = os.path.join(cache_dir, name) + dll_path = os.path.splitext(ranges_path)[0] + overlay_ext() try: - with open(os.path.join(cache_dir, name)) as f: - for ln in f: - p = ln.split() - if len(p) >= 2 and p[0] == 'F': - try: - out.add(int(p[1], 16) & 0x1FFFFFFF) - except ValueError: - pass + with _shard_pair_lock(dll_path): + _recover_shard_pair_locked(dll_path) + if (not os.path.isfile(dll_path) or + os.path.getsize(dll_path) == 0 or + not os.path.isfile(ranges_path) or + os.path.getsize(ranges_path) == 0): + continue + if (expected_abi is not None and + not _dll_abi_matches(dll_path, expected_abi)): + continue + with open(ranges_path) as f: + for ln in f: + p = ln.split() + if len(p) >= 2 and p[0] == 'F': + try: + out.add(int(p[1], 16) & 0x1FFFFFFF) + except ValueError: + pass except OSError: pass return out @@ -2107,6 +2291,8 @@ def compile_interior_fragment(interior: int, data: bytes, load_addr: int, frag_ids = parse_overlay_func_ids(ranges_src, data, load_addr, size) if not frag_ids: return None, 'no-func-ids (empty ranges manifest)' + pair_id = overlay_pair_id(src, frag_ids) + src = add_overlay_pair_export(src, pair_id) # Key the fragment DLL by its func-identity SET (dedup like a region # bundle); the loader keys DLLs by the region_start filename prefix and # content-matches each function, so a fragment is just another DLL for @@ -2115,7 +2301,9 @@ def compile_interior_fragment(interior: int, data: bytes, load_addr: int, struct.pack(' bool: - """Publish a shard only after its compiler has completed successfully. + compiler: str = 'gcc', tcc: str = 'tcc', + func_ids: list | None = None, + pair_id: int | None = None) -> bool: + """Publish a shard only after all of its artifacts are complete. GCC and tcc write their output incrementally. If the process is interrupted while targeting the final cache name, the next run sees that partial file - and used to skip it as if it were complete. Compile to a unique sibling and - atomically replace the final DLL instead; an existing good shard therefore - also survives a failed --force rebuild. + and used to skip it as if it were complete. Compile to a unique sibling. + + When ``func_ids`` is supplied, stage the range manifest too and publish the + two as one recoverable transaction. Moving the old DLL out of its loader- + visible name first means every interruption point contains either a complete + old pair, a complete new pair, or no loadable pair -- never mixed halves. + The pair ID additionally lets a pair-aware loader reject the only filesystem + TOCTOU left: an old manifest parsed immediately before the DLL rename. """ final_out = os.path.abspath(out_dll) out_dir = os.path.dirname(final_out) @@ -2342,6 +2537,7 @@ def compile_dll(c_path: str, out_dll: str, include_dirs: list[str], dir=out_dir) os.close(fd) os.unlink(staged) # let the native compiler create its output normally + staged_ranges = os.path.splitext(staged)[0] + '.ranges' try: if not _compile_dll_direct( c_path, staged, include_dirs, gcc=gcc, flavor=flavor, @@ -2350,25 +2546,278 @@ def compile_dll(c_path: str, out_dll: str, include_dirs: list[str], if not os.path.isfile(staged) or os.path.getsize(staged) == 0: print(' COMPILE ERROR: compiler reported success but emitted no DLL') return False - os.replace(staged, final_out) + if func_ids is None: + os.replace(staged, final_out) + return True + if not func_ids or pair_id is None: + print(' COMPILE ERROR: paired shard requires non-empty ranges and pair ID') + return False + write_overlay_ranges_from(func_ids, staged_ranges, pair_id) + if not os.path.isfile(staged_ranges) or os.path.getsize(staged_ranges) == 0: + print(' COMPILE ERROR: range manifest staging produced no output') + return False + try: + publish_shard_pair(staged, staged_ranges, final_out) + except Exception as exc: + print(f' COMPILE ERROR: could not publish DLL/ranges pair: {exc}') + try: + recover_shard_pair(final_out) + except Exception as recover_exc: + print(f' COMPILE ERROR: shard-pair recovery also failed: ' + f'{recover_exc}') + return False return True finally: # TinyCC emits an export-definition sidecar beside its requested DLL. # It inherits the unique staged basename and must not accumulate in the # content cache after either success or failure. staged_sidecars = [os.path.splitext(staged)[0] + '.def'] - for temporary in [staged, *staged_sidecars]: + for temporary in [staged, staged_ranges, *staged_sidecars]: try: os.unlink(temporary) except FileNotFoundError: pass -def compiled_shard_complete(dll_path: str) -> bool: - """A cached shard is dispatchable only with its non-empty range manifest.""" +def _shard_pair_paths(dll_path: str) -> tuple[str, str, str, str]: + dll_path = os.path.abspath(dll_path) + ranges_path = os.path.splitext(dll_path)[0] + '.ranges' + return (dll_path, ranges_path, + dll_path + '.pair-lock', dll_path + '.pair-txn.json') + + +def _unlink_if_present(path: str) -> None: + try: + os.unlink(path) + except FileNotFoundError: + pass + + +def _best_effort_unlink(path: str) -> None: + """Remove transaction debris when possible (loaded Windows DLLs may refuse).""" + try: + os.unlink(path) + except OSError: + pass + + +def _file_sha256(path: str) -> str | None: + try: + digest = hashlib.sha256() + with open(path, 'rb') as source: + for chunk in iter(lambda: source.read(1024 * 1024), b''): + digest.update(chunk) + return digest.hexdigest() + except OSError: + return None + + +def _write_json_atomic(path: str, value: dict) -> None: + directory = os.path.dirname(path) + fd, staged = tempfile.mkstemp(prefix=f'.{os.path.basename(path)}.', + suffix='.tmp', dir=directory) + try: + with os.fdopen(fd, 'w', encoding='utf-8') as out: + json.dump(value, out, sort_keys=True) + out.flush() + os.fsync(out.fileno()) + os.replace(staged, path) + finally: + _best_effort_unlink(staged) + + +@contextmanager +def _shard_pair_lock(dll_path: str, timeout: float = 120.0): + """Cross-process lock for one canonical DLL/ranges filename pair. + + The lock file is intentionally permanent. OS byte-range/advisory locks are + released by the kernel when a process dies; deleting the file would create + an inode/handle split that permits two simultaneous owners. + """ + _dll, _ranges, lock_path, _journal = _shard_pair_paths(dll_path) + os.makedirs(os.path.dirname(lock_path), exist_ok=True) + lock_file = open(lock_path, 'a+b') + if os.path.getsize(lock_path) == 0: + lock_file.write(b'\0') + lock_file.flush() + deadline = time.monotonic() + timeout + acquired = False + try: + while not acquired: + try: + lock_file.seek(0) + if os.name == 'nt': + import msvcrt + msvcrt.locking(lock_file.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | + fcntl.LOCK_NB) + acquired = True + except OSError: + if time.monotonic() >= deadline: + raise TimeoutError( + f'timed out waiting for shard-pair lock {lock_path}') + time.sleep(0.025) + yield + finally: + if acquired: + lock_file.seek(0) + if os.name == 'nt': + import msvcrt + msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + lock_file.close() + + +def _cleanup_old_pair_debris(dll_path: str) -> None: + for path in glob.glob(os.path.abspath(dll_path) + '.pair-txn.*.old-*'): + _best_effort_unlink(path) + + +def _recover_shard_pair_locked(dll_path: str) -> None: + dll, ranges, _lock, journal_path = _shard_pair_paths(dll_path) + if not os.path.isfile(journal_path): + _cleanup_old_pair_debris(dll) + return + try: + with open(journal_path, encoding='utf-8') as source: + txn = json.load(source) + except (OSError, ValueError) as exc: + raise RuntimeError(f'cannot recover shard transaction {journal_path}: {exc}') + + required = ('backup_dll', 'backup_ranges', 'new_dll_sha256', + 'new_ranges_sha256', 'old_dll_sha256', 'old_ranges_sha256') + if txn.get('schema') != 'psxrecomp shard pair transaction v1' or any( + key not in txn for key in required): + raise RuntimeError(f'invalid shard transaction journal {journal_path}') + + backup_dll = txn['backup_dll'] + backup_ranges = txn['backup_ranges'] + token = txn.get('token', '') + if (not re.fullmatch(r'[0-9a-f]{32}', token) or + backup_dll != f'{dll}.pair-txn.{token}.old-dll' or + backup_ranges != f'{dll}.pair-txn.{token}.old-ranges'): + raise RuntimeError(f'invalid shard transaction ownership in {journal_path}') + new_complete = (_file_sha256(dll) == txn['new_dll_sha256'] and + _file_sha256(ranges) == txn['new_ranges_sha256']) + if new_complete: + # The DLL rename is the commit point. A crash before a journal phase + # update must not roll back a new DLL already paired with its exact + # manifest. Remove the journal first; old loaded-DLL backups are merely + # deferred debris and may remain locked until process exit on Windows. + _unlink_if_present(journal_path) + _best_effort_unlink(backup_dll) + _best_effort_unlink(backup_ranges) + return + + def restore(path: str, backup: str, expected_hash: str | None) -> None: + current_hash = _file_sha256(path) + if expected_hash is None: + if current_hash is not None: + _unlink_if_present(path) + return + if current_hash == expected_hash: + return + if _file_sha256(backup) != expected_hash: + raise RuntimeError( + f'cannot recover {path}: neither canonical nor backup is the old file') + if current_hash is not None: + _unlink_if_present(path) + os.replace(backup, path) + + # Restore the manifest first; restoring the DLL is the rollback commit. + restore(ranges, backup_ranges, txn['old_ranges_sha256']) + restore(dll, backup_dll, txn['old_dll_sha256']) + _unlink_if_present(journal_path) + _best_effort_unlink(backup_dll) + _best_effort_unlink(backup_ranges) + + +def recover_shard_pair(dll_path: str) -> None: + """Recover/finish a dead writer's transaction under its OS-owned lock.""" + with _shard_pair_lock(dll_path): + _recover_shard_pair_locked(dll_path) + + +def publish_shard_pair(staged_dll: str, staged_ranges: str, + final_dll: str) -> None: + """Commit staged DLL/ranges under an interprocess transaction lock.""" + dll, ranges, _lock, journal_path = _shard_pair_paths(final_dll) + token = uuid.uuid4().hex + backup_dll = f'{dll}.pair-txn.{token}.old-dll' + backup_ranges = f'{dll}.pair-txn.{token}.old-ranges' + + with _shard_pair_lock(dll): + _recover_shard_pair_locked(dll) + txn = { + 'schema': 'psxrecomp shard pair transaction v1', + 'token': token, + 'backup_dll': backup_dll, + 'backup_ranges': backup_ranges, + 'old_dll_sha256': _file_sha256(dll), + 'old_ranges_sha256': _file_sha256(ranges), + 'new_dll_sha256': _file_sha256(staged_dll), + 'new_ranges_sha256': _file_sha256(staged_ranges), + } + if txn['new_dll_sha256'] is None or txn['new_ranges_sha256'] is None: + raise RuntimeError('staged DLL/ranges disappeared before publication') + _write_json_atomic(journal_path, txn) + try: + if txn['old_dll_sha256'] is not None: + os.replace(dll, backup_dll) + if txn['old_ranges_sha256'] is not None: + os.replace(ranges, backup_ranges) + + # The new manifest alone is inert. Publishing the DLL is the commit. + os.replace(staged_ranges, ranges) + os.replace(staged_dll, dll) + _recover_shard_pair_locked(dll) # recognizes committed hashes + except BaseException: + _recover_shard_pair_locked(dll) + raise + + +def _dll_abi_matches(dll_path: str, expected_abi: int) -> bool: + """Read overlay_abi from one loaded image, then release it immediately.""" + import _ctypes + import ctypes + library = None + try: + library = (ctypes.WinDLL(dll_path) if os.name == 'nt' + else ctypes.CDLL(dll_path)) + abi_fn = library.overlay_abi + abi_fn.argtypes = [] + abi_fn.restype = ctypes.c_int + return (abi_fn() & 0xFFFFFFFF) == (expected_abi & 0xFFFFFFFF) + except (OSError, AttributeError): + return False + finally: + if library is not None and library._handle: + handle = library._handle + library._handle = 0 + if os.name == 'nt': + _ctypes.FreeLibrary(handle) + else: + _ctypes.dlclose(handle) + + +def compiled_shard_complete(dll_path: str, + expected_abi: int | None = None) -> bool: + """Check pair completeness and optional ABI/flavor under one OS lock.""" ranges = os.path.splitext(dll_path)[0] + '.ranges' - return (os.path.isfile(dll_path) and os.path.getsize(dll_path) > 0 and - os.path.isfile(ranges) and os.path.getsize(ranges) > 0) + with _shard_pair_lock(dll_path): + _recover_shard_pair_locked(dll_path) + complete = (os.path.isfile(dll_path) and + os.path.getsize(dll_path) > 0 and + os.path.isfile(ranges) and + os.path.getsize(ranges) > 0) + if not complete: + return False + return (expected_abi is None or + _dll_abi_matches(dll_path, expected_abi)) # --------------------------------------------------------------------------- @@ -2578,22 +3027,15 @@ def main(): static_entry_sources[entry] = ( data, load_addr, size, phys_addr) - # Merge evidence from a prior build of the SAME bytes: every F entry - # in an existing ranges manifest for this exact image was proven - # compilable before, so a fresh (poorer) capture can't regress - # coverage on rebuild. Callable entries re-enter as captured function - # entries (walk-root eligible); non-callable ones (alias wrappers - # from a prior build) re-enter as dispatch entries so the classifier - # re-derives their interior/alias disposition — feeding them back as - # roots would truncate their hosts. Their prior R ranges additionally - # reconstruct overlapping alias groups through retained_alias lines, so - # stronger new roots and old indirect dispatch entries can coexist. + # Reclassify prior F entry addresses from an identical image. Callable + # entries may become current roots; other entries re-enter as dispatch + # candidates and require a current exact host. Never reuse prior R host + # ranges: later roots can repartition the same bytes incompatibly. if not args.static: ranges_name = f'{phys_addr:08X}_{crc32:08X}.ranges' prior_ranges_path = os.path.join(cache_dir, ranges_name) if os.path.exists(prior_ranges_path): prior_entries = [] - prior_func_ranges = [] with open(prior_ranges_path) as pf: for ln in pf: parts = ln.split() @@ -2602,12 +3044,6 @@ def main(): prior_entries.append(int(parts[1], 16)) except (IndexError, ValueError): pass - elif parts and parts[0] == 'R': - try: - prior_func_ranges.append( - (int(parts[1], 16), int(parts[2], 16))) - except (IndexError, ValueError): - pass if prior_entries: fe = _parse_addr_list(cap.get('function_entry_pcs', [])) de = _parse_addr_list(cap.get('dispatch_entry_pcs', [])) @@ -2618,23 +3054,6 @@ def main(): de.add(a) cap['function_entry_pcs'] = sorted(fe) cap['dispatch_entry_pcs'] = sorted(de) - alias_groups = {} - if len(prior_entries) != len(prior_func_ranges): - print(f' WARNING: malformed prior manifest has ' - f'{len(prior_entries)} F lines and ' - f'{len(prior_func_ranges)} R lines; alias groups ' - f'will not be retained') - else: - for entry, (range_lo, range_size) in zip( - prior_entries, prior_func_ranges): - alias_groups.setdefault( - (range_lo, range_lo + range_size), []).append(entry) - cap['_prior_aliases'] = [ - (entry, range_lo, range_hi) - for (range_lo, range_hi), entries in alias_groups.items() - if len(entries) > 1 or any(e != range_lo for e in entries) - for entry in entries - ] cap.setdefault('schema', 'merged') print(f' merged {len(prior_entries)} prior-manifest entries ' f'from {prior_ranges_path}') @@ -2677,7 +3096,8 @@ def main(): stats.add_skip() return - if (not args.static and compiled_shard_complete(dll_path) and + if (not args.static and compiled_shard_complete( + dll_path, overlay_abi_tag(args.runtime_include, args.flavor)) and not args.force): print(' SKIP: DLL already exists (use --force to recompile)\n') stats.add_skip() @@ -2853,7 +3273,9 @@ def main(): with cov_lock: covered = region_coverage_cache.get(phys_addr) if covered is None: - covered = load_region_coverage(cache_dir, phys_addr) + covered = load_region_coverage( + cache_dir, phys_addr, + overlay_abi_tag(args.runtime_include, args.flavor)) region_coverage_cache[phys_addr] = covered fully_covered = (bool(this_set) and this_set <= covered and not args.force @@ -2866,6 +3288,19 @@ def main(): stats.add_skip() return + if not this_ids: + print(' WARNING: recompiler emitted no usable _full.ranges -- ' + 'preserving any prior DLL/ranges pair and leaving this ' + 'region to the interpreter') + stats.add_fail(_label, 'no_ranges', + 'no usable function identities (DLL not built)') + return + pair_id = overlay_pair_id(src, this_ids) + src = add_overlay_pair_export(src, pair_id) + # Retained source is the exact source compiled into the DLL. + with open(patched_c, 'w') as f: + f.write(src) + # Compile to DLL include_dirs = [args.runtime_include] recomp_root = os.path.dirname(os.path.dirname(args.recompiler)) @@ -2876,14 +3311,16 @@ def main(): success = compile_dll(patched_c, dll_path, include_dirs, gcc=args.gcc, flavor=args.flavor, - compiler=args.compiler, tcc=args.tcc) + compiler=args.compiler, tcc=args.tcc, + func_ids=this_ids, + pair_id=pair_id) if success: - # Emit the per-entry code-range manifest beside the DLL from - # the same func-id list we keyed the dedup on. The loader keys - # it by the same filename stem with .ranges (replacing .dll). + # compile_dll transactionally published the per-entry range + # manifest beside the DLL from the same func-id list used by + # dedup. The loader keys it by the same filename stem. ranges_out = os.path.splitext(dll_path)[0] + '.ranges' if this_ids: - nfn = write_overlay_ranges_from(this_ids, ranges_out) + nfn = len(this_ids) update_bios_resident_marker(dll_path, cap) print(f' ranges: {nfn} functions -> {ranges_out}') # New identities are now available for this region_start; @@ -2951,7 +3388,9 @@ def main(): # range" does NOT make a dispatch target servable — a range- # covered PC with no F entry anywhere still interps its whole # chain on every dispatch. Demand an entry at exactly this PC. - covered_entries = load_region_entry_set(cache_dir, phys_addr) + covered_entries = load_region_entry_set( + cache_dir, phys_addr, + overlay_abi_tag(args.runtime_include, args.flavor)) orphans = sorted(a for a in interior_pcs if (a in executed or a in forced_interiors) and (a & 0x1FFFFFFF) not in covered_entries) From f9d0e9d07cba031941fae896af92d2659c1c56e0 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 19:00:47 -0700 Subject: [PATCH 29/37] docs: refresh README for the multi-game, LLE-first architecture - Add a breadth-first -> depth/optimization pivot note at the top. - Rewrite "What It Is": LLE recompiled BIOS as foundation/oracle, with an opt-in HLE tier, overlay capture-and-compile, and the interpreter framed as a transient safety net the TCC sharding pipeline compiles away. - Add a Games table linking all seven public repos + latest releases (Tomba, Tomba 2, Ape Escape, Mega Man X4/X5/X6, Tsumu Light). - Replace the Tomba-only Status with a framework-wide subsystem view and depth-phase fronts. - Reinforce that this repo ships a BIOS-only runtime; note the games' in-app Dear ImGui launcher. - Trim keyboard/controller map tables (game-facing) down to a short Input note. - Replace the antiquated Disc Speed section with authentic-1x-default plus opt-in load acceleration (turbo, warm CD routes). Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 199 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 111 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index be784b4d..5e467dab 100644 --- a/README.md +++ b/README.md @@ -12,20 +12,47 @@ Background on the original prototype: [![PSXRecomp demo](https://img.youtube.com/vi/CID9oVhgCyY/maxresdefault.jpg)](https://www.youtube.com/watch?v=CID9oVhgCyY) +> **Where the project is headed.** Development so far has been **breadth-first**: +> stand up as many games as possible and get them into a playable alpha, proving +> the framework generalizes. That phase has largely delivered — seven titles now +> run and ship public builds (see [Games](#games)). The project is now pivoting to +> a **depth / optimization** phase: pushing each game toward 100% static coverage, +> tightening timing accuracy, driving load times toward zero, and hardening the +> renderer and audio paths. Expect the fleet to get *faster and more accurate* +> rather than *wider* from here. + ## What It Is PSXRecomp translates PS1 MIPS binaries into C, then compiles that C as a native executable linked against a PS1 hardware runtime. The v4 architecture -recompiles the real SCPH1001 BIOS and runs it as the kernel. There is no HLE -BIOS layer, no stubs, and no general-purpose interpreter fallback for static -code. +recompiles the real `SCPH1001.BIN` BIOS and runs it as the kernel — that +**low-level (LLE) recompiled BIOS is the foundation and the correctness oracle.** +Everything is architected LLE-first: accuracy comes first, and convenience is +layered on top, opt-in, never underneath. -PSXRecomp is a framework. Game-specific projects live in their own +Three things sit on that foundation: + +- **An optional HLE tier.** A high-level BIOS layer can be laid over the + recompiled kernel to skip the boot sequence and service a few BIOS calls + directly — a player-facing convenience and optimization, enabled by default + but fully opt-out (`[runtime] bios_hle = false`). Anything it doesn't + implement falls straight through to the recompiled BIOS, so the LLE path stays + load-bearing and remains the oracle every accuracy check runs against. +- **Capture-and-compile for overlays.** PS1 games stream code off the disc at + runtime (*overlays*) that no ahead-of-time recompiler can see. PSXRecomp + captures each overlay the moment it loads and recompiles it to native code, + cached and reused forever after (`static → gcc → tcc` backend). +- **A general-purpose interpreter — as a transient safety net, not a fixture.** + Anything not yet native (a freshly streamed overlay, RAM-installed code) runs + in a small MIPS interpreter so the machine is always *correct*. But the + interpreter is explicitly meant to be **compiled away**: the same capture + feeds the TCC-backed sharding pipeline, which turns interpreted code into + cached native shards in the background. The more a game runs, the less the + interpreter is doing — the goal state is an idle interpreter and 100% native + execution. + +PSXRecomp is a **framework**. Game-specific projects live in their own repositories and link this one in as a **git submodule** to build a game binary. -The active end-to-end targets are: - -- [TombaRecomp](https://github.com/mstan/TombaRecomp) — *Tomba!* -- [MegaManX6Recomp](https://github.com/mstan/MegaManX6Recomp) — *Mega Man X6* **New here?** The fastest way in: [`docs/EXECUTION_MODEL.md`](docs/EXECUTION_MODEL.md) (how a game actually @@ -34,6 +61,26 @@ runs — static / native-overlay / interpreter), then [`docs/BUILDING.md`](docs/BUILDING.md), and [`CONTRIBUTING.md`](CONTRIBUTING.md). +## Games + +Each game is its own repository that pins a framework commit as a submodule and +ships its own release with an in-app launcher. All are **alpha** — playable, not +yet fully validated end to end. + +| Game | Repository | Latest build | Notes | +|---|---|---|---| +| *Tomba!* | [TombaRecomp](https://github.com/mstan/TombaRecomp) | [releases](https://github.com/mstan/TombaRecomp/releases/latest) | Most mature target; widescreen, supersampling, save/load. | +| *Tomba! 2* | [Tomba2Recomp](https://github.com/mstan/Tomba2Recomp) | [releases](https://github.com/mstan/Tomba2Recomp/releases/latest) | Multi-track disc support, selectable 21:9. | +| *Ape Escape* | [ApeEscapeRecomp](https://github.com/mstan/ApeEscapeRecomp) | [releases](https://github.com/mstan/ApeEscapeRecomp/releases/latest) | New launcher, widescreen, memory-card save/load. | +| *Mega Man X4* | [MegaManX4Recomp](https://github.com/mstan/MegaManX4Recomp) | [releases](https://github.com/mstan/MegaManX4Recomp/releases/latest) | Playable; widescreen. | +| *Mega Man X5* | [MegaManX5Recomp](https://github.com/mstan/MegaManX5Recomp) | [releases](https://github.com/mstan/MegaManX5Recomp/releases/latest) | Early bring-up. | +| *Mega Man X6* | [MegaManX6Recomp](https://github.com/mstan/MegaManX6Recomp) | [releases](https://github.com/mstan/MegaManX6Recomp/releases/latest) | Playable; stages, controller, save/load. | +| *Tsumu Light* | [TsumuLightRecomp](https://github.com/mstan/TsumuLightRecomp) | [releases](https://github.com/mstan/TsumuLightRecomp/releases/latest) | Japanese title (SLPS-02253); early bring-up. | + +Each game repo carries its own build/run instructions, keyboard/controller +mappings, and per-game settings. **This repository builds the framework and a +BIOS-only runtime** — see [Release Package](#release-package) below. + ## Philosophy — toward 100% static recompilation The goal is simple and absolute: **a PS1 game should run as native code, not be @@ -90,47 +137,44 @@ a game is played; this branch is where that machinery is being built. ## Status -Current milestone as of 2026-05-18: +The framework is at **alpha**: the LLE recompiled BIOS boots and hands off to +the game across all seven targets, and each runs as majority-native code with +the capture-and-compile pipeline filling overlays as they're reached. The +breadth-first push is essentially done; work now is depth and optimization. + +Core subsystems, framework-wide: | Subsystem | State | |---|---| -| BIOS recompilation (`SCPH1001.BIN`) | Boots and hands off to Tomba | -| Game EXE recompilation | Tomba title, OPTIONS, NEW GAME, save/load, and gameplay reached | -| CD-ROM / MDEC / XA | Tomba FMVs stream and play at the game's 15 fps cadence | -| Memory cards | Tomba save and load verified | -| SIO0 controllers | Digital pad polling plus DualShock config replies used by Tomba | -| GPU | Functional for BIOS boot, FMVs, menus, and first gameplay area | -| Interrupts, COP0, timers | Working for current Tomba path | -| Dirty-RAM support | BIOS/game RAM-installed dispatch paths handled | -| Controller input | Keyboard plus SDL/XInput-style controller mapping via `input.ini` | +| BIOS recompilation (`SCPH1001.BIN`) | Boots to shell and hands off to game; also the correctness oracle | +| HLE BIOS tier | Optional boot-skip + service layer over the recompiled BIOS; default on, opt-out | +| Game EXE recompilation | Title/menus/save-load/gameplay reached across the fleet | +| Overlay capture → compile → cache | `static → gcc → tcc`; coverage grows as a game is played | +| Interpreter failover | Correctness net for not-yet-native code; being compiled away by sharding | +| CD-ROM / MDEC / XA | FMVs stream and play; XA/CDDA timing stays authentic | +| Memory cards | Save and load verified on multiple titles | +| SIO0 controllers | Digital pad + DualShock config; per-game analog/digital selection | +| GPU | Software + OpenGL + Vulkan backends; widescreen FOV/HUD work per game | +| SPU | Working; reverb/noise/sweep and exact SPU-IRQ accuracy still being tightened | +| Interrupts, COP0, timers, GTE | Working; cycle-accuracy foundation is an active depth-phase focus | -Known follow-up work: +Per-game maturity varies — see the [Games](#games) table. None of the titles is +declared fully validated end to end; users perform the final playthrough +validation on each release. -- The recent Tomba visual burn-down fixed the BIOS PS logo, title/menu glyph - seams, dialog/pause panel seams, terrain shading, and shaded textured branch - rendering observed in the first area. -- SPU coverage is partial; reverb, noise, sweep, and accurate SPU IRQ behavior - are not complete. -- The historical Windows "Not Responding" hang is mitigated but should stay on - the watch list until longer in-game soak tests are clean. -- Neither game is fully validated end to end yet. Tomba has considerably more - coverage; Mega Man X6 is playable (stages, controller, and memory-card - save/load all work). Full playthroughs of both are still unverified. +Open depth-phase fronts: cycle/IRQ-phase timing accuracy, load-time-toward-zero +(data sharding), renderer parity between the software/GL/Vulkan backends, and +driving overlay coverage the last mile to 100% static. -For the current game milestone, build and run the sibling TombaRecomp project: - -```sh -cd ../TombaRecomp -cmake --build build -j16 -./build/psx-runtime.exe --game game.toml -``` - -Running this repository's runtime without `--game` is still useful for -BIOS-only memory card management. +Running this repository's runtime without a game is useful for **BIOS-only +memory-card management**; to build and play a title, use its game repo from the +[Games](#games) table. ## Release Package -The framework release package is BIOS-only: +**This repository's release is BIOS-only** — it is the framework runtime, not a +game. Use it for BIOS boot and memory-card management; to play a title, grab its +release from the [Games](#games) table. 1. Download `PSXRecomp-v*-windows-x64.zip` from Releases. 2. Extract it and run `PSXRecomp.exe`. @@ -140,8 +184,11 @@ The package does not include a PS1 BIOS, game disc image, generated game code, or save data. The selected BIOS path is saved next to the executable as `bios.cfg`; delete that file to pick a different BIOS later. -Game-specific recomp projects, including TombaRecomp, use the same runtime -picker contract but also prompt for a legally obtained game disc image. +The game recomp projects use the same runtime picker contract but ship a +**Dear ImGui launcher**: on first run it prompts for your legally obtained BIOS +and game disc image, then lets you configure video, controls, and per-game +settings before launching. Keyboard/controller mappings live in each game's repo +and launcher, not here. ## Setup @@ -176,42 +223,13 @@ generated C is optimized. Game projects generate their own `generated/_*.c` files and link this runtime through CMake — see [`docs/BUILDING.md`](docs/BUILDING.md#build-and-run-a-game). -## Keyboard Map +## Input -| PSX button | Keyboard | -|---|---| -| D-Pad Up / Down / Left / Right | Arrow keys | -| Cross | X | -| Square | Z | -| Circle | S | -| Triangle | A | -| L1 / R1 | Q / W | -| L2 / R2 | E / R | -| L3 / R3 (stick clicks) | T / Y | -| Start | Enter | -| Select | Right Shift | -| Turbo | Tab (hold) | -| Fullscreen | F11 / Alt+Enter / Cmd+F | - -## Controller Map - -Xbox-style controller defaults are enabled when a controller is connected: - -| PSX button | Xbox controller | -|---|---| -| D-Pad Up / Down / Left / Right | D-pad or left stick | -| Cross | A | -| Circle | B | -| Square | X | -| Triangle | Y | -| L1 / R1 | LB / RB | -| L2 / R2 | LT / RT | -| L3 / R3 | Left / right stick click | -| Start | Menu | -| Select | View / Back | - -Release builds create/use `input.ini` next to the executable. Edit that file to -change controller device index, deadzone, or button mapping. +Keyboard and Xbox-style controller input work out of the box; the default +fullscreen toggle is F11 / Alt+Enter / Cmd+F. **Full button maps, controller +configuration, and rebinding live in each game's repo and in-app launcher** — +they're game-facing, not part of the framework. The BIOS-only framework runtime +accepts keyboard input for navigating the BIOS shell and memory-card tools. ## Architecture @@ -233,20 +251,25 @@ See [`CONTRIBUTING.md`](CONTRIBUTING.md) and [`CLAUDE.md`](CLAUDE.md) for the development rules, and [`docs/internal/`](docs/internal/) for the phased plans and deep design notes (`PLAN.md`, `FAITHFUL_TIMING_PLAN.md`, …). -## Disc Speed +## Load Times -Per-game `disc_speed` in `game.toml [runtime]` compresses CD-ROM timing: +The runtime models **authentic 1× CD-ROM timing by default** — the same read and +seek delays as real hardware. On top of that faithful baseline, load-time +acceleration is **opt-in**, per game, so the accurate path is never compromised: -| Value | Effect | -|---|---| -| `"1x"` | Authentic PSX timing. Default for all games. | -| `"2x"` / `"4x"` | 2× / 4× faster reads and seeks. | -| `"instant"` | Minimum-floor delays. **Known to hang some games** during early initialization; root cause not yet identified. Use `"4x"` instead until resolved. | +- **Turbo** — a hold-to-fast-forward key that compresses loads on demand. +- **`[runtime] turbo_loads` / `idle_skip`** — automatic acceleration during load + waits, with `turbo_audio_sink` keeping the SPU timeline coherent through the + burst. +- **Warm CD routes (`[[runtime.warm_cd_routes]]`)** — narrowly-scoped fast + read cadence armed on a specific `SetLoc`, restoring authentic timing the + moment the read pattern diverges. -FMV playback is always protected: the CD-ROM layer reverts to 1× whenever XA -audio streaming is active, regardless of `disc_speed`. The speed switch fires -only after the BIOS has handed off to the game EXE — boot and the license -screen always run at authentic 1×. +FMV/XA and CDDA streaming, seek, and motor timing always stay authentic +regardless of the accelerators. Driving load times toward zero (via data +sharding) is an active depth-phase effort — see +[`docs/LOAD_TIME_ZERO.md`](docs/LOAD_TIME_ZERO.md) and +[`docs/disc-speed.md`](docs/disc-speed.md). ## Help make your game faster — just by playing it From d73dc5f36e266ba84bd9db46c0dda79756ba2d39 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sun, 19 Jul 2026 13:40:03 -0700 Subject: [PATCH 30/37] runtime: add transactional overlay differential replay --- runtime/include/lockstep.h | 14 ++ runtime/include/psx_cycles.h | 14 ++ runtime/src/debug_server.c | 4 + runtime/src/dirty_ram_interp.c | 71 +++++++- runtime/src/gte.cpp | 68 +++++-- runtime/src/overlay_loader.c | 311 +++++++++++++++++++++++++++++++-- runtime/src/psx_cycles.c | 35 +++- runtime/src/psx_icache.c | 37 +++- 8 files changed, 513 insertions(+), 41 deletions(-) diff --git a/runtime/include/lockstep.h b/runtime/include/lockstep.h index 1229830f..775634b0 100644 --- a/runtime/include/lockstep.h +++ b/runtime/include/lockstep.h @@ -25,6 +25,20 @@ extern int g_ls_mode; * perturbs real global timing state. */ extern int g_ls_replay_active; +/* Side-effect-free whole-call trace used by the overlay differential gate. + * RECORD executes the interpreter authoritatively and records every memory + * operation. REPLAY feeds the recorded reads to native code and verifies + * writes without touching RAM, scratchpad, or MMIO a second time. These calls + * are deliberately single-owner/non-nestable; a false return fails the + * comparison closed. */ +int ls_shadow_record_begin(void); +int ls_shadow_record_end(uint32_t *ops, int *saw_exception); +int ls_shadow_replay_begin(void); +int ls_shadow_replay_end(uint32_t *ops, int *mismatch_kind, + uint32_t *pc, uint32_t *addr, + uint32_t *expected, uint32_t *actual); +void ls_shadow_abort(void); + /* Memory chokepoint hooks, called from the psx_read_ and psx_write_ funcs. */ uint32_t ls_read_hook(uint32_t addr, int size, uint32_t real_val); void ls_write_hook(uint32_t addr, int size, uint32_t val); diff --git a/runtime/include/psx_cycles.h b/runtime/include/psx_cycles.h index a13bab2b..be6506f0 100644 --- a/runtime/include/psx_cycles.h +++ b/runtime/include/psx_cycles.h @@ -20,6 +20,20 @@ extern uint64_t psx_cycle_count; * psx_cycles.c "Event-deadline device servicing"). */ void psx_advance_cycles(uint32_t cycles); +/* Diagnostic replay clock: advance the CPU-visible guest clock without + * servicing devices, then restore the authoritative live clock afterward. + * Used only while g_ls_replay_active is set. */ +int psx_cycle_replay_begin(uint64_t start_cycle); +uint64_t psx_cycle_replay_end(void); + +/* Transactional I-cache view for the overlay differential harness. The + * interpreter keeps the authoritative post-call cache; native replay starts + * from the saved entry tags, mutates a temporary view, then restores live. */ +int psx_icache_shadow_record_begin(void); +int psx_icache_shadow_replay_begin(void); +void psx_icache_shadow_replay_end(void); +void psx_icache_shadow_abort(void); + /* Event-deadline device model (psx_cycles.c): catch every device up to the * charged guest-cycle position and force a deadline recompute. memory.c calls * this at the top of every device-MMIO read/write so handlers always see diff --git a/runtime/src/debug_server.c b/runtime/src/debug_server.c index 437d17de..40f56722 100644 --- a/runtime/src/debug_server.c +++ b/runtime/src/debug_server.c @@ -2008,6 +2008,10 @@ static inline void cyc_watch_observe(uint32_t block_leader_phys); /* defined be volatile uint32_t g_psx_last_fn_entry = 0; void debug_server_log_call_entry(uint32_t func_addr) { + /* Whole-call native replay is diagnostic and must not double-consume or + * overwrite live trace/stack-watch state. Architectural CPU state is + * compared by the caller; these observers are intentionally inert. */ + { extern int g_ls_replay_active; if (g_ls_replay_active) return; } g_psx_last_fn_entry = func_addr; #ifdef PSX_STACK_GUARD g_psx_recent_fn[g_psx_recent_fn_i++ & (PSX_RECENT_FN_CAP - 1u)] = func_addr; diff --git a/runtime/src/dirty_ram_interp.c b/runtime/src/dirty_ram_interp.c index 6559f371..8cf93d7b 100644 --- a/runtime/src/dirty_ram_interp.c +++ b/runtime/src/dirty_ram_interp.c @@ -2580,6 +2580,8 @@ static int s_ls_trace_n = 0, s_ls_trace_idx = 0; static int s_ls_overflow = 0, s_ls_mismatch = 0; static int s_ls_replay_done = 0; /* trace exhausted: stop replay (benign count mismatch) */ static uint32_t s_ls_cur_pc = 0; /* pc of the instruction being replayed */ +static int s_ls_shadow_owner = 0; /* 0 none, 1 record, 2 replay */ +static int s_ls_shadow_saw_exception = 0; static CPUState s_ls_R0; /* register snapshot at block entry */ static uint32_t s_ls_block = 0; static int s_ls_prev_valid = 0; @@ -2634,6 +2636,71 @@ void ls_func_set_record_only(int on) { s_lsf_record_only = on; } void ls_note_exception_entry(void) { if (s_lsf_active) s_lsf_saw_irq = 1; + if (s_ls_shadow_owner == 1) s_ls_shadow_saw_exception = 1; +} + +int ls_shadow_record_begin(void) { + if (s_ls_shadow_owner || g_ls_mode != 0 || g_ls_replay_active || + s_lsf_active || s_ls_prev_valid) + return 0; + s_ls_trace_n = 0; + s_ls_trace_idx = 0; + s_ls_overflow = 0; + s_ls_mismatch = 0; + s_ls_m_kind = 0; + s_ls_replay_done = 0; + s_ls_shadow_saw_exception = 0; + s_ls_shadow_owner = 1; + g_ls_mode = 1; + return 1; +} + +int ls_shadow_record_end(uint32_t *ops, int *saw_exception) { + if (s_ls_shadow_owner != 1) return 0; + g_ls_mode = 0; + s_ls_shadow_owner = 0; + if (ops) *ops = (uint32_t)s_ls_trace_n; + if (saw_exception) *saw_exception = s_ls_shadow_saw_exception; + return !s_ls_overflow; +} + +int ls_shadow_replay_begin(void) { + if (s_ls_shadow_owner || g_ls_mode != 0 || g_ls_replay_active || + s_ls_overflow) + return 0; + s_ls_trace_idx = 0; + s_ls_mismatch = 0; + s_ls_m_kind = 0; + s_ls_replay_done = 0; + s_ls_shadow_owner = 2; + g_ls_replay_active = 1; + g_ls_mode = 2; + return 1; +} + +int ls_shadow_replay_end(uint32_t *ops, int *mismatch_kind, + uint32_t *pc, uint32_t *addr, + uint32_t *expected, uint32_t *actual) { + if (s_ls_shadow_owner != 2) return 0; + int complete = !s_ls_mismatch && !s_ls_replay_done && + s_ls_trace_idx == s_ls_trace_n; + g_ls_mode = 0; + g_ls_replay_active = 0; + s_ls_shadow_owner = 0; + if (ops) *ops = (uint32_t)s_ls_trace_idx; + if (mismatch_kind) *mismatch_kind = s_ls_m_kind; + if (pc) *pc = s_ls_m_pc; + if (addr) *addr = s_ls_m_addr; + if (expected) *expected = s_ls_m_exp; + if (actual) *actual = s_ls_m_act; + return complete; +} + +void ls_shadow_abort(void) { + g_ls_mode = 0; + g_ls_replay_active = 0; + s_ls_shadow_owner = 0; + s_ls_shadow_saw_exception = 0; } void ls_suppress_begin(void) { @@ -2648,7 +2715,7 @@ uint32_t ls_read_hook(uint32_t addr, int size, uint32_t real_val) { if (g_ls_mode == 2) { /* replay: serve from trace */ if (s_ls_trace_idx >= s_ls_trace_n) { s_ls_replay_done = 1; return 0; } /* count mismatch: benign block-boundary artifact */ ls_op_t *o = &s_ls_trace[s_ls_trace_idx]; - if (o->is_write || o->addr != addr) { + if (o->is_write || o->size != (uint8_t)size || o->addr != addr) { if (!s_ls_mismatch) { s_ls_mismatch = 1; s_ls_m_kind = 1; s_ls_m_pc = s_ls_cur_pc; s_ls_m_addr = addr; s_ls_m_exp = addr; s_ls_m_act = o->addr; } return 0; @@ -2669,7 +2736,7 @@ void ls_write_hook(uint32_t addr, int size, uint32_t val) { if (g_ls_mode == 2) { /* replay: verify against trace */ if (s_ls_trace_idx >= s_ls_trace_n) { s_ls_replay_done = 1; return; } /* count mismatch: benign block-boundary artifact */ ls_op_t *o = &s_ls_trace[s_ls_trace_idx]; - if (!o->is_write || o->addr != addr) { + if (!o->is_write || o->size != (uint8_t)size || o->addr != addr) { if (!s_ls_mismatch) { s_ls_mismatch = 1; s_ls_m_kind = 2; s_ls_m_pc = s_ls_cur_pc; s_ls_m_addr = addr; s_ls_m_exp = addr; s_ls_m_act = o->addr; } return; diff --git a/runtime/src/gte.cpp b/runtime/src/gte.cpp index b6471ac6..e3b5492a 100644 --- a/runtime/src/gte.cpp +++ b/runtime/src/gte.cpp @@ -185,6 +185,7 @@ static void depth_cue_from_ir(GTEState* gte, uint32_t instr) { static int32_t s_ws_xnum = 1, s_ws_xden = 1; extern "C" int gpu_ws_present_native_43(void); /* gpu.c — suppress on 4:3 frames */ extern "C" void psx_ws_note_gte_project(int nverts); /* gpu.c — gte_game_mode stamp */ +static int s_gte_replay_sandbox = 0; /* Optional visual-only geometry precision cache. The PS1-visible SXY FIFO * remains integer and fully faithful; this side cache retains the discarded @@ -243,6 +244,7 @@ extern "C" void gte_precision_tracking_set(int enabled) { } extern "C" void gte_precision_store_word(uint32_t addr, uint8_t reg) { + if (s_gte_replay_sandbox) return; if (!s_precision_tracking || reg < 12 || reg > 15) return; int index = reg == 15 ? 2 : (int)reg - 12; const PreciseProjection &projection = s_precise_sxy[index]; @@ -256,6 +258,7 @@ extern "C" void gte_precision_store_word(uint32_t addr, uint8_t reg) { } extern "C" void gte_precision_invalidate_word(uint32_t addr) { + if (s_gte_replay_sandbox) return; if (!s_precision_tracking) return; uint32_t physical; if (!precision_ram_address(addr, &physical)) return; @@ -312,6 +315,7 @@ extern "C" int gte_geometry_correction_lookup(uint32_t packed, } static inline void geom_note(uint32_t packed, int64_t x16, int64_t y16) { + if (s_gte_replay_sandbox) return; if (!s_geom_enabled) return; /* Saturated off-screen projections are unsuitable for subpixel recovery. */ int32_t x = (int16_t)(packed & 0xFFFFu); @@ -408,6 +412,7 @@ extern "C" void gte_dome_probe(int on, int thr) { if (on) for (int i = 0; i < DOME_PROBE_SLOTS; i++) { s_dome_probe[i].func = 0; s_dome_probe[i].count = 0; s_dome_probe[i].max_sz = 0; } } static inline void dome_probe_note(int32_t sz) { + if (s_gte_replay_sandbox) return; if (!s_dome_probe_on || sz < s_dome_probe_thr) return; /* Tally the guest RA (the GAME fn that jal'd to libgte for this projection), * NOT g_debug_current_func_addr (which is the libgte leaf 0x80000F40). */ @@ -475,6 +480,7 @@ static inline int gte_sxx(int32_t p){ int v=p&0xFFFF; return v>=0x8000? v-0x1000 static inline int gte_syy(int32_t p){ int v=(p>>16)&0xFFFF; return v>=0x8000? v-0x10000:v; } static void gte_rtp_record(const GTEState* g, uint32_t cmd) { + if (s_gte_replay_sandbox) return; if (!s_gte_rtp_ring) { s_gte_rtp_ring = (GteRtpRec*)calloc(GTE_RTP_RING_CAP, sizeof(GteRtpRec)); if (!s_gte_rtp_ring) return; @@ -553,6 +559,7 @@ static uint64_t s_gte_intpl_seq = 0; static void gte_intpl_record(const GTEState* g, const int16_t pre_ir[4], const int32_t pre_fc[3]) { + if (s_gte_replay_sandbox) return; if (!s_gte_intpl_ring) { s_gte_intpl_ring = (GteIntplRec*)calloc(GTE_INTPL_RING_CAP, sizeof(GteIntplRec)); if (!s_gte_intpl_ring) return; @@ -769,10 +776,15 @@ void gte_rtps_internal(GTEState* gte, int16_t* V, bool setMac0) { /* Depth-gated: un-squash only FAR geometry (the backdrop); keep near * props squashed so they stay aligned. Record SZ stats for tuning. */ int32_t sz = gte->SZ[3]; - if (sz < s_ws_sz_min) s_ws_sz_min = sz; - if (sz > s_ws_sz_max) s_ws_sz_max = sz; - s_ws_sz_n++; - if (sz >= s_ws_far_threshold) { do_squash = false; s_ws_sz_far++; } + if (!s_gte_replay_sandbox) { + if (sz < s_ws_sz_min) s_ws_sz_min = sz; + if (sz > s_ws_sz_max) s_ws_sz_max = sz; + s_ws_sz_n++; + } + if (sz >= s_ws_far_threshold) { + do_squash = false; + if (!s_gte_replay_sandbox) s_ws_sz_far++; + } } if (do_squash) xterm = xterm * s_ws_xnum / s_ws_xden; @@ -780,12 +792,14 @@ void gte_rtps_internal(GTEState* gte, int16_t* V, bool setMac0) { else if (s_ws_dome_on && s_ws_dome_num != s_ws_dome_den && !gpu_ws_present_native_43()) { int32_t sz = gte->SZ[3]; - if (sz < s_ws_sz_min) s_ws_sz_min = sz; - if (sz > s_ws_sz_max) s_ws_sz_max = sz; - s_ws_sz_n++; + if (!s_gte_replay_sandbox) { + if (sz < s_ws_sz_min) s_ws_sz_min = sz; + if (sz > s_ws_sz_max) s_ws_sz_max = sz; + s_ws_sz_n++; + } if (sz >= s_ws_far_threshold) { xterm = xterm * s_ws_dome_num / s_ws_dome_den; - s_ws_sz_far++; + if (!s_gte_replay_sandbox) s_ws_sz_far++; } } dome_probe_note(gte->SZ[3]); /* locate the dome draw fn (far-vertex tally) */ @@ -794,14 +808,16 @@ void gte_rtps_internal(GTEState* gte, int16_t* V, bool setMac0) { int64_t sx = sx16 >> 16; int64_t sy = sy16 >> 16; gte->push_sxy(sx, sy); - s_precise_sxy[0] = s_precise_sxy[1]; - s_precise_sxy[1] = s_precise_sxy[2]; - s_precise_sxy[2].packed = (uint32_t)gte->SXY[2]; - s_precise_sxy[2].x16 = (int32_t)sx16; - s_precise_sxy[2].y16 = (int32_t)sy16; - s_precise_sxy[2].z = gte->SZ[3]; - s_precise_sxy[2].valid = gte->SZ[3] != 0; - s_precise_sxy[3] = s_precise_sxy[2]; + if (!s_gte_replay_sandbox) { + s_precise_sxy[0] = s_precise_sxy[1]; + s_precise_sxy[1] = s_precise_sxy[2]; + s_precise_sxy[2].packed = (uint32_t)gte->SXY[2]; + s_precise_sxy[2].x16 = (int32_t)sx16; + s_precise_sxy[2].y16 = (int32_t)sy16; + s_precise_sxy[2].z = gte->SZ[3]; + s_precise_sxy[2].valid = gte->SZ[3] != 0; + s_precise_sxy[3] = s_precise_sxy[2]; + } geom_note((uint32_t)gte->SXY[2], sx16, sy16); // Step 5: Depth cueing (MAC0/IR0) — only for last vertex of RTPT or RTPS @@ -1426,10 +1442,24 @@ uint32_t gte_cfc2(GTEState* gte, uint8_t reg) { // --------------------------------------------------------------------------- static uint64_t s_gte_exec_count = 0; extern "C" uint64_t gte_get_exec_count(void) { return s_gte_exec_count; } +static uint32_t s_gte_replay_saved_caller_ra = 0; +extern "C" int gte_replay_side_effects_begin(void) { + using namespace PSXRecomp::GTE; + if (s_gte_replay_sandbox) return 0; + s_gte_replay_saved_caller_ra = s_gte_caller_ra; + s_gte_replay_sandbox = 1; + return 1; +} +extern "C" void gte_replay_side_effects_end(void) { + using namespace PSXRecomp::GTE; + if (!s_gte_replay_sandbox) return; + s_gte_caller_ra = s_gte_replay_saved_caller_ra; + s_gte_replay_sandbox = 0; +} extern "C" void gte_execute(CPUState* cpu, uint32_t cmd) { using namespace PSXRecomp::GTE; - s_gte_exec_count++; + if (!s_gte_replay_sandbox) s_gte_exec_count++; s_gte_caller_ra = cpu->gpr[31]; /* dome-locate probe: game fn that issued this projection */ GTEState gte; @@ -1445,7 +1475,7 @@ extern "C" void gte_execute(CPUState* cpu, uint32_t cmd) { /* GTE-activity gameplay detector ([widescreen] gte_game_mode): note every * perspective projection so gpu.c can stamp real 3D frames as gameplay * (no-op unless the game opts in — early-out on the config flag). */ - if (func == 0x01 || func == 0x30) + if (!s_gte_replay_sandbox && (func == 0x01 || func == 0x30)) psx_ws_note_gte_project(func == 0x30 ? 3 : 1); /* INTPL ring: snapshot inputs before the op mutates IR (outputs recorded * after the switch). */ @@ -1533,7 +1563,7 @@ extern "C" void gte_write_data(CPUState* cpu, uint8_t reg, uint32_t val) { GTEState gte; gte_load_data(gte, cpu); gte_mtc2(>e, reg, val); - if (reg >= 12 && reg <= 15) + if (!s_gte_replay_sandbox && reg >= 12 && reg <= 15) for (int i = 0; i < 4; i++) s_precise_sxy[i].valid = 0; for (int i = 0; i < 32; i++) cpu->gte_data[i] = gte_mfc2(>e, i); for (int i = 0; i < 32; i++) cpu->gte_ctrl[i] = gte_cfc2(>e, i); diff --git a/runtime/src/overlay_loader.c b/runtime/src/overlay_loader.c index 3655758e..408ff099 100644 --- a/runtime/src/overlay_loader.c +++ b/runtime/src/overlay_loader.c @@ -7,6 +7,7 @@ #include "interrupts.h" #include "debug_server.h" #include "psx_cycles.h" +#include "lockstep.h" #include "overlay_posix.h" #include #include @@ -332,6 +333,11 @@ static const void *s_shadow_cand = NULL; * a larger epoch is an exception armed INSIDE the shadow (contained). */ static int s_shadow_saved_native_exec = 0; static int s_shadow_saved_supp = 0; +static int s_shadow_saved_exec_phase = 0; +static int s_shadow_saved_call_bail = 0; +static uint32_t s_shadow_saved_debug_func = 0; +static uint32_t s_shadow_saved_irq_calls = 0; +static uint64_t s_shadow_saved_irq_supp = 0; static uint64_t s_shadow_epoch = 0; static uint32_t s_shadow_escapes = 0; static uint32_t s_shadow_escapes_native = 0; @@ -339,6 +345,39 @@ static uint32_t s_shadow_escapes_native = 0; * live-execution mode are gone; overlay gaps fall to the interpreter.) */ static void run_shadow_diff(CPUState *cpu, Candidate *c, uint32_t addr); +static int cand_range_contains(const Candidate *c, uint32_t phys) { + for (int r = 0; r < c->nranges; r++) + if (phys >= c->range_lo[r] && phys < c->range_lo[r] + c->range_len[r]) + return 1; + return 0; +} + +static int cand_ranges_overlap(const Candidate *a, const Candidate *b) { + for (int ar = 0; ar < a->nranges; ar++) { + uint32_t alo = a->range_lo[ar], ahi = alo + a->range_len[ar]; + for (int br = 0; br < b->nranges; br++) { + uint32_t blo = b->range_lo[br], bhi = blo + b->range_len[br]; + if (alo < bhi && blo < ahi) return 1; + } + } + return 0; +} + +/* An explicit entry filter owns its whole code envelope, including separately + * seeded continuation candidates. This prevents an IRQ resume at a mid-function + * PC from bypassing the diagnostic quarantine and running native live. The scan + * is intentionally diagnostic-only (s_diff_addr != 0). */ +static int cand_selected_for_diff(const Candidate *c) { + if (!s_diff_addr) return 1; + uint32_t target = s_diff_addr & 0x1FFFFFFFu; + if (c->addr == target || cand_range_contains(c, target)) return 1; + for (int i = 0; i < s_cand_n; i++) { + const Candidate *owner = &s_cand[i]; + if (owner->addr == target && cand_ranges_overlap(c, owner)) return 1; + } + return 0; +} + /* ---- Counters (surfaced via overlay_loader_status) --------------------- */ static int s_ndlls = 0; /* DLLs LoadLibrary'd */ static uint64_t s_load_total_us = 0; @@ -1372,6 +1411,9 @@ void overlay_loader_get_irq_suppress(int *mode, uint32_t *rl, uint64_t *supp) { } static int overlay_irq_suppressed_now(void) { + /* Differential replay (and its authoritative interpreter pass) is atomic. + * Never let a previously armed rate-limit punch a real IRQ into a shadow. */ + if (s_in_shadow) { s_irq_suppressed++; return 1; } if (s_suppress_irq) { if (s_irq_ratelimit == 0) { s_irq_suppressed++; return 1; } if ((++s_irq_callcount % s_irq_ratelimit) != 0) { s_irq_suppressed++; return 1; } @@ -2394,10 +2436,9 @@ retry_candidates: * too — the blocklist blocks LIVE native, not the sandbox). */ int in_own_shadow = (s_in_shadow && (const void *)c == s_shadow_cand); if (!in_own_shadow) { - int want_diff = s_diff_mode && - (!s_diff_addr || phys == (s_diff_addr & 0x1FFFFFFFu)); + int want_diff = s_diff_mode && cand_selected_for_diff(c); if (want_diff && addr < 0x10000u) want_diff = 0; - if (want_diff && c->diff_passes < OVERLAY_DIFF_BUDGET) { + if (want_diff && (s_diff_addr || c->diff_passes < OVERLAY_DIFF_BUDGET)) { if (_probe) s_cps_probe_outcome = 5; s_diffgate_interp++; s_disp_interp++; @@ -2501,8 +2542,7 @@ retry_candidates: * gcc/DLL candidates are the trusted tier (validated at dev time) and * run native directly; they are diffed only in explicit dev diff mode * (PSX_OVERLAY_DIFF / overlay_diff cmd). */ - int want_diff = s_diff_mode && - (!s_diff_addr || phys == (s_diff_addr & 0x1FFFFFFFu)); + int want_diff = s_diff_mode && cand_selected_for_diff(c); /* Kernel-window candidates (call gates 0xA0/0xB0/0xC0 and the RAM * kernel) are NOT shadow-diffable: the gates tail-jump via RAM * tables into kernel SERVICES whose behavior depends on scheduler/ @@ -2512,7 +2552,8 @@ retry_candidates: * ring). The diff harness validates OVERLAY shard codegen against * the interp oracle; kernel gates stay on their normal route. */ if (want_diff && addr < 0x10000u) want_diff = 0; - if (want_diff && !s_in_shadow && c->diff_passes < OVERLAY_DIFF_BUDGET) { + if (want_diff && !s_in_shadow && + (s_diff_addr || c->diff_passes < OVERLAY_DIFF_BUDGET)) { /* NEVER start a shadow inside an exception dispatch: the guest's * ReturnFromException longjmps to the setjmp frame BELOW us * (psx_check_interrupts), unwinding run_shadow_diff WITHOUT its @@ -2762,16 +2803,19 @@ int overlay_fp_enabled(void) { * computation is correct and the fault is timing/interrupt-ordering. */ #define SHADOW_RAM_SIZE (2u * 1024u * 1024u) #define SHADOW_SPAD_SIZE 1024u -static uint8_t s_ram0[SHADOW_RAM_SIZE]; /* pre-call main-RAM snapshot */ -static uint8_t s_ramN[SHADOW_RAM_SIZE]; /* post-native main-RAM */ -static uint8_t s_ramI[SHADOW_RAM_SIZE]; /* post-interp main-RAM (kept) */ -static uint8_t s_spad0[SHADOW_SPAD_SIZE]; /* pre-call scratchpad snapshot*/ -static uint8_t s_spadI[SHADOW_SPAD_SIZE]; /* post-interp scratchpad (kept)*/ -static uint64_t s_shadow_skipped_dev = 0; /* candidates skipped: touch MMIO */ +static uint8_t s_ram0[SHADOW_RAM_SIZE], s_ramN[SHADOW_RAM_SIZE], s_ramI[SHADOW_RAM_SIZE]; +static uint8_t s_spad0[SHADOW_SPAD_SIZE], s_spadI[SHADOW_SPAD_SIZE]; +static uint64_t s_shadow_skipped_dev = 0; /* unsafe/incomplete traces skipped */ /* s_diff_mode / s_in_shadow declared above (before dispatch). */ typedef struct { uint64_t seq; uint32_t addr; + int kind; /* architectural/trace difference class */ + int index; /* register/array index, or -1 */ + uint64_t cycle_native, cycle_interp; + uint32_t trace_ops, replay_ops; + int trace_kind; + uint32_t trace_pc, trace_addr, trace_expected, trace_actual; int reg; /* first differing gpr index, -1 none */ uint32_t reg_native, reg_interp; int hi_diff, lo_diff; @@ -2793,7 +2837,7 @@ static uint32_t s_detail_nat_hi, s_detail_nat_lo, s_detail_int_hi, s_detail_int_ void overlay_loader_set_diff_mode(int on) { s_diff_mode = on ? 1 : 0; } -static void run_shadow_diff(CPUState *cpu, Candidate *c, uint32_t addr) { +static void run_shadow_diff_legacy(CPUState *cpu, Candidate *c, uint32_t addr) { extern uint8_t *memory_get_ram_ptr(void); extern uint8_t *memory_get_scratchpad_ptr(void); extern int dirty_ram_dispatch(CPUState *cpu, uint32_t addr, uint32_t stop_addr); @@ -2960,6 +3004,215 @@ static void run_shadow_diff(CPUState *cpu, Candidate *c, uint32_t addr) { s_in_shadow = 0; } +enum { + SHDIFF_NONE = 0, SHDIFF_PC, SHDIFF_GPR, SHDIFF_HI, SHDIFF_LO, + SHDIFF_COP0, SHDIFF_GTE_DATA, SHDIFF_GTE_CTRL, + SHDIFF_MULDIV_TS, SHDIFF_GTE_TS, SHDIFF_READ_ABSORB, + SHDIFF_READ_WHICH, SHDIFF_READ_FUDGE, SHDIFF_LD_WHICH, + SHDIFF_LD_ABSORB, SHDIFF_CYCLE, SHDIFF_TRACE, SHDIFF_SETUP +}; + +static int shadow_cpu_diff(const CPUState *native, const CPUState *interp, + int *index, uint32_t *nv, uint32_t *iv) { +#define SHCMP_SCALAR(kind_, field_) do { \ + if (native->field_ != interp->field_) { \ + *index = -1; *nv = (uint32_t)native->field_; *iv = (uint32_t)interp->field_; \ + return (kind_); \ + } \ +} while (0) +#define SHCMP_ARRAY(kind_, field_, count_) do { \ + for (int _i = 0; _i < (count_); _i++) { \ + if (native->field_[_i] != interp->field_[_i]) { \ + *index = _i; *nv = (uint32_t)native->field_[_i]; \ + *iv = (uint32_t)interp->field_[_i]; return (kind_); \ + } \ + } \ +} while (0) + if (native->pc != interp->pc) { + *index = -1; *nv = native->pc; *iv = interp->pc; return SHDIFF_PC; + } + SHCMP_ARRAY(SHDIFF_GPR, gpr, 32); + SHCMP_SCALAR(SHDIFF_HI, hi); + SHCMP_SCALAR(SHDIFF_LO, lo); + SHCMP_ARRAY(SHDIFF_COP0, cop0, 32); + SHCMP_ARRAY(SHDIFF_GTE_DATA, gte_data, 32); + SHCMP_ARRAY(SHDIFF_GTE_CTRL, gte_ctrl, 32); + SHCMP_SCALAR(SHDIFF_MULDIV_TS, muldiv_ts_done); + SHCMP_SCALAR(SHDIFF_GTE_TS, gte_ts_done); + SHCMP_ARRAY(SHDIFF_READ_ABSORB, read_absorb, 33); + SHCMP_SCALAR(SHDIFF_READ_WHICH, read_absorb_which); + SHCMP_SCALAR(SHDIFF_READ_FUDGE, read_fudge); + SHCMP_SCALAR(SHDIFF_LD_WHICH, ld_which_t); + SHCMP_SCALAR(SHDIFF_LD_ABSORB, ld_absorb); +#undef SHCMP_ARRAY +#undef SHCMP_SCALAR + *index = -1; *nv = 0; *iv = 0; + return SHDIFF_NONE; +} + +/* Authoritative interpreter RECORD + side-effect-free native REPLAY. + * Memory reads are replayed and writes verified/no-op, the replay clock advances + * without servicing devices, and GTE visual/diagnostic sidecars are sandboxed. + * Thus RAM/MMIO/devices mutate exactly once while the comparison covers the full + * CPU architectural and timing state plus the ordered memory-operation trace. */ +static void run_shadow_diff(CPUState *cpu, Candidate *c, uint32_t addr) { + extern int dirty_ram_dispatch(CPUState *cpu, uint32_t addr, uint32_t stop_addr); + extern void psx_dispatch_call(CPUState *cpu, uint32_t addr, uint32_t return_addr); + extern int gte_replay_side_effects_begin(void); + extern void gte_replay_side_effects_end(void); + extern uint32_t g_debug_current_func_addr; + extern uint64_t psx_exception_setjmp_epoch(void); + + s_in_shadow = 1; + s_shadow_epoch = psx_exception_setjmp_epoch(); + int saved_supp = s_suppress_irq; + int saved_native = s_native_exec; + s_suppress_irq = 1; + s_native_exec = 0; + s_shadow_saved_supp = saved_supp; + s_shadow_saved_native_exec = saved_native; + s_shadow_saved_exec_phase = g_exec_phase; + s_shadow_saved_call_bail = g_psx_call_bail; + s_shadow_saved_debug_func = g_debug_current_func_addr; + s_shadow_saved_irq_calls = s_irq_callcount; + s_shadow_saved_irq_supp = s_irq_suppressed; + + CPUState cpu0 = *cpu; + uint64_t cycle0 = psx_get_cycle_count(); + uint32_t stop_ra = cpu0.gpr[31]; + uint32_t record_ops = 0, replay_ops = 0; + int saw_exception = 0; + int icache_record_ok = psx_icache_shadow_record_begin(); + int record_owned = 0; + int record_ok = icache_record_ok && ls_shadow_record_begin(); + if (record_ok) { + record_owned = 1; + dirty_ram_dispatch(cpu, addr, stop_ra); + record_ok = ls_shadow_record_end(&record_ops, &saw_exception); + record_owned = 0; + } else { + dirty_ram_dispatch(cpu, addr, stop_ra); + } + s_shadow_calls++; + + CPUState cpuI = *cpu; + uint64_t cycleI = psx_get_cycle_count(); + if (!record_ok || saw_exception) { + s_shadow_skipped_dev++; + if (record_owned) ls_shadow_abort(); + psx_icache_shadow_abort(); + s_native_exec = saved_native; + s_suppress_irq = saved_supp; + s_in_shadow = 0; + return; + } + + CPUState cpuN = cpu0; + int trace_kind = 0; + uint32_t trace_pc = 0, trace_addr = 0, trace_expected = 0, trace_actual = 0; + int replay_owned = 0; + int replay_ok = ls_shadow_replay_begin(); + if (replay_ok) replay_owned = 1; + int icache_ok = replay_ok && psx_icache_shadow_replay_begin(); + int clock_ok = icache_ok && psx_cycle_replay_begin(cycle0); + int gte_ok = clock_ok && gte_replay_side_effects_begin(); + uint32_t saved_debug_func = g_debug_current_func_addr; + uint32_t saved_irq_callcount = s_irq_callcount; + uint64_t saved_irq_suppressed = s_irq_suppressed; + int saved_bail = g_psx_call_bail; + + if (gte_ok) { + s_shadow_cand = c; + { + int prev_phase = g_exec_phase; + OverlayFlushFn prev_flush = overlay_flush_enter(c); + g_exec_phase = 2; + c->fn(&cpuN); + overlay_flush_leave(prev_flush); + g_exec_phase = prev_phase; + } + { + int guard = 0; + while (cpuN.pc != 0 && !g_psx_call_bail && guard++ < 8192) { + uint32_t tv = cpuN.pc; + if ((tv & 0x1FFFFFFFu) == (stop_ra & 0x1FFFFFFFu)) break; + cpuN.pc = 0; + int prev_phase = g_exec_phase; + g_exec_phase = 3; + psx_dispatch_call(&cpuN, tv, stop_ra); + g_exec_phase = prev_phase; + } + } + s_shadow_cand = NULL; + } + + if (gte_ok) gte_replay_side_effects_end(); + uint64_t cycleN = clock_ok ? psx_cycle_replay_end() : cycle0; + psx_icache_shadow_replay_end(); + if (replay_ok) { + replay_ok = ls_shadow_replay_end(&replay_ops, &trace_kind, &trace_pc, + &trace_addr, &trace_expected, &trace_actual); + replay_owned = 0; + } else if (replay_owned) { + ls_shadow_abort(); + replay_owned = 0; + } + psx_icache_shadow_abort(); + g_debug_current_func_addr = saved_debug_func; + s_irq_callcount = saved_irq_callcount; + s_irq_suppressed = saved_irq_suppressed; + g_psx_call_bail = saved_bail; + + int index = -1; + uint32_t native_value = 0, interp_value = 0; + int kind = (!gte_ok || !replay_ok) ? SHDIFF_TRACE : + shadow_cpu_diff(&cpuN, &cpuI, &index, &native_value, &interp_value); + if (kind == SHDIFF_NONE && cycleN - cycle0 != cycleI - cycle0) { + kind = SHDIFF_CYCLE; + native_value = (uint32_t)(cycleN - cycle0); + interp_value = (uint32_t)(cycleI - cycle0); + } + + if (kind == SHDIFF_NONE) { + if (c->diff_passes < OVERLAY_DIFF_BUDGET) c->diff_passes++; + } else { + c->diff_passes = 0; + c->device_touch = 1; /* fail closed: all later calls stay interpreted */ + s_shadow_divs++; + if (!s_detail_captured) { + s_detail_captured = 1; + s_detail_addr = c->addr; + memcpy(s_detail_nat_gpr, cpuN.gpr, sizeof s_detail_nat_gpr); + memcpy(s_detail_int_gpr, cpuI.gpr, sizeof s_detail_int_gpr); + s_detail_nat_hi = cpuN.hi; s_detail_nat_lo = cpuN.lo; + s_detail_int_hi = cpuI.hi; s_detail_int_lo = cpuI.lo; + } + int addr_recs = 0; + for (int i = 0; i < s_sdiv_n; i++) + if (s_sdiv[i].addr == c->addr && ++addr_recs >= 16) break; + if (s_sdiv_n < SDIV_CAP && addr_recs < 16) { + ShadowDiv *d = &s_sdiv[s_sdiv_n++]; + memset(d, 0, sizeof *d); + d->seq = s_shadow_calls; d->addr = c->addr; + d->kind = kind; d->index = index; + d->cycle_native = cycleN - cycle0; + d->cycle_interp = cycleI - cycle0; + d->trace_ops = record_ops; d->replay_ops = replay_ops; + d->trace_kind = trace_kind; d->trace_pc = trace_pc; + d->trace_addr = trace_addr; d->trace_expected = trace_expected; + d->trace_actual = trace_actual; + d->reg = kind == SHDIFF_GPR ? index : -1; + d->reg_native = native_value; d->reg_interp = interp_value; + d->hi_diff = kind == SHDIFF_HI; d->lo_diff = kind == SHDIFF_LO; + d->ram_off = -1; + } + } + + s_native_exec = saved_native; + s_suppress_irq = saved_supp; + s_in_shadow = 0; +} + /* Called by deferred_exception_longjmp() (interrupts.c) before it unwinds. * If the longjmp target frame predates a live shadow run (target_epoch <= * s_shadow_epoch), the unwind blows through run_shadow_diff and its epilogue @@ -2981,12 +3234,27 @@ void overlay_loader_shadow_escape_fixup(uint64_t target_epoch) { g_overlay_flush_pending_cycles = NULL; if (!s_in_shadow) return; if (target_epoch > s_shadow_epoch) return; /* contained: leave armed */ + if (s_shadow_cand != NULL) { + Candidate *escaped = (Candidate *)s_shadow_cand; + escaped->diff_passes = 0; + escaped->device_touch = 1; /* native escape is unverified: fail closed */ + } + { extern void gte_replay_side_effects_end(void); gte_replay_side_effects_end(); } + (void)psx_cycle_replay_end(); + psx_icache_shadow_abort(); + ls_shadow_abort(); s_shadow_escapes++; if (s_shadow_cand != NULL) s_shadow_escapes_native++; s_in_shadow = 0; s_shadow_cand = NULL; s_native_exec = s_shadow_saved_native_exec; s_suppress_irq = s_shadow_saved_supp; + g_exec_phase = s_shadow_saved_exec_phase; + g_psx_call_bail = s_shadow_saved_call_bail; + { extern uint32_t g_debug_current_func_addr; + g_debug_current_func_addr = s_shadow_saved_debug_func; } + s_irq_callcount = s_shadow_saved_irq_calls; + s_irq_suppressed = s_shadow_saved_irq_supp; } int overlay_loader_dump_shadow_detail(char *out, int cap) { @@ -3026,13 +3294,22 @@ int overlay_loader_dump_shadow(char *out, int cap) { for (int i = 0; i < s_sdiv_n && n < cap - 200; i++) { ShadowDiv *d = &s_sdiv[i]; n += snprintf(out + n, cap - n, - "%s{\"seq\":%llu,\"addr\":\"0x%08X\",\"reg\":%d," + "%s{\"seq\":%llu,\"addr\":\"0x%08X\",\"kind\":%d,\"index\":%d,\"reg\":%d," "\"reg_native\":\"0x%08X\",\"reg_interp\":\"0x%08X\"," "\"hi\":%d,\"lo\":%d,\"ram_off\":%lld," - "\"ram_native\":\"0x%08X\",\"ram_interp\":\"0x%08X\"}", - i ? "," : "", (unsigned long long)d->seq, d->addr, d->reg, + "\"ram_native\":\"0x%08X\",\"ram_interp\":\"0x%08X\"," + "\"cycle_native\":%llu,\"cycle_interp\":%llu," + "\"trace_ops\":%u,\"replay_ops\":%u,\"trace_kind\":%d," + "\"trace_pc\":\"0x%08X\",\"trace_addr\":\"0x%08X\"," + "\"trace_expected\":\"0x%08X\",\"trace_actual\":\"0x%08X\"}", + i ? "," : "", (unsigned long long)d->seq, d->addr, + d->kind, d->index, d->reg, d->reg_native, d->reg_interp, d->hi_diff, d->lo_diff, - (long long)d->ram_off, d->ram_native, d->ram_interp); + (long long)d->ram_off, d->ram_native, d->ram_interp, + (unsigned long long)d->cycle_native, + (unsigned long long)d->cycle_interp, + d->trace_ops, d->replay_ops, d->trace_kind, + d->trace_pc, d->trace_addr, d->trace_expected, d->trace_actual); } n += snprintf(out + n, cap - n, "]}"); return n; diff --git a/runtime/src/psx_cycles.c b/runtime/src/psx_cycles.c index 3d7f8dca..8e051272 100644 --- a/runtime/src/psx_cycles.c +++ b/runtime/src/psx_cycles.c @@ -17,6 +17,26 @@ #endif uint64_t psx_cycle_count = 0; +static int s_cycle_replay_active = 0; +static uint64_t s_cycle_replay_live = 0; + +int psx_cycle_replay_begin(uint64_t start_cycle) { + extern int g_ls_replay_active; + if (!g_ls_replay_active || s_cycle_replay_active) return 0; + s_cycle_replay_live = psx_cycle_count; + psx_cycle_count = start_cycle; + s_cycle_replay_active = 1; + return 1; +} + +uint64_t psx_cycle_replay_end(void) { + uint64_t replay_cycle = psx_cycle_count; + if (s_cycle_replay_active) { + psx_cycle_count = s_cycle_replay_live; + s_cycle_replay_active = 0; + } + return replay_cycle; +} /* Throttle watchdog check to once per ~64K cycles to keep hot-path cost * negligible (most blocks emit 5-30 cycles, so the check fires every @@ -188,7 +208,15 @@ static void psx_advance_cycles_exact(uint32_t cycles) { } void psx_advance_cycles(uint32_t cycles) { - { extern int g_ls_replay_active; if (g_ls_replay_active) return; } /* lockstep replay: no global cycle/device mutation */ + { extern int g_ls_replay_active; + if (g_ls_replay_active) { + /* Replay owns a private-in-time view of the global clock. Advance it + * so GTE/muldiv deadlines and memory wait states see the same time as + * the authoritative pass, but never service devices or observers. */ + if (s_cycle_replay_active) psx_cycle_count += (uint64_t)cycles; + return; + } + } if (cycles == 0) return; uint32_t charged_cycles = cycles; #ifdef PSX_COSIM @@ -321,9 +349,12 @@ void psx_idle_note_check(CPUState *cpu, uint32_t check_pc) { extern int psx_get_in_exception(void); extern uint64_t g_guest_store_count, g_mmio_access_count; + /* A speculative replay must not clear or train the authoritative live idle + * detector. Its clock/device view is transactional and cannot be skipped. */ + if (g_ls_replay_active) return; if (!idle_skip_on() || g_idle_note_suppress) return; if (check_pc == 0 || psx_get_in_exception() || g_psx_call_bail || - g_precise_mode || g_ls_mode != 0 || g_ls_replay_active) { + g_precise_mode || g_ls_mode != 0) { s_idle_pc = 0; s_idle_streak = 0; return; diff --git a/runtime/src/psx_icache.c b/runtime/src/psx_icache.c index a38dea5f..658dd64f 100644 --- a/runtime/src/psx_icache.c +++ b/runtime/src/psx_icache.c @@ -30,6 +30,7 @@ #include "cpu_state.h" #include #include +#include extern void psx_advance_cycles(uint32_t cycles); @@ -52,6 +53,36 @@ int psx_icache_enabled(void) { * never equal an aligned fetch address (aligned addrs have bits 0-1 = 0), so every * line starts cold (miss). */ static uint32_t s_icache_tv[1024]; +static uint32_t s_icache_shadow_entry[1024]; +static uint32_t s_icache_shadow_live[1024]; +static int s_icache_shadow_state = 0; /* 0 none, 1 recorded, 2 replay */ + +int psx_icache_shadow_record_begin(void) { + if (s_icache_shadow_state != 0) return 0; + memcpy(s_icache_shadow_entry, s_icache_tv, sizeof s_icache_tv); + s_icache_shadow_state = 1; + return 1; +} + +int psx_icache_shadow_replay_begin(void) { + if (s_icache_shadow_state != 1) return 0; + memcpy(s_icache_shadow_live, s_icache_tv, sizeof s_icache_tv); + memcpy(s_icache_tv, s_icache_shadow_entry, sizeof s_icache_tv); + s_icache_shadow_state = 2; + return 1; +} + +void psx_icache_shadow_replay_end(void) { + if (s_icache_shadow_state != 2) return; + memcpy(s_icache_tv, s_icache_shadow_live, sizeof s_icache_tv); + s_icache_shadow_state = 0; +} + +void psx_icache_shadow_abort(void) { + if (s_icache_shadow_state == 2) + memcpy(s_icache_tv, s_icache_shadow_live, sizeof s_icache_tv); + s_icache_shadow_state = 0; +} void psx_icache_reset(void) { for (int i = 0; i < 1024; i++) s_icache_tv[i] = 0x1u; /* bit0 set => never matches */ @@ -60,7 +91,11 @@ void psx_icache_reset(void) { /* Charge the fetch cost for the instruction at `addr` and update the cache tags. * `addr` is the runtime guest virtual PC (aligned). Gated on PSX_ENABLE_BLOCK_CYCLES. */ void psx_icache_fetch(CPUState* cpu, uint32_t addr) { - { extern int g_ls_replay_active; if (g_ls_replay_active) return; } /* lockstep replay: no global icache mutation */ + { extern int g_ls_replay_active; + /* Ordinary lockstep replay must not perturb the shared cache. The whole- + * call shadow owns a private transactional cache view and may evolve it. */ + if (g_ls_replay_active && s_icache_shadow_state != 2) return; + } #ifdef PSX_ENABLE_BLOCK_CYCLES if (!psx_icache_enabled()) return; uint32_t idx = (addr & 0xFFCu) >> 2; From ffda40934b5800230dab9b8451ea2f6776124cb1 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sun, 19 Jul 2026 13:40:25 -0700 Subject: [PATCH 31/37] runtime: preserve causality at device deadlines --- runtime/include/dma.h | 4 + runtime/src/dma.c | 47 ++++++++++ runtime/src/psx_cycles.c | 14 ++- runtime/src/sio.c | 20 +++-- .../tests/test_psx_cycle_event_boundaries.c | 86 +++++++++++++++++++ runtime/tests/test_sio_card_protocol.c | 60 ++++++++++++- 6 files changed, 223 insertions(+), 8 deletions(-) create mode 100644 runtime/tests/test_psx_cycle_event_boundaries.c diff --git a/runtime/include/dma.h b/runtime/include/dma.h index 98b5d58e..dccddd3d 100644 --- a/runtime/include/dma.h +++ b/runtime/include/dma.h @@ -31,6 +31,10 @@ void dma_advance(uint32_t cycles); /* Cycle-budgeted precise event slicing: guest CPU cycles until a DELIVERABLE * DMA IRQ (bit3 unmasked in i_mask). UINT32_MAX if none. */ uint32_t dma_cycles_to_irq(uint32_t i_mask); +/* Mask-blind scheduler boundary: next DMA word movement or delayed completion, + * not merely the final IRQ. RAM-producing DMA must become visible at its + * per-word due cycle even when guest code polls ordinary RAM instead of DICR. */ +uint32_t dma_cycles_to_internal_event(void); /* Strict deliverability form for wait-loop elision: unlike the conservative * scheduler bound, ignores channel completions whose DICR IRQ is disabled. */ uint32_t dma_cycles_to_deliverable_irq(uint32_t i_mask); diff --git a/runtime/src/dma.c b/runtime/src/dma.c index f1ea9df6..e5ed38ec 100644 --- a/runtime/src/dma.c +++ b/runtime/src/dma.c @@ -881,6 +881,53 @@ uint32_t dma_cycles_to_irq(uint32_t i_mask) { return best; } +uint32_t dma_cycles_to_internal_event(void) { + uint32_t best = 0xFFFFFFFFu; + + /* Async MDEC channels move one word whenever their cycle accumulator + * reaches the channel cost. Completion-only scheduling batches many RAM + * writes at a later service boundary, which is observably different when + * the CPU polls the destination buffer. */ + for (int ch = 0; ch < 2; ch++) { + const DMAAsyncChannel *a = &mdec_async[ch]; + if (!a->active || a->remaining_words == 0 || + !((channels[ch].chcr >> 24) & 1u) || !channel_enabled(ch)) + continue; + uint32_t direction = channels[ch].chcr & 1u; + if ((ch == 0 && direction == 0) || (ch == 1 && direction != 0)) + return 1u; /* invalid route is completed on the next DMA tick */ + if ((ch == 0 && !mdec_dma_write_ready()) || + (ch == 1 && !mdec_dma_read_ready())) + continue; + uint32_t cpw = ch == 0 ? DMA_MDEC_IN_CYCLES_PER_WORD + : DMA_MDEC_OUT_CYCLES_PER_WORD; + uint32_t d = a->cycles_accum < cpw ? cpw - a->cycles_accum : 1u; + if (d < best) best = d; + } + + /* CD-ROM DMA writes guest RAM incrementally. Expose each word on time; + * waiting only for the channel-complete IRQ can delay hundreds of writes. */ + if (cdrom_async.active && cdrom_async.remaining_words != 0 && + ((channels[3].chcr >> 24) & 1u) && channel_enabled(3)) { + if ((channels[3].chcr & 1u) != 0) { + return 1u; /* unsupported RAM->CD direction cancels next tick */ + } + if (cdrom_dma_ready()) { + uint32_t d = cdrom_async.cycles_accum < DMA_CDROM_CYCLES_PER_WORD + ? DMA_CDROM_CYCLES_PER_WORD - cdrom_async.cycles_accum + : 1u; + if (d < best) best = d; + } + } + + for (int ch = 0; ch < 7; ch++) { + if (delayed_complete[ch].active && + delayed_complete[ch].cycles_remaining < best) + best = delayed_complete[ch].cycles_remaining; + } + return best; +} + uint32_t dma_cycles_to_deliverable_irq(uint32_t i_mask) { if (!(i_mask & (1u << 3))) return 0xFFFFFFFFu; uint32_t best = 0xFFFFFFFFu; diff --git a/runtime/src/psx_cycles.c b/runtime/src/psx_cycles.c index 8e051272..cf1f9a9a 100644 --- a/runtime/src/psx_cycles.c +++ b/runtime/src/psx_cycles.c @@ -108,7 +108,7 @@ static uint32_t devices_cycles_to_next_internal_event(void) { uint32_t best = interrupts_cycles_to_vblank(); /* frame pacing always */ uint32_t t = timers_cycles_to_irq(0xFFFFFFFFu); if (t < best) best = t; uint32_t c = cdrom_cycles_to_irq(0xFFFFFFFFu); if (c < best) best = c; - uint32_t d = dma_cycles_to_irq(0xFFFFFFFFu); if (d < best) best = d; + uint32_t d = dma_cycles_to_internal_event(); if (d < best) best = d; uint32_t s = sio_cycles_to_irq(0xFFFFFFFFu); if (s < best) best = s; if (best == 0) best = 1; /* due/overdue: process within one cycle */ return best; @@ -153,6 +153,18 @@ static void psx_devices_service_to_now(void) { uint32_t to_ev = devices_cycles_to_next_internal_event(); if (to_ev > 0 && to_ev < step) step = to_ev; if (step == 0) step = 1; + /* Preserve interval causality at an advertised event boundary. + * Devices are advanced in a fixed dependency order (CD before DMA, + * MDEC input before output). Passing the whole D-cycle interval to + * each device lets a state created by an earlier device at cycle D + * be treated by a later device as though it existed for all D + * cycles. Advance the quiet prefix first, then the boundary cycle: + * this is exactly the conservative timeline with O(events), not + * O(cycles), host calls. */ + if (step > 1u && to_ev == step) { + advance_devices(step - 1u); + step = 1u; + } advance_devices(step); interrupts_service_scheduled_events(); } diff --git a/runtime/src/sio.c b/runtime/src/sio.c index 0b5058c0..dc70668b 100644 --- a/runtime/src/sio.c +++ b/runtime/src/sio.c @@ -2006,16 +2006,20 @@ void sio_advance(uint32_t cycles) { int remaining = (int)cycles; int transitions = 0; const int MAX_TRANSITIONS = 8; - while (remaining > 0 && transitions < MAX_TRANSITIONS) { + while (transitions < MAX_TRANSITIONS && + (remaining > 0 || + (sio_shift_active && sio_shift_remaining <= 0) || + (sio_pending_ack && sio_ack_remaining <= 0))) { int dt = remaining; int next_event = -1; if (sio_shift_active && sio_shift_remaining > 0 - && sio_shift_remaining < dt) { + && sio_shift_remaining <= dt) { dt = sio_shift_remaining; next_event = 0; } if (sio_pending_ack && sio_ack_remaining > 0 - && sio_ack_remaining < dt) { + && (sio_ack_remaining < dt || + (sio_ack_remaining == dt && next_event < 0))) { dt = sio_ack_remaining; next_event = 1; } @@ -2057,16 +2061,20 @@ void sio_tick(int cycles) { int remaining = cycles; int transitions = 0; const int MAX_TRANSITIONS = 8; - while (remaining > 0 && transitions < MAX_TRANSITIONS) { + while (transitions < MAX_TRANSITIONS && + (remaining > 0 || + (sio_shift_active && sio_shift_remaining <= 0) || + (sio_pending_ack && sio_ack_remaining <= 0))) { int dt = remaining; int next_event = -1; if (sio_shift_active && sio_shift_remaining > 0 - && sio_shift_remaining < dt) { + && sio_shift_remaining <= dt) { dt = sio_shift_remaining; next_event = 0; } if (sio_pending_ack && sio_ack_remaining > 0 - && sio_ack_remaining < dt) { + && (sio_ack_remaining < dt || + (sio_ack_remaining == dt && next_event < 0))) { dt = sio_ack_remaining; next_event = 1; } diff --git a/runtime/tests/test_psx_cycle_event_boundaries.c b/runtime/tests/test_psx_cycle_event_boundaries.c new file mode 100644 index 00000000..ce29ec05 --- /dev/null +++ b/runtime/tests/test_psx_cycle_event_boundaries.c @@ -0,0 +1,86 @@ +/* + * Pin cross-device causality at a scheduler deadline. + * + * Build from runtime/tests: + * gcc -std=c99 -Wall -Wextra -ffunction-sections -fdata-sections \ + * -Wl,--gc-sections -I../include -o test_psx_cycle_event_boundaries \ + * test_psx_cycle_event_boundaries.c ../src/psx_cycles.c + */ + +#include "psx_cycles.h" + +#include +#include + +int g_ls_replay_active = 0; +int g_ls_mode = 0; +int g_precise_mode = 0; +int g_psx_call_bail = 0; +uint32_t i_mask = 0; +uint64_t g_guest_store_count = 0; +uint64_t g_mmio_access_count = 0; + +static uint32_t s_cd_cycles_remaining = 5; +static int s_cd_ready = 0; +static uint32_t s_dma_ready_cycles = 0; + +void sio_advance(uint32_t cycles) { (void)cycles; } + +void cdrom_advance(uint32_t cycles) { + if (s_cd_ready) return; + if (cycles >= s_cd_cycles_remaining) { + s_cd_cycles_remaining = 0; + s_cd_ready = 1; + } else { + s_cd_cycles_remaining -= cycles; + } +} + +void dma_advance(uint32_t cycles) { + if (s_cd_ready) s_dma_ready_cycles += cycles; +} + +void timers_advance(uint32_t cycles) { (void)cycles; } +void interrupts_advance_cycles(uint32_t cycles) { (void)cycles; } +void interrupts_service_scheduled_events(void) {} + +uint32_t interrupts_cycles_to_vblank(void) { return UINT32_MAX; } +uint32_t timers_cycles_to_irq(uint32_t mask) { (void)mask; return UINT32_MAX; } +uint32_t cdrom_cycles_to_irq(uint32_t mask) { + (void)mask; + return s_cd_ready ? UINT32_MAX : s_cd_cycles_remaining; +} +uint32_t dma_cycles_to_internal_event(void) { return UINT32_MAX; } +uint32_t dma_cycles_to_deliverable_irq(uint32_t mask) { + (void)mask; + return UINT32_MAX; +} +uint32_t sio_cycles_to_irq(uint32_t mask) { (void)mask; return UINT32_MAX; } +int psx_get_in_exception(void) { return 0; } + +void starvation_watchdog_check(void) {} +void starvation_ring_pc_sample(void) {} + +int main(void) { + psx_cycles_resync_after_restore(); + psx_advance_cycles(5); + + if (psx_get_cycle_count() != 5) { + fprintf(stderr, "FAIL cycle count: expected 5 got %llu\n", + (unsigned long long)psx_get_cycle_count()); + return 1; + } + if (!s_cd_ready) { + fprintf(stderr, "FAIL CD event did not fire at cycle 5\n"); + return 1; + } + if (s_dma_ready_cycles != 1) { + fprintf(stderr, + "FAIL retroactive DMA credit: expected 1 boundary cycle got %u\n", + s_dma_ready_cycles); + return 1; + } + + fprintf(stderr, "PASS cross-device deadline preserves D-1 + 1 causality\n"); + return 0; +} diff --git a/runtime/tests/test_sio_card_protocol.c b/runtime/tests/test_sio_card_protocol.c index 7fe64395..607286dd 100644 --- a/runtime/tests/test_sio_card_protocol.c +++ b/runtime/tests/test_sio_card_protocol.c @@ -66,6 +66,13 @@ void card_data_writes_arm(uint8_t value, uint16_t mc_state, uint8_t mc_data_idx, uint8_t slot) { (void)value; (void)mc_state; (void)mc_data_idx; (void)slot; } +void event_ring_record_aux(uint16_t kind, uint8_t detail, uint32_t aux) { + (void)kind; (void)detail; (void)aux; +} +void psx_irq_raise(uint32_t bit, uint32_t detail) { + (void)detail; + i_stat |= 1u << bit; +} /* ---- Test harness ---- */ @@ -356,9 +363,54 @@ static void test_directory_frame_layout(void) { * Just check that read succeeded. */ } -int main(void) { +/* Exact scheduler deadlines must fire on the call that reaches the boundary. + * The old strict-inequality loops subtracted the countdown to zero without + * processing the event, after which sio_cycles_to_irq() no longer advertised + * it and the event could be delayed until an unrelated scheduler wake. */ +static void test_cycle_paced_exact_boundaries(void) { + const uint32_t irq_sio0 = 1u << 7; + const uint16_t ctrl = CTRL_TX_EN | CTRL_SELECT | CTRL_ACK_IRQ_EN; + + sio_init(); + memcard_init("test_dir"); + i_stat = 0; + sio_write(SIO_CTRL, ctrl); + sio_write(SIO_TX_DATA, 0x81); + + EXPECT_EQ("paced.shift.initial", 1088, sio_cycles_to_irq(irq_sio0)); + sio_advance(1087); + EXPECT_EQ("paced.shift.remaining", 1, sio_cycles_to_irq(irq_sio0)); + EXPECT_EQ("paced.shift.no_irq", 0, i_stat & irq_sio0); + sio_advance(1); + EXPECT_EQ("paced.ack.initial", 170, sio_cycles_to_irq(irq_sio0)); + EXPECT_EQ("paced.ack.no_irq", 0, i_stat & irq_sio0); + sio_advance(169); + EXPECT_EQ("paced.ack.remaining", 1, sio_cycles_to_irq(irq_sio0)); + sio_advance(1); + EXPECT_EQ("paced.ack.irq", irq_sio0, i_stat & irq_sio0); + + /* sio_tick() retains the same event walker for access-paced callers. */ + sio_init(); + memcard_init("test_dir"); + i_stat = 0; + sio_write(SIO_CTRL, ctrl); + sio_write(SIO_TX_DATA, 0x81); + sio_tick(1088); + EXPECT_EQ("paced.tick.ack", 170, sio_cycles_to_irq(irq_sio0)); + sio_tick(170); + EXPECT_EQ("paced.tick.irq", irq_sio0, i_stat & irq_sio0); +} + +int main(int argc, char **argv) { fprintf(stderr, "=== sio card protocol tests ===\n"); + if (argc == 2 && strcmp(argv[1], "exact-boundaries") == 0) { + test_cycle_paced_exact_boundaries(); + fprintf(stderr, "test_cycle_paced_exact_boundaries: %d/%d ok\n", + g_checks - g_failures, g_checks); + return g_failures > 0 ? 1 : 0; + } + test_no_card_present(); fprintf(stderr, "test_no_card_present: %d/%d ok\n", g_checks - g_failures, g_checks); @@ -399,6 +451,12 @@ int main(void) { (g_checks - prev_checks) - (g_failures - prev_fails), g_checks - prev_checks); + prev_checks = g_checks; prev_fails = g_failures; + test_cycle_paced_exact_boundaries(); + fprintf(stderr, "test_cycle_paced_exact_boundaries: %d/%d ok\n", + (g_checks - prev_checks) - (g_failures - prev_fails), + g_checks - prev_checks); + fprintf(stderr, "=== TOTAL: %d checks, %d failures ===\n", g_checks, g_failures); return g_failures > 0 ? 1 : 0; From 02f29b979db61db46cd938aea1ad2c9735e37aa1 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sun, 19 Jul 2026 14:29:57 -0700 Subject: [PATCH 32/37] aot: harden exact interior fragment recovery --- .../tests/test_aot_overlay_discovery.py | 106 +++++++++ tools/compile_overlays.py | 212 +++++++++++++++--- 2 files changed, 287 insertions(+), 31 deletions(-) diff --git a/recompiler/tests/test_aot_overlay_discovery.py b/recompiler/tests/test_aot_overlay_discovery.py index e9f93179..bef3f77e 100644 --- a/recompiler/tests/test_aot_overlay_discovery.py +++ b/recompiler/tests/test_aot_overlay_discovery.py @@ -1061,6 +1061,10 @@ m.publish_shard_pair(sys.argv[2], sys.argv[3], sys.argv[4]) assert MOD.compiled_shard_complete(pair_dll, 14) assert not MOD.compiled_shard_complete( pair_dll, 14 | (1 << 16)) # same cg dir, stale flavor + assert MOD.cached_shard_manifest_status( + pair_dll, 14, manifest, pair_id) == 'match' + assert MOD.cached_shard_manifest_status( + pair_dll, 14, manifest, pair_id ^ 1) == 'mismatch' source = pathlib.Path(tmp) / "loaded.c" final = str(pathlib.Path(tmp) / "shard.dll") @@ -1098,6 +1102,107 @@ m.publish_shard_pair(sys.argv[2], sys.argv[3], sys.argv[4]) MOD._compile_dll_direct = original +def check_interior_fragment_contract(): + entry = LOAD + 0x20 + good = [(entry, 0x12345678, [(entry, 8)])] + assert MOD.validate_interior_fragment_ids(entry, good, {entry}) is None + assert "missing from generated C" in MOD.validate_interior_fragment_ids( + entry, good, set()) + assert "0 manifest identities" in MOD.validate_interior_fragment_ids( + entry, [], {entry}) + assert "2 manifest identities" in MOD.validate_interior_fragment_ids( + entry, good + good, {entry}) + assert "0 ranges" in MOD.validate_interior_fragment_ids( + entry, [(entry, 0, [])], {entry}) + assert "17 ranges" in MOD.validate_interior_fragment_ids( + entry, [(entry, 0, [(entry + i * 4, 4) for i in range(17)])], + {entry}) + assert "outside its guarded ranges" in MOD.validate_interior_fragment_ids( + entry, [(entry, 0, [(entry + 4, 4)])], {entry}) + assert "fragment entry" in MOD.validate_interior_fragment_ids( + entry, good + [(entry + 0x40, 0, + [(entry + 0x40 + i * 4, 4) for i in range(17)])], + {entry, entry + 0x40}) + assert "empty guarded range" in MOD.validate_interior_fragment_ids( + entry, good + [(entry + 0x40, 0, [(entry + 0x40, 0)])], + {entry, entry + 0x40}) + assert MOD.interior_failure_is_deterministic( + "generated-c-audit: unsupported") + assert MOD.interior_failure_is_deterministic( + "requested-entry-audit: omitted") + assert not MOD.interior_failure_is_deterministic( + "fragment cache-key collision/stale pair") + + audit = { + "included_reasons": {}, + "executed_pcs": set(), + "producer_ranges": [(LOAD, LOAD + 0x100)], + "accepted_cross_producer_calls": {LOAD + 0x200}, + } + data = bytes(0x100) + assert MOD.make_interior_fragment_job( + LOAD & 0x1FFFFFFF, LOAD, len(data), data, audit, set()) is None + job = MOD.make_interior_fragment_job( + LOAD & 0x1FFFFFFF, LOAD, len(data), data, audit, {entry}) + assert job is not None + assert job["candidates"] == {entry} + assert job["forced"] == {entry} + assert job["executed"] == set() + assert job["producer_ranges"] == ((LOAD, LOAD + 0x100),) + assert job["cross_call_allow"] == (LOAD + 0x200,) + assert MOD.make_interior_fragment_job( + LOAD & 0x1FFFFFFF, LOAD, len(data), data, audit, + {LOAD + 0x200}) is None + + with tempfile.TemporaryDirectory() as td: + dll = pathlib.Path(td) / "fragment.dll" + ranges = pathlib.Path(td) / "fragment.ranges" + dll.write_bytes(b"DLL") + ranges.write_text("EXPECTED\n") + assert MOD.cached_shard_manifest_status( + str(dll), None, "EXPECTED\n") == "match" + assert MOD.cached_shard_manifest_status( + str(dll), None, "OTHER\n") == "mismatch" + ranges.unlink() + assert MOD.cached_shard_manifest_status( + str(dll), None, "EXPECTED\n") == "missing" + + old_run = MOD.subprocess.run + seen_seeds = [] + + class Args: + recompiler = "recompiler" + game_toml = "game.toml" + + class Failed: + returncode = 1 + stdout = "" + stderr = "expected stop" + + def stop_after_seeds(cmd, **_kwargs): + seeds = pathlib.Path(cmd[cmd.index("--seeds") + 1]) + seen_seeds.extend(seeds.read_text().splitlines()) + return Failed() + + try: + MOD.subprocess.run = stop_after_seeds + ids, status = MOD.compile_interior_fragment( + entry, data, LOAD, len(data), LOAD & 0x1FFFFFFF, "unused", + Args(), {}, {}, + ((LOAD, LOAD + 0x40), (LOAD + 0x80, LOAD + 0x100)), + (LOAD + 0x200, LOAD + 0x180, LOAD + 0x200)) + assert ids is None and status.startswith("recompiler-error") + finally: + MOD.subprocess.run = old_run + assert seen_seeds == [ + f"producer_range 0x{LOAD:08X} 0x{LOAD + 0x40:08X}", + f"producer_range 0x{LOAD + 0x80:08X} 0x{LOAD + 0x100:08X}", + f"cross_call_allow 0x{LOAD + 0x180:08X}", + f"cross_call_allow 0x{LOAD + 0x200:08X}", + f"dispatch_root 0x{entry:08X}", + ] + + def check_tcc_runtime_define_parity(): """TCC shards must use the same release/cycle contract as GCC shards.""" seen = [] @@ -1158,6 +1263,7 @@ def main(): check_recompiler_pointer_table_alias(args.recompiler) check_retained_alias_contract(args.recompiler) check_atomic_dll_publication() + check_interior_fragment_contract() check_tcc_runtime_define_parity() data = bytearray(0x200) diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index 7e0dc481..7631ca7e 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -2113,6 +2113,11 @@ def _interior_fail_key(phys_addr: int, interior: int, region_crc: int) -> str: return f'{phys_addr:08X}_{interior & 0x1FFFFFFF:08X}_{region_crc:08X}' +def interior_failure_is_deterministic(reason: str) -> bool: + """Only guest-byte/codegen audit verdicts belong in the persistent memo.""" + return reason.startswith(('generated-c-audit:', 'requested-entry-audit:')) + + def load_interior_fail_memo(cache_dir: str) -> set: memo = set() try: @@ -2210,9 +2215,75 @@ def generate_interior_fragment_static(interior: int, data: bytes, } +def validate_interior_fragment_ids(interior: int, frag_ids: list, + generated_defs: set[int]) -> str | None: + """Return a fail-closed reason when a fragment omitted its requested entry. + + A non-empty manifest is not sufficient: the recompiler can emit some other + reachable function while rejecting/demoting the exact interior PC. The + loader can also retain at most 16 ranges for one function. Refuse to + publish unless the requested entry has one unambiguous, runtime-representable + identity whose guarded bytes include the entry itself. + """ + entry = (interior & 0x1FFFFFFF) | 0x80000000 + if entry not in generated_defs: + return f'requested entry 0x{entry:08X} missing from generated C' + matches = [(ev, crc, ranges) for ev, crc, ranges in frag_ids if ev == entry] + if len(matches) != 1: + return (f'requested entry 0x{entry:08X} has {len(matches)} ' + 'manifest identities (expected exactly 1)') + ranges = matches[0][2] + if not 1 <= len(ranges) <= 16: + return (f'requested entry 0x{entry:08X} has {len(ranges)} ranges ' + '(runtime supports 1..16)') + if not _addr_in_func_ids(entry, matches): + return f'requested entry 0x{entry:08X} is outside its guarded ranges' + for ev, _crc, ranges in frag_ids: + if not 1 <= len(ranges) <= 16: + return (f'fragment entry 0x{ev:08X} has {len(ranges)} ranges ' + '(runtime supports 1..16)') + if any(length <= 0 for _lo, length in ranges): + return f'fragment entry 0x{ev:08X} has an empty guarded range' + return None + + +def make_interior_fragment_job(phys_addr: int, load_addr: int, size: int, + data: bytes, seed_audit: dict, + forced_interiors: set[int]) -> dict | None: + """Build one explicit fragment job without mutating the shared seed set.""" + interiors = { + a for a, reason in seed_audit['included_reasons'].items() + if reason == 'DISPATCH_INTERIOR' + } + dispatch_roots = { + a for a, reason in seed_audit['included_reasons'].items() + if reason in ('DISPATCH_ENTRY', 'DISPATCH_ROOT') + } + executed = set(seed_audit.get('executed_pcs', set())) + region_hi = phys_addr + size + forced = { + a for a in forced_interiors + if phys_addr <= (a & 0x1FFFFFFF) < region_hi + } + if not (((interiors or dispatch_roots) and executed) or forced): + return None + return { + 'phys_addr': phys_addr, + 'load_addr': load_addr, + 'size': size, + 'data': data, + 'candidates': interiors | dispatch_roots | forced, + 'executed': executed, + 'forced': forced, + 'producer_ranges': tuple(seed_audit['producer_ranges']), + 'cross_call_allow': tuple(seed_audit['accepted_cross_producer_calls']), + } + + def compile_interior_fragment(interior: int, data: bytes, load_addr: int, size: int, phys_addr: int, cache_dir: str, - args, sub_env: dict): + args, sub_env: dict, toml_doc: dict, + producer_ranges=(), cross_call_allow=()): """Compile an ISOLATED interior-entry 'island' fragment that ENTERS at an executed orphan DISPATCH_INTERIOR PC (a host that static analysis never discovered, e.g. an FMV driver reached via a computed jump) and covers the @@ -2236,6 +2307,10 @@ def compile_interior_fragment(interior: int, data: bytes, load_addr: int, f.write(make_psxexe(load_addr, interior, data)) seeds_path = os.path.join(tmp, 'seeds.txt') with open(seeds_path, 'w') as f: + for range_lo, range_hi in producer_ranges: + f.write(f'producer_range 0x{range_lo:08X} 0x{range_hi:08X}\n') + for target in sorted(set(cross_call_allow)): + f.write(f'cross_call_allow 0x{target:08X}\n') f.write(f'dispatch_root 0x{interior:08X}\n') out_dir_tmp = os.path.join(tmp, 'out') os.makedirs(out_dir_tmp) @@ -2258,7 +2333,7 @@ def compile_interior_fragment(interior: int, data: bytes, load_addr: int, with open(full_c) as f: src = patch_generated_c(f.read(), load_addr, size) c_audit = audit_generated_c(src, load_addr, size, - binascii.crc32(data) & 0xFFFFFFFF, {}) + binascii.crc32(data) & 0xFFFFFFFF, toml_doc) if c_audit['unknown_bad'] or c_audit['unsupported_todo_addrs']: # RETAIN the failed fragment's C for triage. Region shards already # keep their _patched.c on audit failure; interior fragments used to @@ -2291,6 +2366,10 @@ def compile_interior_fragment(interior: int, data: bytes, load_addr: int, frag_ids = parse_overlay_func_ids(ranges_src, data, load_addr, size) if not frag_ids: return None, 'no-func-ids (empty ranges manifest)' + identity_error = validate_interior_fragment_ids( + interior, frag_ids, c_audit['defs']) + if identity_error: + return None, f'requested-entry-audit: {identity_error}' pair_id = overlay_pair_id(src, frag_ids) src = add_overlay_pair_export(src, pair_id) # Key the fragment DLL by its func-identity SET (dedup like a region @@ -2301,10 +2380,15 @@ def compile_interior_fragment(interior: int, data: bytes, load_addr: int, struct.pack(' bool: _ctypes.dlclose(handle) +def _dll_pair_id_matches(dll_path: str, expected_pair_id: int) -> bool: + """Read overlay_pair_id from one loaded image, then release it.""" + import _ctypes + import ctypes + library = None + try: + library = (ctypes.WinDLL(dll_path) if os.name == 'nt' + else ctypes.CDLL(dll_path)) + pair_fn = library.overlay_pair_id + pair_fn.argtypes = [] + pair_fn.restype = ctypes.c_uint64 + return pair_fn() == (expected_pair_id & 0xFFFFFFFFFFFFFFFF) + except (OSError, AttributeError): + return False + finally: + if library is not None and library._handle: + handle = library._handle + library._handle = 0 + if os.name == 'nt': + _ctypes.FreeLibrary(handle) + else: + _ctypes.dlclose(handle) + + def compiled_shard_complete(dll_path: str, expected_abi: int | None = None) -> bool: """Check pair completeness and optional ABI/flavor under one OS lock.""" @@ -2820,6 +2928,33 @@ def compiled_shard_complete(dll_path: str, _dll_abi_matches(dll_path, expected_abi)) +def cached_shard_manifest_status(dll_path: str, expected_abi: int | None, + expected_manifest: str, + expected_pair_id: int | None = None) -> str: + """Return ``match``, ``mismatch``, or ``missing`` under the pair lock.""" + ranges = os.path.splitext(dll_path)[0] + '.ranges' + with _shard_pair_lock(dll_path): + _recover_shard_pair_locked(dll_path) + complete = (os.path.isfile(dll_path) and + os.path.getsize(dll_path) > 0 and + os.path.isfile(ranges) and + os.path.getsize(ranges) > 0) + if not complete: + return 'missing' + if (expected_abi is not None and + not _dll_abi_matches(dll_path, expected_abi)): + return 'missing' + if (expected_pair_id is not None and + not _dll_pair_id_matches(dll_path, expected_pair_id)): + return 'mismatch' + try: + with open(ranges) as f: + actual_manifest = f.read() + except OSError: + return 'missing' + return 'match' if actual_manifest == expected_manifest else 'mismatch' + + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -2885,7 +3020,10 @@ def main(): 'captures within one region stay ordered. 1 = the ' 'sequential path. --static always runs sequential.') args = ap.parse_args() - forced_interiors = {int(v, 0) for v in args.force_interior} + forced_interiors = { + (int(v, 0) & 0x1FFFFFFF) | 0x80000000 + for v in args.force_interior + } # ---- Framework-injected cache location wins over CLI flags ---------------- # The runtime (autocompile.c) exports PSX_OVERLAY_CACHE_DIR / PSX_OVERLAY_CAPTURES @@ -3071,14 +3209,16 @@ def main(): # set — the 0x80024548 class), and the fragment pass is the demand- # driven recovery path for exactly that. if not args.static: - _interiors = {a for a, r in seed_audit['included_reasons'].items() - if r == 'DISPATCH_INTERIOR'} - _disp_roots = {a for a, r in seed_audit['included_reasons'].items() - if r in ('DISPATCH_ENTRY', 'DISPATCH_ROOT')} - _executed = seed_audit.get('executed_pcs', set()) - if (_interiors or _disp_roots) and _executed: - interior_frag_jobs.append((phys_addr, load_addr, size, data, - _interiors | _disp_roots, _executed)) + # A manual exact demand must not depend on the capture still + # retaining executed_pcs or on today's classifier assigning the PC + # DISPATCH_INTERIOR. That provenance is precisely what + # --force-interior restores. Keep it isolated from the shared + # region seed set and queue the fragment directly. + fragment_job = make_interior_fragment_job( + phys_addr, load_addr, size, data, seed_audit, + forced_interiors) + if fragment_job is not None: + interior_frag_jobs.append(fragment_job) if not args.static: dll_path = os.path.join(cache_dir, f'{phys_addr:08X}_{crc32:08X}{overlay_ext()}') @@ -3375,14 +3515,14 @@ def main(): fail_memo = load_interior_fail_memo(cache_dir) memo_skipped = 0 for job in interior_frag_jobs: - phys_addr, load_addr, size, data, interior_pcs, executed = job + phys_addr = job['phys_addr'] + load_addr = job['load_addr'] + size = job['size'] + data = job['data'] + interior_pcs = set(job['candidates']) + executed = job['executed'] + forced = job['forced'] region_crc = binascii.crc32(data) & 0xFFFFFFFF - region_lo = load_addr & 0x1FFFFFFF - region_hi = region_lo + size - interior_pcs = set(interior_pcs) - interior_pcs.update( - a for a in forced_interiors - if region_lo <= (a & 0x1FFFFFFF) < region_hi) # ENTRY-based orphan test, not range-based: native code is # enterable only at manifest F entries, so "inside a compiled # range" does NOT make a dispatch target servable — a range- @@ -3391,9 +3531,14 @@ def main(): covered_entries = load_region_entry_set( cache_dir, phys_addr, overlay_abi_tag(args.runtime_include, args.flavor)) - orphans = sorted(a for a in interior_pcs - if (a in executed or a in forced_interiors) - and (a & 0x1FFFFFFF) not in covered_entries) + orphans = sorted( + a for a in interior_pcs + # Explicit exact demands are byte-variant-specific. Do not let + # an F entry from some other region-byte variant suppress the + # fragment before its own generated identity is known. + if (a in forced or + (a in executed and + (a & 0x1FFFFFFF) not in covered_entries))) if not orphans: continue built = 0 @@ -3407,7 +3552,9 @@ def main(): stats.add_skip() continue frag_ids, status = compile_interior_fragment( - a, data, load_addr, size, phys_addr, cache_dir, args, frag_env) + a, data, load_addr, size, phys_addr, cache_dir, args, + frag_env, toml, job['producer_ranges'], + job['cross_call_allow']) if frag_ids: built += 1 if status == 'cached': @@ -3419,14 +3566,17 @@ def main(): else: # An executed orphan interior that would NOT build is a real # coverage hole: that PC's whole dispatch chain runs - # interpreted forever. Tally it loudly with the reason, and - # memoize it so later cycles don't re-walk the same doomed bytes. + # interpreted forever. Tally it loudly with the reason. + # Persist only deterministic generated-code audit verdicts; + # compiler/spawn errors and cache-pair mismatches depend on + # mutable external state and must remain retryable. stats.add_fail(f'interior 0x{a:08X} @region 0x{phys_addr:08X}', 'fragment', status) - fail_memo.add(key) - append_interior_fail_memo(cache_dir, key, status) + if interior_failure_is_deterministic(status): + fail_memo.add(key) + append_interior_fail_memo(cache_dir, key, status) print(f' interior fragments @0x{phys_addr:08X}: {built}/{len(orphans)} ' - f'executed orphan interior(s) -> isolated island shards') + f'exact-demand orphan interior(s) -> isolated island shards') if memo_skipped: print(f' interior fail-memo: skipped {memo_skipped} known-doomed ' f'interior(s) (data-as-code from identical bytes; ' From 870dc6450a2655fa3473035ef8f771dcfd64b348 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sun, 19 Jul 2026 16:06:13 -0700 Subject: [PATCH 33/37] chore: checkpoint before strict jump-table discovery From 504db264457560d20f75296bfc82c1e7dd6caa46 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sun, 19 Jul 2026 17:15:17 -0700 Subject: [PATCH 34/37] feat: add bounded indirect overlay discovery --- recompiler/include/control_flow.h | 2 + recompiler/include/function_analysis.h | 34 ++ recompiler/src/code_generator.cpp | 134 +---- recompiler/src/control_flow.cpp | 2 + recompiler/src/function_analysis.cpp | 536 +++++++++++++----- recompiler/src/main_psx.cpp | 11 + recompiler/tests/reachable_discovery_test.cpp | 169 ++++++ recompiler/tests/recompiler_patch_test.cpp | 103 ++++ recompiler/tests/test_aot_base_recovery.py | 98 ++++ .../tests/test_aot_overlay_discovery.py | 183 +++++- tools/aot_overlay_spike/extract_generic.py | 148 ++++- tools/compile_overlays.py | 230 +++++++- 12 files changed, 1364 insertions(+), 286 deletions(-) diff --git a/recompiler/include/control_flow.h b/recompiler/include/control_flow.h index 4ad386aa..858a2f23 100644 --- a/recompiler/include/control_flow.h +++ b/recompiler/include/control_flow.h @@ -53,6 +53,8 @@ struct BasicBlock { struct ControlFlowGraph { uint32_t function_start; uint32_t function_end; + uint32_t producer_lo = 0; + uint32_t producer_hi = 0; std::map blocks; // Map: block start address -> block std::vector block_order; // Blocks in address order diff --git a/recompiler/include/function_analysis.h b/recompiler/include/function_analysis.h index a32e2df8..b3260caf 100644 --- a/recompiler/include/function_analysis.h +++ b/recompiler/include/function_analysis.h @@ -29,6 +29,10 @@ struct Function { // emitter generate ONE shared body per host with an entry switch instead // of duplicating the host's blocks per alias. std::vector alias_group_entries; + // Exact-entry composites retain the source producer that owns this code. + // Zero means an ordinary single-image function. + uint32_t producer_lo = 0; + uint32_t producer_hi = 0; }; struct AbsorbedEntry { @@ -60,6 +64,36 @@ struct FunctionAnalysisResult { std::set exact_reachable_pcs; }; +// A canonical, bounds-checked MIPS jump table recovered from a jr $rN site. +// `table_base` and each pair's first value are the runtime addresses stored by +// the guest; the pair's second value is the corresponding address in the +// active executable image (normally identical, except for BIOS shell remaps). +struct ExactJumpTable { + uint32_t table_base = 0; + uint32_t table_count = 0; + std::vector> targets; +}; + +using ExactAddressMapper = uint32_t (*)(uint32_t, const PS1Executable&); + +// Recognize only the canonical bounded-switch dependency chain: +// sltiu/beq guard -> sll index,2 -> addu table address -> lw target -> jr. +// The table constant may use either same-register or cross-register +// `lui source; addiu base,source,lo`. `producer_lo/producer_hi` bound both +// table storage and case code for composite images; zero/zero means the full +// executable. Every dependency, table word, and target must pass the hard +// safety checks; otherwise the whole table is rejected. +bool resolve_exact_bounded_jump_table( + const PS1Executable& exe, + uint32_t entry, + uint32_t hard_cap, + uint32_t jr_pc, + uint32_t jr_rs, + ExactJumpTable& table, + ExactAddressMapper runtime_to_image = nullptr, + uint32_t producer_lo = 0, + uint32_t producer_hi = 0); + class FunctionAnalyzer { public: explicit FunctionAnalyzer(const PS1Executable& exe); diff --git a/recompiler/src/code_generator.cpp b/recompiler/src/code_generator.cpp index cb4f7d99..b3a26914 100644 --- a/recompiler/src/code_generator.cpp +++ b/recompiler/src/code_generator.cpp @@ -1749,89 +1749,15 @@ std::string CodeGenerator::translate_basic_block( // P1: lui $R,hi; addiu $R,$R,lo; addu $D,$idx,$R; lw $D,0($D) // P2: lui $R,hi; addu $D,$R,$idx; lw $D,off($D) // table_base = lui_val + addiu_val + lw_offset - uint32_t table_base = 0; - uint32_t table_count = 0; - { - uint32_t lw_base_reg = 0xFF; // base reg in the LW - int32_t lw_offset = 0; // offset in the LW instruction - uint32_t addu_cand[2] = {0xFF, 0xFF}; // rs, rt of ADDU - uint32_t table_reg = 0xFF; // candidate for table base reg - uint32_t lui_val = 0; - int16_t addiu_val[2] = {0, 0}; // per-candidate ADDIU - bool found_addiu[2]= {false, false}; - bool found_lui = false; - - for (int back = 1; back <= 40; back++) { - uint32_t sa = block.exit_instr.address - (uint32_t)(back * 4); - if (sa < cfg.function_start) break; - auto sopt = exe_.read_word(sa); - if (!sopt) break; - uint32_t si = *sopt; - uint32_t s_op = (si >> 26) & 0x3F; - uint32_t s_rs = (si >> 21) & 0x1F; - uint32_t s_rt = (si >> 16) & 0x1F; - uint32_t s_rd = (si >> 11) & 0x1F; - uint32_t s_fn = si & 0x3F; - - // LW $jr_reg, offset($base) - if (s_op == 0x23 && s_rt == jr_rs && lw_base_reg == 0xFF) { - lw_base_reg = s_rs; - lw_offset = (int32_t)(int16_t)(si & 0xFFFF); - continue; - } - // ADDU $lw_base, $a, $b — save both operands as candidates - if (s_op == 0x00 && s_fn == 0x21 && - s_rd == lw_base_reg && lw_base_reg != 0xFF && - addu_cand[0] == 0xFF) { - addu_cand[0] = s_rs; - addu_cand[1] = s_rt; - table_reg = s_rt; // initial guess; may flip on LUI match - continue; - } - // ADDIU $cand, $cand, imm (scanning back → before LUI) - if (s_op == 0x09 && addu_cand[0] != 0xFF) { - for (int c = 0; c < 2; c++) { - if (!found_addiu[c] && addu_cand[c] != 0xFF && - s_rs == addu_cand[c] && s_rt == addu_cand[c]) { - addiu_val[c] = (int16_t)(si & 0xFFFF); - found_addiu[c] = true; - break; - } - } - continue; - } - // LUI $cand, hi — determines which candidate is the table base - if (s_op == 0x0F && addu_cand[0] != 0xFF && !found_lui) { - for (int c = 0; c < 2; c++) { - if (addu_cand[c] != 0xFF && s_rt == addu_cand[c]) { - lui_val = ((uint32_t)(si & 0xFFFF)) << 16; - table_reg = addu_cand[c]; - // carry over the ADDIU for this candidate - if (!found_addiu[c]) { addiu_val[c] = 0; } - // store addiu in slot 0 for convenience - addiu_val[0] = addiu_val[c]; - found_addiu[0] = found_addiu[c]; - found_lui = true; - break; - } - } - continue; - } - // SLTIU — table entry count - if (s_op == 0x0B && table_count == 0) { - table_count = si & 0xFFFF; - continue; - } - if (found_lui && table_count != 0) break; - } - - if (found_lui && table_count > 0 && table_count < 512) { - uint32_t base = lui_val + - (found_addiu[0] ? (uint32_t)(int32_t)addiu_val[0] : 0u); - table_base = base + (uint32_t)lw_offset; - } - } - + ExactJumpTable exact_table; + bool have_exact_table = resolve_exact_bounded_jump_table( + exe_, cfg.function_start, cfg.function_end, + block.exit_instr.address, jr_rs, exact_table, + ram_to_rom, cfg.producer_lo, cfg.producer_hi); + uint32_t table_base = have_exact_table + ? exact_table.table_base : 0u; + uint32_t table_count = have_exact_table + ? exact_table.table_count : 0u; // If we found a valid table, read entries and emit switch. // table_base may be a RAM address (BIOS shell code copies // ROM 0xBFC18000+ to RAM 0x80030000+). Translate to ROM @@ -1842,11 +1768,8 @@ std::string CodeGenerator::translate_basic_block( uint32_t rom_table_base = ram_to_rom(table_base, exe_); std::set seen; std::vector> targets; // {runtime_addr, rom_addr} - for (uint32_t i = 0; i < table_count; i++) { - auto eopt = exe_.read_word(rom_table_base + i * 4); - if (!eopt) continue; - uint32_t runtime_addr = *eopt; - uint32_t rom_addr = ram_to_rom(runtime_addr, exe_); + for (const auto& [runtime_addr, rom_addr] : + exact_table.targets) { if (seen.insert(rom_addr).second && (cfg.blocks.count(rom_addr) || extra_labels_.count(rom_addr))) { targets.push_back({runtime_addr, rom_addr}); @@ -2323,33 +2246,18 @@ void CodeGenerator::scan_jr_tables( for (const auto& [baddr, blk] : cfg.blocks) { if (blk.exit_instr.type != ControlFlowType::JumpRegister) continue; uint32_t jr_r = get_rs(blk.exit_instr.instruction); - // Run same backward scan as translate_basic_block to find table_base/count - uint32_t tb = 0, tc = 0; - { - uint32_t lw_base = 0xFF; int32_t lw_off = 0; - uint32_t ac[2] = {0xFF,0xFF}; uint32_t lui_v = 0; - int16_t av[2] = {0,0}; bool fa[2] = {false,false}, fl = false; - for (int b = 1; b <= 40; b++) { - uint32_t sa = blk.exit_instr.address - (uint32_t)(b*4); - if (sa < cfg.function_start) break; - auto so = exe_.read_word(sa); if (!so) break; - uint32_t si=*so, s_op=(si>>26)&0x3F, s_rs=(si>>21)&0x1F, - s_rt=(si>>16)&0x1F, s_rd=(si>>11)&0x1F, s_fn=si&0x3F; - if (s_op==0x23 && s_rt==jr_r && lw_base==0xFF) { lw_base=s_rs; lw_off=(int32_t)(int16_t)(si&0xFFFF); continue; } - if (s_op==0x00 && s_fn==0x21 && s_rd==lw_base && lw_base!=0xFF && ac[0]==0xFF) { ac[0]=s_rs; ac[1]=s_rt; continue; } - if (s_op==0x09 && ac[0]!=0xFF) { for(int c=0;c<2;c++){if(!fa[c]&&ac[c]!=0xFF&&s_rs==ac[c]&&s_rt==ac[c]){av[c]=(int16_t)(si&0xFFFF);fa[c]=true;break;}} continue; } - if (s_op==0x0F && ac[0]!=0xFF && !fl) { for(int c=0;c<2;c++){if(ac[c]!=0xFF&&s_rt==ac[c]){lui_v=((uint32_t)(si&0xFFFF))<<16;if(!fa[c]){av[c]=0;}av[0]=av[c];fa[0]=fa[c];fl=true;break;}} continue; } - if (s_op==0x0B && tc==0) { tc=si&0xFFFF; continue; } - if (fl && tc!=0) break; - } - if (fl && tc>0 && tc<512) tb = (lui_v+(fa[0]?(uint32_t)(int32_t)av[0]:0u)) + (uint32_t)lw_off; + ExactJumpTable exact_table; + if (!resolve_exact_bounded_jump_table( + exe_, cfg.function_start, cfg.function_end, + blk.exit_instr.address, jr_r, exact_table, ram_to_rom, + cfg.producer_lo, cfg.producer_hi)) { + continue; } + uint32_t tb = exact_table.table_base; + uint32_t tc = exact_table.table_count; if (tb==0 || tc==0) continue; - uint32_t rom_tb = ram_to_rom(tb, exe_); - for (uint32_t i = 0; i < tc; i++) { - auto eopt = exe_.read_word(rom_tb + i*4); - if (!eopt) continue; - uint32_t t = ram_to_rom(*eopt, exe_); + for (const auto& target_pair : exact_table.targets) { + uint32_t t = target_pair.second; if (t >= cfg.function_start && t < cfg.function_end) { out_edges[baddr].push_back(t); if (!cfg.blocks.count(t)) diff --git a/recompiler/src/control_flow.cpp b/recompiler/src/control_flow.cpp index 3a08662d..68e45234 100644 --- a/recompiler/src/control_flow.cpp +++ b/recompiler/src/control_flow.cpp @@ -359,6 +359,8 @@ ControlFlowGraph ControlFlowAnalyzer::analyze_function(const Function& func) { uint32_t walk_lo = func.alias_walk_lo ? func.alias_walk_lo : func.start_addr; cfg.function_start = walk_lo; cfg.function_end = func.end_addr; + cfg.producer_lo = func.producer_lo; + cfg.producer_hi = func.producer_hi; // Simplified single-pass analysis: scan instructions to find block boundaries // without building full CFG. This avoids the crash in build_basic_blocks. diff --git a/recompiler/src/function_analysis.cpp b/recompiler/src/function_analysis.cpp index 59758edd..bfb86862 100644 --- a/recompiler/src/function_analysis.cpp +++ b/recompiler/src/function_analysis.cpp @@ -759,6 +759,353 @@ static ExactCf exact_classify_cf(uint32_t pc, uint32_t instr) { } // namespace +bool resolve_exact_bounded_jump_table( + const PS1Executable& exe, + uint32_t entry, + uint32_t hard_cap, + uint32_t jr_pc, + uint32_t jr_rs, + ExactJumpTable& table, + ExactAddressMapper runtime_to_image, + uint32_t producer_lo, + uint32_t producer_hi) { + table = {}; + if (entry >= hard_cap || jr_pc < entry || jr_pc >= hard_cap || + (entry & 3u) != 0u || (hard_cap & 3u) != 0u || + (jr_pc & 3u) != 0u || jr_rs == 0u || jr_rs == 31u) { + return false; + } + + auto read = [&](uint32_t pc) { return exe.read_word(pc); }; + auto mapped = [&](uint32_t addr) { + return runtime_to_image ? runtime_to_image(addr, exe) : addr; + }; + if (producer_lo == 0u && producer_hi == 0u) { + producer_lo = exe.header.load_address; + producer_hi = exe.end_address(); + } + if (producer_lo >= producer_hi || (producer_lo & 3u) != 0u || + (producer_hi & 3u) != 0u) { + return false; + } + auto jr_word = read(jr_pc); + if (!jr_word.has_value() || + *jr_word != ((jr_rs & 0x1Fu) << 21u | 0x08u)) { + return false; + } + auto is_control = [&](uint32_t pc, uint32_t instr) { + (void)pc; + return exact_classify_cf(pc, instr).kind != ExactCfKind::Normal; + }; + auto in_delay_slot = [&](uint32_t pc) { + if (pc < entry + 4u) return false; + auto prev = read(pc - 4u); + return prev.has_value() && is_control(pc - 4u, *prev); + }; + auto has_call_between = [&](uint32_t lo, uint32_t hi) { + for (uint32_t pc = lo; pc < hi; pc += 4u) { + auto word = read(pc); + if (!word.has_value()) return true; + ExactCf cf = exact_classify_cf(pc, *word); + if (cf.kind == ExactCfKind::Jal || + cf.kind == ExactCfKind::Jalr) { + return true; + } + } + return false; + }; + auto writes_between = [&](uint32_t lo, uint32_t hi, uint32_t reg) { + for (uint32_t pc = lo; pc < hi; pc += 4u) { + auto word = read(pc); + if (!word.has_value() || exact_instruction_writes_gpr(*word, reg)) + return true; + } + return false; + }; + // Reject a definition that a direct edge from before it can skip. Edges + // originating after the definition are harmless: that path has already + // executed the definition (Ape's self-looping out-of-range guard does this). + auto inbound_skips = [&](uint32_t def_pc, uint32_t use_pc) { + for (uint32_t source = entry; source < def_pc; source += 4u) { + auto word = read(source); + if (!word.has_value()) return true; + ExactCf cf = exact_classify_cf(source, *word); + bool direct = cf.kind == ExactCfKind::Branch || + cf.kind == ExactCfKind::Jump || + cf.kind == ExactCfKind::Jal; + if (direct && cf.target > def_pc && cf.target <= use_pc) + return true; + } + return false; + }; + + // Keep this in parity with compile_overlays.py: the canonical scheduling + // form permits zero or one NOP between LW and JR, never a raw window. + uint32_t lw_pc = jr_pc - 4u, lw_base = 0; + int32_t lw_offset = 0; + if (lw_pc < entry) return false; + auto lw_word = read(lw_pc); + if (!lw_word.has_value()) return false; + if (*lw_word == 0u) { + if (lw_pc < entry + 4u) return false; + lw_pc -= 4u; + lw_word = read(lw_pc); + } + if (!lw_word.has_value()) return false; + uint32_t lw_op = (*lw_word >> 26) & 0x3Fu; + lw_base = (*lw_word >> 21) & 0x1Fu; + uint32_t lw_rt = (*lw_word >> 16) & 0x1Fu; + if (lw_op != 0x23u || lw_rt != jr_rs || lw_base == 0u || + lw_pc < entry + 8u || in_delay_slot(lw_pc)) { + return false; + } + lw_offset = static_cast( + static_cast(*lw_word & 0xFFFFu)); + + // The address add and index scale must be the two instructions immediately + // before the load. This deliberately excludes raw-window coincidences. + uint32_t addu_pc = lw_pc - 4u; + uint32_t sll_pc = addu_pc - 4u; + auto addu_word = read(addu_pc); + auto sll_word = read(sll_pc); + if (!addu_word.has_value() || !sll_word.has_value()) return false; + uint32_t addu_op = (*addu_word >> 26) & 0x3Fu; + uint32_t addu_rs = (*addu_word >> 21) & 0x1Fu; + uint32_t addu_rt = (*addu_word >> 16) & 0x1Fu; + uint32_t addu_rd = (*addu_word >> 11) & 0x1Fu; + uint32_t addu_shamt = (*addu_word >> 6) & 0x1Fu; + uint32_t addu_fn = *addu_word & 0x3Fu; + uint32_t sll_op = (*sll_word >> 26) & 0x3Fu; + uint32_t sll_rs = (*sll_word >> 21) & 0x1Fu; + uint32_t index_reg = (*sll_word >> 16) & 0x1Fu; + uint32_t scaled_reg = (*sll_word >> 11) & 0x1Fu; + uint32_t sll_shamt = (*sll_word >> 6) & 0x1Fu; + uint32_t sll_fn = *sll_word & 0x3Fu; + if (addu_op != 0u || addu_fn != 0x21u || addu_shamt != 0u || + addu_rd != lw_base || + sll_op != 0u || sll_fn != 0u || sll_rs != 0u || sll_shamt != 2u || + index_reg == 0u || scaled_reg == 0u || + (scaled_reg != addu_rs && scaled_reg != addu_rt)) { + return false; + } + uint32_t base_reg = scaled_reg == addu_rs ? addu_rt : addu_rs; + if (base_reg == 0u || base_reg == scaled_reg || + in_delay_slot(addu_pc)) { + return false; + } + + // Exact canonical guard: sltiu; beq; nop; sll. This is the same accepted + // suffix as the Python capture verifier and excludes unrelated bounds. + if (sll_pc < entry + 12u) return false; + uint32_t guard_pc = sll_pc - 8u; + uint32_t bound_pc = sll_pc - 12u; + auto guard_word_opt = read(guard_pc); + auto bound_word = read(bound_pc); + auto guard_delay = read(sll_pc - 4u); + if (!guard_word_opt.has_value() || !bound_word.has_value() || + !guard_delay.has_value() || *guard_delay != 0u) { + return false; + } + uint32_t guard_word = *guard_word_opt; + uint32_t guard_op = (guard_word >> 26) & 0x3Fu; + uint32_t guard_rs = (guard_word >> 21) & 0x1Fu; + uint32_t guard_rt = (guard_word >> 16) & 0x1Fu; + uint32_t bound_reg = guard_rs == 0u ? guard_rt : guard_rs; + if (guard_op != 0x04u || bound_reg == 0u || + (guard_rs != 0u && guard_rt != 0u)) { + return false; + } + uint32_t bound_op = (*bound_word >> 26) & 0x3Fu; + uint32_t bound_rs = (*bound_word >> 21) & 0x1Fu; + uint32_t bound_rt = (*bound_word >> 16) & 0x1Fu; + uint32_t count = *bound_word & 0xFFFFu; + if (bound_op != 0x0Bu || bound_rs != index_reg || + bound_rt != bound_reg || count == 0u || count >= 512u || + in_delay_slot(bound_pc)) { + return false; + } + uint32_t guard_target = exact_branch_target(guard_pc, guard_word); + if ((guard_target & 3u) != 0u || guard_target < entry || + guard_target >= hard_cap || + (guard_target >= sll_pc && guard_target < jr_pc + 8u)) { + return false; + } + + // Resolve the table-base reaching definition. Cross-register constants + // (`lui rA; addiu rB,rA,lo`) are valid, but both definitions must be local, + // unskippable, outside delay slots, and unclobbered before use. + uint32_t low_pc = 0, source_reg = base_reg; + int16_t low = 0; + uint32_t lui_pc = 0, upper = 0; + for (uint32_t back = 1; back <= 32u; back++) { + if (addu_pc < entry + back * 4u) break; + uint32_t pc = addu_pc - back * 4u; + auto word = read(pc); + if (!word.has_value()) return false; + if (!exact_instruction_writes_gpr(*word, base_reg)) continue; + uint32_t op = (*word >> 26) & 0x3Fu; + uint32_t rs = (*word >> 21) & 0x1Fu; + uint32_t rt = (*word >> 16) & 0x1Fu; + if (op == 0x09u && rt == base_reg && rs != 0u) { + low_pc = pc; + source_reg = rs; + low = static_cast(*word & 0xFFFFu); + } else if (op == 0x0Fu && rs == 0u && rt == base_reg) { + lui_pc = pc; + upper = (*word & 0xFFFFu) << 16; + } else { + return false; + } + break; + } + if (low_pc != 0u) { + for (uint32_t back = 1; back <= 32u; back++) { + if (low_pc < entry + back * 4u) break; + uint32_t pc = low_pc - back * 4u; + auto word = read(pc); + if (!word.has_value()) return false; + if (!exact_instruction_writes_gpr(*word, source_reg)) continue; + uint32_t op = (*word >> 26) & 0x3Fu; + uint32_t rs = (*word >> 21) & 0x1Fu; + uint32_t rt = (*word >> 16) & 0x1Fu; + if (op != 0x0Fu || rs != 0u || rt != source_reg) return false; + lui_pc = pc; + upper = (*word & 0xFFFFu) << 16; + break; + } + } + if (lui_pc == 0u || in_delay_slot(lui_pc) || + (low_pc != 0u && in_delay_slot(low_pc)) || + inbound_skips(lui_pc, addu_pc) || + (low_pc != 0u && inbound_skips(low_pc, addu_pc)) || + has_call_between(lui_pc, addu_pc)) { + return false; + } + // No other control transfer may intervene between the proven constant and + // the canonical suffix. The bounds BEQ is the sole exception. + for (uint32_t pc = lui_pc + 4u; pc < sll_pc; pc += 4u) { + auto word = read(pc); + if (!word.has_value() || + (pc != guard_pc && is_control(pc, *word))) { + return false; + } + } + uint32_t base_def_end = low_pc != 0u ? low_pc + 4u : lui_pc + 4u; + if (writes_between(base_def_end, addu_pc, base_reg) || + (low_pc != 0u && + writes_between(lui_pc + 4u, low_pc, source_reg))) { + return false; + } + + int64_t base64 = static_cast(upper); + if (low_pc != 0u) base64 += static_cast(low); + base64 += lw_offset; + if (base64 < 0 || base64 > 0xFFFFFFFFll || + (static_cast(base64) & 3u) != 0u) { + return false; + } + uint32_t runtime_base = static_cast(base64); + uint64_t runtime_end = static_cast(runtime_base) + + static_cast(count) * 4u; + if (runtime_end > 0x100000000ull) return false; + + ExactJumpTable resolved; + resolved.table_base = runtime_base; + resolved.table_count = count; + resolved.targets.reserve(count); + for (uint32_t i = 0; i < count; i++) { + uint32_t runtime_slot = runtime_base + i * 4u; + uint32_t image_slot = mapped(runtime_slot); + if (image_slot < producer_lo || image_slot >= producer_hi || + producer_hi - image_slot < 4u) { + return false; + } + auto target_word = read(image_slot); + if (!target_word.has_value() || (*target_word & 3u) != 0u) + return false; + uint32_t runtime_target = *target_word; + uint32_t image_target = mapped(runtime_target); + if (image_target < entry || image_target >= hard_cap || + image_target < producer_lo || image_target >= producer_hi || + (image_target & 3u) != 0u) { + return false; + } + auto target_first = read(image_target); + if (!target_first.has_value() || + !exact_is_valid_mips_word(*target_first)) { + return false; + } + resolved.targets.emplace_back(runtime_target, image_target); + } + + // Later case blocks may legitimately loop to the dispatch suffix (Ape + // Escape does), but arbitrary later bytes must not create an inbound edge + // that skips the reaching definitions. Prove the exemption by walking + // only from the table's validated case targets. + std::set case_reachable; + std::queue case_work; + for (const auto& target_pair : resolved.targets) + case_work.push(target_pair.second); + while (!case_work.empty() && case_reachable.size() < 2048u) { + uint32_t pc = case_work.front(); + case_work.pop(); + if (case_reachable.count(pc) || pc < entry || pc >= hard_cap || + pc < producer_lo || pc >= producer_hi) { + continue; + } + auto word = read(pc); + if (!word.has_value() || !exact_is_valid_mips_word(*word)) continue; + case_reachable.insert(pc); + ExactCf cf = exact_classify_cf(pc, *word); + uint32_t delay = pc + 4u; + switch (cf.kind) { + case ExactCfKind::Normal: + case_work.push(pc + 4u); + break; + case ExactCfKind::Branch: + case_reachable.insert(delay); + case_work.push(pc + 8u); + case_work.push(cf.target); + break; + case ExactCfKind::BranchNever: + case_reachable.insert(delay); + case_work.push(pc + 8u); + break; + case ExactCfKind::BranchNeverLikely: + case_work.push(pc + 8u); + break; + case ExactCfKind::Jump: + case_reachable.insert(delay); + case_work.push(cf.target); + break; + case ExactCfKind::Jal: + case ExactCfKind::Jalr: + case_reachable.insert(delay); + case_work.push(pc + 8u); + break; + case ExactCfKind::JrRa: + case ExactCfKind::JrOther: + case_reachable.insert(delay); + break; + } + } + for (uint32_t source = entry; source < hard_cap; source += 4u) { + auto word = read(source); + if (!word.has_value()) return false; + ExactCf cf = exact_classify_cf(source, *word); + bool direct = cf.kind == ExactCfKind::Branch || + cf.kind == ExactCfKind::Jump || + cf.kind == ExactCfKind::Jal; + if (direct && cf.target > lui_pc && cf.target <= jr_pc && + (source < lui_pc || source > jr_pc) && + !case_reachable.count(source)) { + return false; + } + } + table = std::move(resolved); + return true; +} + FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( const std::vector& entries, const std::vector>& producer_ranges, @@ -821,89 +1168,6 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( return cross_call_allow.count(addr) != 0; }; - [[maybe_unused]] auto find_jump_table_targets = [&](uint32_t entry, uint32_t hard_cap, - uint32_t jr_pc, uint32_t jr_rs) { - std::vector targets; - uint32_t lw_base = 0xFFu; - int32_t lw_offset = 0; - uint32_t addu_cand[2] = {0xFFu, 0xFFu}; - uint32_t lui_val = 0; - int16_t addiu_val[2] = {0, 0}; - bool found_addiu[2] = {false, false}; - bool found_lui = false; - uint32_t table_count = 0; - - for (int back = 1; back <= 40; back++) { - uint32_t scan_addr = jr_pc - static_cast(back * 4); - if (scan_addr < entry) break; - auto scan_opt = exe_.read_word(scan_addr); - if (!scan_opt.has_value()) break; - - uint32_t instr = *scan_opt; - uint32_t op = (instr >> 26) & 0x3Fu; - uint32_t rs = (instr >> 21) & 0x1Fu; - uint32_t rt = (instr >> 16) & 0x1Fu; - uint32_t rd = (instr >> 11) & 0x1Fu; - uint32_t fn = instr & 0x3Fu; - - if (op == 0x23u && rt == jr_rs && lw_base == 0xFFu) { - lw_base = rs; - lw_offset = static_cast(static_cast(instr & 0xFFFFu)); - continue; - } - if (op == 0x00u && fn == 0x21u && rd == lw_base && lw_base != 0xFFu && - addu_cand[0] == 0xFFu) { - addu_cand[0] = rs; - addu_cand[1] = rt; - continue; - } - if (op == 0x09u && addu_cand[0] != 0xFFu) { - for (int c = 0; c < 2; c++) { - if (!found_addiu[c] && addu_cand[c] != 0xFFu && - rs == addu_cand[c] && rt == addu_cand[c]) { - addiu_val[c] = static_cast(instr & 0xFFFFu); - found_addiu[c] = true; - break; - } - } - continue; - } - if (op == 0x0Fu && addu_cand[0] != 0xFFu && !found_lui) { - for (int c = 0; c < 2; c++) { - if (addu_cand[c] != 0xFFu && rt == addu_cand[c]) { - lui_val = (instr & 0xFFFFu) << 16; - if (!found_addiu[c]) addiu_val[c] = 0; - addiu_val[0] = addiu_val[c]; - found_addiu[0] = found_addiu[c]; - found_lui = true; - break; - } - } - continue; - } - if (op == 0x0Bu && table_count == 0) { - table_count = instr & 0xFFFFu; - continue; - } - if (found_lui && table_count != 0) break; - } - - if (!found_lui || table_count == 0 || table_count >= 512) return targets; - - uint32_t table_base = lui_val + - (found_addiu[0] ? static_cast(static_cast(addiu_val[0])) : 0u) + - static_cast(lw_offset); - for (uint32_t i = 0; i < table_count; i++) { - auto entry_opt = exe_.read_word(table_base + i * 4u); - if (!entry_opt.has_value()) continue; - uint32_t target = *entry_opt; - if (target >= entry && target < hard_cap && in_exe(target)) { - targets.push_back(target); - } - } - return targets; - }; - auto walk = [&](uint32_t entry, uint32_t hard_cap) { ExactWalkResult wr; std::queue work; @@ -999,10 +1263,32 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( std::make_pair(pc, true)); } } else { - // Fail closed: the former 40-word raw backscan could - // combine unrelated loads/constants into a synthetic - // jump table. Unresolved JR cases stay interpreted until - // register/control dependence is proven. + ExactJumpTable table; + uint32_t producer_lo = exe_.header.load_address; + uint32_t producer_hi = exe_.end_address(); + if (entry_producer >= 0) { + producer_lo = producer_ranges[entry_producer].first; + producer_hi = producer_ranges[entry_producer].second; + } + if (resolve_exact_bounded_jump_table( + exe_, entry, hard_cap, pc, jr_rs, table, + nullptr, producer_lo, producer_hi)) { + bool all_owned = std::all_of( + table.targets.begin(), table.targets.end(), + [&](const auto& target_pair) { + return in_function(target_pair.second); + }); + if (all_owned) { + for (const auto& target_pair : table.targets) { + uint32_t target_pc = target_pair.second; + wr.jump_table_targets.insert(target_pc); + work.push(target_pc); + } + } + } + // Otherwise fail closed. Unresolved JR cases remain + // interpreted until the complete dependency chain and + // every table entry are proven. } } break; @@ -1117,6 +1403,11 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( if (func.end_addr <= func.start_addr) func.end_addr = func.start_addr + 4u; func.size = func.end_addr - func.start_addr; func.name = fmt::format("func_{:08X}", func.start_addr); + if (producer_for(entry) >= 0) { + const auto& producer = producer_ranges[producer_for(entry)]; + func.producer_lo = producer.first; + func.producer_hi = producer.second; + } auto first_instr = exe_.read_word(func.start_addr); if (first_instr.has_value()) { @@ -1558,72 +1849,17 @@ FunctionAnalysisResult FunctionAnalyzer::analyze() { }; return valid[(instr >> 26) & 0x3Fu]; }; - // Resolve a `jr $rN` jump table by the standard Psy-Q pattern - // (sltiu bound; sll idx,2; lui/addiu base; addu; lw target; jr). Returns - // true and fills `targets` (in-function entries) when a table is - // recognized; false when the indirect jump is unresolved. + // Mirror exact-entry discovery's strict bounded-switch recognizer so the + // final extent cannot disagree about which JR targets are code. auto resolve_jt_p3 = [&](uint32_t entry, uint32_t hard_cap, uint32_t jr_pc, uint32_t jr_rs, std::vector& targets) -> bool { - uint32_t lw_base = 0xFFu; int32_t lw_offset = 0; - uint32_t addu_cand[2] = {0xFFu, 0xFFu}; - uint32_t lui_val = 0; int16_t addiu_val[2] = {0, 0}; - bool found_addiu[2] = {false, false}; - bool found_lui = false; uint32_t table_count = 0; - for (int back = 1; back <= 40; back++) { - uint32_t scan_addr = jr_pc - static_cast(back * 4); - if (scan_addr < entry) break; - auto scan_opt = exe_.read_word(scan_addr); - if (!scan_opt.has_value()) break; - uint32_t instr = *scan_opt; - uint32_t op = (instr >> 26) & 0x3Fu; - uint32_t rs = (instr >> 21) & 0x1Fu; - uint32_t rt = (instr >> 16) & 0x1Fu; - uint32_t rd = (instr >> 11) & 0x1Fu; - uint32_t fn = instr & 0x3Fu; - if (op == 0x23u && rt == jr_rs && lw_base == 0xFFu) { - lw_base = rs; - lw_offset = static_cast(static_cast(instr & 0xFFFFu)); - continue; - } - if (op == 0x00u && fn == 0x21u && rd == lw_base && lw_base != 0xFFu && - addu_cand[0] == 0xFFu) { - addu_cand[0] = rs; addu_cand[1] = rt; continue; - } - if (op == 0x09u && addu_cand[0] != 0xFFu) { - for (int c = 0; c < 2; c++) { - if (!found_addiu[c] && addu_cand[c] != 0xFFu && - rs == addu_cand[c] && rt == addu_cand[c]) { - addiu_val[c] = static_cast(instr & 0xFFFFu); - found_addiu[c] = true; break; - } - } - continue; - } - if (op == 0x0Fu && addu_cand[0] != 0xFFu && !found_lui) { - for (int c = 0; c < 2; c++) { - if (addu_cand[c] != 0xFFu && rt == addu_cand[c]) { - lui_val = (instr & 0xFFFFu) << 16; - if (!found_addiu[c]) addiu_val[c] = 0; - addiu_val[0] = addiu_val[c]; - found_addiu[0] = found_addiu[c]; - found_lui = true; break; - } - } - continue; - } - if (op == 0x0Bu && table_count == 0) { table_count = instr & 0xFFFFu; continue; } - if (found_lui && table_count != 0) break; - } - if (!found_lui || table_count == 0 || table_count >= 512) return false; - uint32_t table_base = lui_val + - (found_addiu[0] ? static_cast(static_cast(addiu_val[0])) : 0u) + - static_cast(lw_offset); - for (uint32_t k = 0; k < table_count; k++) { - auto entry_opt = exe_.read_word(table_base + k * 4u); - if (!entry_opt.has_value()) continue; - uint32_t target = *entry_opt; - if (target >= entry && target < hard_cap && in_exe_p3(target)) targets.push_back(target); + ExactJumpTable table; + if (!resolve_exact_bounded_jump_table( + exe_, entry, hard_cap, jr_pc, jr_rs, table)) { + return false; } + for (const auto& target_pair : table.targets) + targets.push_back(target_pair.second); return true; }; struct ExtentP3 { uint32_t terminus; bool clean_code; bool hit_invalid; bool resolved_jt; bool unresolved_indirect; }; diff --git a/recompiler/src/main_psx.cpp b/recompiler/src/main_psx.cpp index f2dc0fb3..d97ab2d0 100644 --- a/recompiler/src/main_psx.cpp +++ b/recompiler/src/main_psx.cpp @@ -48,6 +48,15 @@ void materialize_alias_groups(PSXRecomp::FunctionAnalysisResult& result, by_host[ae.host_start].push_back(&ae); } for (const auto& [host_start, group] : by_host) { + uint32_t producer_lo = 0; + uint32_t producer_hi = 0; + for (const auto& function : result.functions) { + if (function.start_addr == host_start) { + producer_lo = function.producer_lo; + producer_hi = function.producer_hi; + break; + } + } std::vector entries; for (const AliasEntry* ae : group) entries.push_back(ae->addr); for (const AliasEntry* ae : group) { @@ -62,6 +71,8 @@ void materialize_alias_groups(PSXRecomp::FunctionAnalysisResult& result, af.is_data_section = false; af.alias_walk_lo = ae->host_start; af.alias_group_entries = entries; + af.producer_lo = producer_lo; + af.producer_hi = producer_hi; result.functions.push_back(af); } } diff --git a/recompiler/tests/reachable_discovery_test.cpp b/recompiler/tests/reachable_discovery_test.cpp index 805fd93b..38b7b7f2 100644 --- a/recompiler/tests/reachable_discovery_test.cpp +++ b/recompiler/tests/reachable_discovery_test.cpp @@ -54,6 +54,54 @@ std::set starts(const PSXRecomp::FunctionAnalysisResult& result) { return out; } +struct ApeSwitchFixture { + std::vector image; + uint32_t entry = kLoad + 0x500u; + uint32_t jr_pc = kLoad + 0x528u; + uint32_t table = kLoad + 0xA00u; + std::vector cases = { + kLoad + 0x540u, kLoad + 0x550u, kLoad + 0x560u}; +}; + +ApeSwitchFixture make_ape_switch() { + ApeSwitchFixture f; + f.image = make_exe_buffer(0x1000u, f.entry); + const size_t text = 2048u; + put32(f.image, text + 0x500, 0x3C088001u); // lui t0,0x8001 + put32(f.image, text + 0x504, 0x25100A00u); // addiu s0,t0,0x0a00 + put32(f.image, text + 0x508, 0x00000000u); + put32(f.image, text + 0x50C, 0x2C620003u); // sltiu v0,v1,3 + put32(f.image, text + 0x510, 0x1040001Bu); // beq v0,zero,+0x580 + put32(f.image, text + 0x514, 0x00000000u); + put32(f.image, text + 0x518, 0x00031080u); // sll v0,v1,2 + put32(f.image, text + 0x51C, 0x00501021u); // addu v0,v0,s0 + put32(f.image, text + 0x520, 0x8C420000u); // lw v0,0(v0) + put32(f.image, text + 0x524, 0x00000000u); + put32(f.image, text + 0x528, 0x00400008u); // jr v0 + put32(f.image, text + 0x52C, 0x00000000u); + for (size_t i = 0; i < f.cases.size(); i++) { + size_t off = static_cast(f.cases[i] - kLoad); + put32(f.image, text + off, 0x24020001u + static_cast(i)); + put32(f.image, text + off + 4u, 0x03E00008u); + put32(f.image, text + off + 8u, 0x00000000u); + put32(f.image, text + 0xA00u + i * 4u, f.cases[i]); + } + put32(f.image, text + 0x580, 0x03E00008u); + put32(f.image, text + 0x584, 0x00000000u); + return f; +} + +bool resolves_ape_switch(const ApeSwitchFixture& f, + uint32_t entry, + uint32_t producer_lo = kLoad, + uint32_t producer_hi = kLoad + 0x1000u) { + auto exe = parse(f.image); + PSXRecomp::ExactJumpTable table; + return PSXRecomp::resolve_exact_bounded_jump_table( + exe, entry, kLoad + 0x1000u, f.jr_pc, 2u, table, + nullptr, producer_lo, producer_hi); +} + } // namespace int main() { @@ -488,6 +536,127 @@ int main() { !call_delay.count(kLoad + 0x400), "indirect resolver rejects a LUI in a call delay slot"); + // Ape Escape's save-screen dispatcher builds its table base across + // registers (lui t0; addiu s0,t0,lo). Prove the complete bounded switch, + // then make every adversarial mutation fail closed as one whole table. + auto ape_switch = make_ape_switch(); + auto ape_switch_exe = parse(ape_switch.image); + PSXRecomp::ExactJumpTable ape_table; + CHECK(PSXRecomp::resolve_exact_bounded_jump_table( + ape_switch_exe, ape_switch.entry, kLoad + 0x1000u, + ape_switch.jr_pc, 2u, ape_table) && + ape_table.table_base == ape_switch.table && + ape_table.table_count == ape_switch.cases.size() && + ape_table.targets.size() == ape_switch.cases.size(), + "Ape cross-register bounded switch resolves exactly"); + PSXRecomp::FunctionAnalyzer ape_switch_analyzer(ape_switch_exe); + const auto ape_result = ape_switch_analyzer.analyze_exact_entries( + {ape_switch.entry}); + bool all_ape_cases_reachable = true; + for (uint32_t target : ape_switch.cases) + all_ape_cases_reachable &= ape_result.exact_reachable_pcs.count(target) != 0; + CHECK(all_ape_cases_reachable, + "exact reachable discovery follows every validated Ape case"); + + auto ori_switch = make_ape_switch(); + put32(ori_switch.image, text + 0x504, 0x35100A00u); // ori s0,t0,0x0a00 + CHECK(!resolves_ape_switch(ori_switch, ori_switch.entry), + "unproven ORI table constants fail closed"); + + auto no_lw_nop_switch = make_ape_switch(); + no_lw_nop_switch.jr_pc = kLoad + 0x524u; + put32(no_lw_nop_switch.image, text + 0x524, 0x00400008u); + put32(no_lw_nop_switch.image, text + 0x528, 0x00000000u); + CHECK(resolves_ape_switch(no_lw_nop_switch, no_lw_nop_switch.entry), + "zero-NOP LW-to-JR schedule matches Python parity"); + + auto two_lw_nop_switch = make_ape_switch(); + two_lw_nop_switch.jr_pc = kLoad + 0x52Cu; + put32(two_lw_nop_switch.image, text + 0x528, 0x00000000u); + put32(two_lw_nop_switch.image, text + 0x52C, 0x00400008u); + CHECK(!resolves_ape_switch(two_lw_nop_switch, + two_lw_nop_switch.entry), + "two-NOP LW-to-JR schedule fails closed"); + + auto wrong_jr_switch = make_ape_switch(); + put32(wrong_jr_switch.image, text + 0x528, + 0x00400808u); // reserved rd field makes this noncanonical JR + CHECK(!resolves_ape_switch(wrong_jr_switch, wrong_jr_switch.entry), + "jump-table proof checks the exact JR encoding"); + + auto noncanonical_lui_switch = make_ape_switch(); + put32(noncanonical_lui_switch.image, text + 0x500, + 0x3C288001u); // LUI's reserved rs field is nonzero + CHECK(!resolves_ape_switch(noncanonical_lui_switch, + noncanonical_lui_switch.entry), + "jump-table proof rejects noncanonical LUI reserved fields"); + + auto cross_producer_switch = make_ape_switch(); + CHECK(!resolves_ape_switch(cross_producer_switch, + cross_producer_switch.entry, + kLoad + 0x400u, kLoad + 0x900u), + "producer A cannot borrow table slots from adjacent producer B"); + + auto producer_case_escape = make_ape_switch(); + put32(producer_case_escape.image, text + 0x540, + 0x08000000u | (((kLoad + 0xB20u) >> 2) & 0x03FFFFFFu)); + put32(producer_case_escape.image, text + 0x544, 0x00000000u); + put32(producer_case_escape.image, text + 0xB20, + 0x08000000u | (((kLoad + 0x518u) >> 2) & 0x03FFFFFFu)); + put32(producer_case_escape.image, text + 0xB24, 0x00000000u); + CHECK(!resolves_ape_switch(producer_case_escape, + producer_case_escape.entry, + kLoad + 0x400u, kLoad + 0xA20u), + "case walk cannot escape through an adjacent producer"); + auto cross_producer_exe = parse(cross_producer_switch.image); + PSXRecomp::FunctionAnalyzer cross_producer_analyzer(cross_producer_exe); + const auto cross_producer_result = + cross_producer_analyzer.analyze_exact_entries( + {cross_producer_switch.entry}, + {{kLoad + 0x400u, kLoad + 0x900u}, + {kLoad + 0x900u, kLoad + 0x1000u}}); + CHECK(!cross_producer_result.exact_reachable_pcs.count(kLoad + 0x540u), + "reachable discovery rejects a cross-producer table"); + CHECK(!cross_producer_result.functions.empty() && + cross_producer_result.functions.front().producer_lo == + kLoad + 0x400u && + cross_producer_result.functions.front().producer_hi == + kLoad + 0x900u, + "exact functions retain producer bounds for code generation"); + + auto invalid_target_switch = make_ape_switch(); + put32(invalid_target_switch.image, text + 0x550, 0xFFFFFFFFu); + CHECK(!resolves_ape_switch(invalid_target_switch, + invalid_target_switch.entry), + "one invalid target instruction rejects the entire table"); + + auto clobbered_switch = make_ape_switch(); + put32(clobbered_switch.image, text + 0x508, + 0x24100000u); // addiu s0,zero,0 + CHECK(!resolves_ape_switch(clobbered_switch, clobbered_switch.entry), + "table-base clobber rejects the bounded switch"); + + auto skipped_def_switch = make_ape_switch(); + put32(skipped_def_switch.image, text + 0x700, + 0x08000000u | + (((kLoad + 0x504u) >> 2) & 0x03FFFFFFu)); + put32(skipped_def_switch.image, text + 0x704, 0x00000000u); + CHECK(!resolves_ape_switch(skipped_def_switch, + skipped_def_switch.entry), + "later host bytes cannot jump into and skip a protected definition"); + + auto bad_bound_switch = make_ape_switch(); + put32(bad_bound_switch.image, text + 0x50C, + 0x2C820003u); // sltiu v0,a0,3: wrong index dependency + CHECK(!resolves_ape_switch(bad_bound_switch, bad_bound_switch.entry), + "unrelated SLTIU bound cannot validate a jump table"); + + auto outside_target_switch = make_ape_switch(); + put32(outside_target_switch.image, text + 0xA04, kLoad + 0x2000u); + CHECK(!resolves_ape_switch(outside_target_switch, + outside_target_switch.entry), + "one out-of-image table target rejects the entire table"); + // A packed BIOS thunk is also a backward-scan boundary. The frameless // leaf after its delay slot must not be swallowed into the thunk run. auto thunk_image = make_exe_buffer(0x1000); diff --git a/recompiler/tests/recompiler_patch_test.cpp b/recompiler/tests/recompiler_patch_test.cpp index acfe0f64..94c89327 100644 --- a/recompiler/tests/recompiler_patch_test.cpp +++ b/recompiler/tests/recompiler_patch_test.cpp @@ -47,6 +47,15 @@ void append_word(std::vector& bytes, uint32_t word) { bytes.push_back(static_cast(word >> 24)); } +void write_word(PSXRecomp::PS1Executable& exe, uint32_t address, + uint32_t word) { + const size_t offset = static_cast(address - exe.header.load_address); + exe.code_data[offset + 0] = static_cast(word); + exe.code_data[offset + 1] = static_cast(word >> 8); + exe.code_data[offset + 2] = static_cast(word >> 16); + exe.code_data[offset + 3] = static_cast(word >> 24); +} + std::string generate_first_instruction(uint32_t first_word, std::vector patches, bool overlay_mode) { @@ -245,6 +254,99 @@ void codegen_tests() { "config merge rejects cross-config ID conflicts"); } +void jump_table_producer_codegen_test() { + constexpr uint32_t base = 0x80010000u; + constexpr uint32_t entry = base + 0x500u; + PSXRecomp::PS1Executable exe{}; + exe.header.load_address = base; + exe.header.initial_pc = entry; + exe.header.file_size = 0x1000u; + exe.code_data.resize(0x1000u, 0u); + + write_word(exe, base + 0x500u, 0x3C088001u); // lui t0,0x8001 + write_word(exe, base + 0x504u, 0x25100A00u); // addiu s0,t0,0x0a00 + write_word(exe, base + 0x50Cu, 0x2C620003u); // sltiu v0,v1,3 + write_word(exe, base + 0x510u, 0x1040001Bu); // beq v0,zero,+0x580 + write_word(exe, base + 0x518u, 0x00031080u); // sll v0,v1,2 + write_word(exe, base + 0x51Cu, 0x00501021u); // addu v0,v0,s0 + write_word(exe, base + 0x520u, 0x8C420000u); // lw v0,0(v0) + write_word(exe, base + 0x528u, 0x00400008u); // jr v0 + const uint32_t cases[] = { + base + 0x540u, base + 0x550u, base + 0x560u}; + for (size_t i = 0; i < 3u; i++) { + write_word(exe, cases[i], 0x24020001u + static_cast(i)); + write_word(exe, cases[i] + 4u, 0x03E00008u); + write_word(exe, base + 0xA00u + static_cast(i * 4u), + cases[i]); + } + write_word(exe, base + 0x580u, 0x03E00008u); + write_word(exe, base + 0x590u, + 0x08000000u | ((entry >> 2) & 0x03FFFFFFu)); + + PSXRecomp::Function function{}; + function.start_addr = entry; + function.end_addr = base + 0x588u; + function.size = function.end_addr - function.start_addr; + function.name = "producer_switch"; + function.producer_lo = base + 0x400u; + function.producer_hi = base + 0x900u; // table belongs to adjacent producer + + PSXRecomp::ControlFlowAnalyzer analyzer(exe); + const auto bounded_cfg = analyzer.analyze_function(function); + PSXRecomp::CodeGenerator bounded_generator(exe); + const std::string bounded = + bounded_generator.generate_function(function, bounded_cfg).full_code; + check(bounded.find("/* jump table") == std::string::npos, + "codegen cannot resurrect an adjacent-producer jump table"); + + function.producer_lo = 0u; + function.producer_hi = 0u; + const auto single_image_cfg = analyzer.analyze_function(function); + PSXRecomp::CodeGenerator single_image_generator(exe); + const std::string single_image = single_image_generator + .generate_function(function, single_image_cfg).full_code; + check(single_image.find("/* jump table") != std::string::npos, + "codegen still emits the same table for a single owned image"); + + // Exercise the actual overlapping-alias emitter path. The alias begins at + // a later block which jumps back through the host's complete table setup, + // so an unbounded alias really would expose the adjacent producer bytes. + // Producer ownership must survive Function -> alias CFG -> shared body. + PSXRecomp::Function alias = function; + alias.start_addr = base + 0x590u; + alias.end_addr = base + 0x598u; + alias.size = alias.end_addr - alias.start_addr; + alias.name = "producer_switch_alias"; + alias.alias_walk_lo = entry; + alias.alias_group_entries = {alias.start_addr}; + alias.producer_lo = base + 0x400u; + alias.producer_hi = base + 0x900u; + + const auto alias_cfg = analyzer.analyze_function(alias); + check(alias_cfg.producer_lo == alias.producer_lo && + alias_cfg.producer_hi == alias.producer_hi, + "alias CFG retains its host producer bounds"); + PSXRecomp::CodeGenerator alias_generator(exe); + const auto alias_generated = alias_generator.generate_alias_group( + {&alias}, alias_cfg, ""); + check(!alias_generated.empty() && + alias_generated.front().full_code.find("/* jump table") == + std::string::npos, + "alias emitter cannot expose an adjacent producer jump table"); + + alias.producer_lo = 0u; + alias.producer_hi = 0u; + const auto unbounded_alias_cfg = analyzer.analyze_function(alias); + PSXRecomp::CodeGenerator unbounded_alias_generator(exe); + const auto unbounded_alias_generated = + unbounded_alias_generator.generate_alias_group( + {&alias}, unbounded_alias_cfg, ""); + check(!unbounded_alias_generated.empty() && + unbounded_alias_generated.front().full_code.find("/* jump table") != + std::string::npos, + "alias regression fixture reaches the table when ownership is absent"); +} + } // namespace int main() { @@ -257,6 +359,7 @@ int main() { parser_tests(root); capture_history_config_tests(root); codegen_tests(); + jump_table_producer_codegen_test(); } catch (const std::exception& e) { fmt::print(stderr, "FAIL unexpected exception: {}\n", e.what()); ++failures; diff --git a/recompiler/tests/test_aot_base_recovery.py b/recompiler/tests/test_aot_base_recovery.py index 7c5289f7..f0671080 100644 --- a/recompiler/tests/test_aot_base_recovery.py +++ b/recompiler/tests/test_aot_base_recovery.py @@ -182,6 +182,104 @@ def main(): assert {x['base'] for x in recovered} == {archive_base} assert all(len(x['direct_seeds']) == 8 for x in recovered) + # Ape HED member 0186 shape: the switch host is not directly called, but a + # return-adjacent framed prologue proves its real root. Its bounded table + # labels and reachable JAL continuation are dispatch evidence only. + ape_base = 0x80136000 + ape_members = [] + anchor_offsets = (0x80, 0x94, 0xB0, 0xD8, + 0x110, 0x154, 0x1A8, 0x210) + table_values = ([ape_base + 0x22E4, ape_base + 0x2354, + ape_base + 0x22F4, ape_base + 0x22B8] + + [ape_base + 0x2354] * 2 + + [ape_base + 0x2304, ape_base + 0x2354, + ape_base + 0x2314, ape_base + 0x2354, + ape_base + 0x2334, ape_base + 0x2354, + ape_base + 0x2344, ape_base + 0x2354] + + [ape_base + 0x22B8, ape_base + 0x2354, + ape_base + 0x2354, ape_base + 0x2324] + + [ape_base + 0x22B8] * 28) + assert len(table_values) == 46 + for ident in range(1, 5): + body = bytearray(0x5000) + for index, target_off in enumerate(anchor_offsets): + target = ape_base + target_off + struct.pack_into('> 2) & 0x03FFFFFF)) + struct.pack_into('> 2) & 0x03FFFFFF)) + put(producer_escape, 0x6C, 0x00000000) + put(producer_escape, 0x260, + 0x08000000 | (((LOAD + 0x50) >> 2) & 0x03FFFFFF)) + put(producer_escape, 0x264, 0x00000000) + assert MOD._find_jump_table_targets( + bytes(producer_escape), LOAD, len(data), entry, LOAD + 0x300, + LOAD + 0x60, 2, (LOAD, LOAD + 0x240)) == set() + + ori_base = bytearray(data) + put(ori_base, 0x3C, 0x34500200) # ori s0,v0,0x200 is outside grammar + assert MOD._find_jump_table_targets( + bytes(ori_base), LOAD, len(data), entry, LOAD + 0x100, + LOAD + 0x60, 2) == set() + + skipped_constant = bytearray(data) + put(skipped_constant, 0x34, 0x08000000 | + (((LOAD + 0x40) >> 2) & 0x03FFFFFF)) + assert MOD._find_jump_table_targets( + bytes(skipped_constant), LOAD, len(data), entry, LOAD + 0x100, + LOAD + 0x60, 2) == set() + + later_inbound = bytearray(data) + put(later_inbound, 0xA0, 0x08000000 | + (((LOAD + 0x50) >> 2) & 0x03FFFFFF)) + assert MOD._find_jump_table_targets( + bytes(later_inbound), LOAD, len(data), entry, LOAD + 0x100, + LOAD + 0x60, 2) == set() + + foreign_target = bytearray(data) + put(foreign_target, table - LOAD, LOAD + 0x180) + assert MOD._find_jump_table_targets( + bytes(foreign_target), LOAD, len(data), entry, LOAD + 0x100, + LOAD + 0x60, 2) == set() + assert MOD._find_jump_table_targets( + bytes(data), LOAD, len(data), entry, LOAD + 0x100, + LOAD + 0x60, 2, (LOAD, LOAD + 0x180)) == set() + + def check_composite_call_boundaries(): data = bytearray(0x100) cross_target = LOAD + 0x50 @@ -487,7 +663,7 @@ def check_recomputed_evidence_and_unconditional_branch(): assert MOD._classify_cf(LOAD, (0x17 << 26) | 0x1F) == ( 'branch_never_likely', 0) - # Unresolved JR-table inference is disabled until dependence is proven. + # A non-canonical collection of branch-shaped words remains unresolved. assert MOD._find_jump_table_targets( bytes(branch), LOAD, len(branch), LOAD, LOAD + len(branch), LOAD + 0x20, 8) == set() @@ -607,6 +783,8 @@ def check_optional_enrichment_fallback(): 'end': f'0x{LOAD + 0x2C:08X}', }], '_prior_aliases': [(direct, LOAD, LOAD + 0x2C)], + 'static_jump_table_proofs': [{'host_entry': f'0x{LOAD:08X}'}], + 'static_call_continuation_pcs': [f'0x{LOAD + 0x10:08X}'], 'optional_enrichment_fallback_entry_pcs': [f'0x{direct:08X}'], } fallback = MOD.optional_enrichment_fallback_capture(cap) @@ -616,6 +794,8 @@ def check_optional_enrichment_fallback(): assert fallback['static_discovery_entry_pcs'] == encoded assert fallback['seeds'] == encoded assert 'static_alias_ranges' not in fallback + assert 'static_jump_table_proofs' not in fallback + assert 'static_call_continuation_pcs' not in fallback assert '_prior_aliases' not in fallback assert 'optional_enrichment_fallback_entry_pcs' not in fallback assert MOD.optional_enrichment_fallback_capture(fallback) is None @@ -1246,6 +1426,7 @@ def main(): raise SystemExit(f"recompiler not found: {args.recompiler}") check_composite_call_boundaries() + check_bounded_jump_table_discovery() check_static_discovery_provenance() check_owned_direct_call_is_interior() check_discovered_host_owns_same_round_call_target() diff --git a/tools/aot_overlay_spike/extract_generic.py b/tools/aot_overlay_spike/extract_generic.py index 872dd3ef..0c37b9cb 100644 --- a/tools/aot_overlay_spike/extract_generic.py +++ b/tools/aot_overlay_spike/extract_generic.py @@ -382,14 +382,92 @@ def split_indexed_archive(data, alignment=0x800): return None return members +def return_adjacent_framed_entries(data, base): + """Strong framed roots immediately following a completed function. + + Mixed archive members may contain prologue-shaped data, so a stack-frame + instruction alone is insufficient. Require a preceding ``jr ra`` and its + arbitrary architectural delay slot, at most six alignment NOPs, plus an + in-frame save of ``ra`` before the new function's first control transfer. + Every stack-relative word save observed in that prefix must fit the frame. + """ + entries=set() + for off in prologue_offsets(data): + word=_word(data,off) + frame_imm=word&0xFFFF + frame_size=0x10000-frame_imm + if not (8 <= frame_size <= 0x400 and (frame_size&3)==0): + continue + boundary=False + for padding in range(7): + jr_off=off-8-padding*4 + if _word(data,jr_off)!=0x03E00008: + continue + if all(_word(data,p)==0 for p in range(jr_off+8,off,4)): + boundary=True + break + if not boundary: + continue + saved_ra=False + consistent=True + for pc in range(off+4,min(len(data),off+4+12*4),4): + candidate=_word(data,pc) + if candidate is None: + consistent=False + break + if _is_control_flow_word(candidate): + break + op=(candidate>>26)&0x3F + rs=(candidate>>21)&0x1F + rt=(candidate>>16)&0x1F + if op==0x2B and rs==29: + displacement=candidate&0xFFFF + if displacement&0x8000: + displacement-=0x10000 + if displacement<0 or displacement+4>frame_size or displacement&3: + consistent=False + break + if rt==31: + saved_ra=True + if consistent and saved_ra: + entries.add(base+off) + return sorted(entries) + +def _is_control_flow_word(word): + """Use the shared overlay decoder without exposing another public API.""" + return co._is_control_flow(word) + +def reachable_static_dispatches(data,base,roots): + """Dispatch-only PCs plus inspectable indirect-jump provenance.""" + roots=sorted({root for root in roots + if base <= root < base+len(data) and (root&3)==0}) + targets=set() + continuations=set() + proofs=[] + producer=((base,base+len(data)),) + for index,entry in enumerate(roots): + hard_cap=roots[index+1] if index+1=8 agreeing jal/prologue pairs and a 2x vote peak. A base becomes trusted only when at least two independent members anchor it. - Other members may reuse that exact consensus with >=2 agreeing pairs. Only - locally callable direct-JAL targets become roots; pointer tables and apparent - prologues are deliberately insufficient inside a mixed code/data archive. + Other members may reuse that exact consensus with >=2 agreeing pairs. + Locally callable direct-JAL targets and return-adjacent framed functions are + roots. Computed-switch destinations are retained separately as dispatch-only + evidence; generic pointer tables and arbitrary prologues remain insufficient + inside a mixed code/data archive. """ from collections import Counter if not members: @@ -417,12 +495,18 @@ def recover_consensus_members(members, base_lo, base_hi=0x80200000): if score < 2 or score < runner*2: continue direct=direct_jal_roots(body,base) - seeds=list(direct) if len(direct) < 2: continue + framed=return_adjacent_framed_entries(body,base) + seeds=sorted(set(direct)|set(framed)) + static_dispatch=reachable_static_dispatches(body,base,seeds) recovered.append({ 'id':ident, 'file_offset':off, 'data':body, 'base':base, 'score':score, 'runner':runner, 'direct_seeds':direct, + 'return_framed_seeds':framed, + 'static_dispatch_pcs':static_dispatch['dispatch_pcs'], + 'jump_table_proofs':static_dispatch['jump_table_proofs'], + 'call_continuation_pcs':static_dispatch['call_continuation_pcs'], 'seeds':seeds, }) return recovered @@ -609,13 +693,15 @@ def filter_full_discovery_seeds(body, base, candidates, declared_entry): def enrich_positioned_member(member, recompiler, tmp): """Add normal-mode entries/aliases to a consensus-positioned member. - Direct JAL targets established the member's link base and remain its only - hard roots. Normal mode contributes decoder-classified candidates and exact - overlapping alias recipes; the unproven image-start fallback is removed. + Direct JAL targets established the member's link base. Those targets plus + strict return-adjacent framed roots remain the hard-root fallback. Normal + mode contributes decoder-classified candidates and exact overlapping alias + recipes; the unproven image-start fallback is removed. Downstream overlay compilation still re-walks and audits every emitted byte. """ body=member['data']; base=member['base'] direct=member['direct_seeds'] + roots=member['seeds'] if not direct: return member wrapped=make_psx_exe(body,base,direct[0]) @@ -631,15 +717,16 @@ def enrich_positioned_member(member, recompiler, tmp): if base <= alias[0] < hi and base <= alias[1] < alias[2] <= hi and alias[0] != base and - not any(alias[1] < root < alias[2] for root in direct)] + not any(alias[1] < root < alias[2] for root in roots)] out=dict(member) - out['seeds']=sorted(set(discovered)|set(direct)) + out['seeds']=sorted(set(discovered)|set(roots)) out['static_alias_ranges']=sorted(set(aliases)) out['normal_seed_count']=len(discovered) return out def rec(load_addr, data, seeds, dispatch_extra=None, producer_ranges=None, - static_discovery=None, static_alias_ranges=None): + static_discovery=None, static_alias_ranges=None, + jump_table_proofs=None, call_continuations=None): # function_entry_pcs = prologue-scanned function starts (walk roots). # dispatch_entry_pcs = those PLUS any extra dispatch targets (e.g. the # overlay's own export table). compile_overlays promotes clean starts to @@ -664,8 +751,28 @@ def rec(load_addr, data, seeds, dispatch_extra=None, producer_ranges=None, {"entry":f"0x{entry:08X}", "start":f"0x{lo:08X}", "end":f"0x{hi:08X}"} for entry,lo,hi in sorted(set(static_alias_ranges))] + if jump_table_proofs: + out['static_jump_table_proofs']=[{ + 'host_entry':f"0x{proof['host_entry']:08X}", + 'jr_pc':f"0x{proof['jr_pc']:08X}", + 'table_base':f"0x{proof['table_base']:08X}", + 'count':proof['count'], + 'targets':[f"0x{target:08X}" for target in proof['targets']], + } for proof in jump_table_proofs] + if call_continuations: + out['static_call_continuation_pcs']=[ + f"0x{addr:08X}" for addr in sorted(set(call_continuations))] return out +def add_consensus_fallback(record,member): + """Fail closed to base-proving direct calls if enrichment is rejected.""" + if (member.get('normal_seed_count') or + member.get('return_framed_seeds') or + member.get('static_dispatch_pcs')): + record['optional_enrichment_fallback_entry_pcs'] = [ + f"0x{addr:08X}" for addr in member['direct_seeds']] + return record + def bios_resident_records(bios_path=None, manifest_path=None): """Build guarded captures for BIOS-installed RAM code known by exact hash. @@ -956,12 +1063,13 @@ def main(): producer_lo=max(base,page_base) record=rec( page_base,region,member['seeds'], + dispatch_extra=member.get('static_dispatch_pcs'), producer_ranges=((producer_lo,base+len(body)),), static_discovery=member['direct_seeds'], - static_alias_ranges=member.get('static_alias_ranges')) - if member.get('normal_seed_count'): - record['optional_enrichment_fallback_entry_pcs'] = [ - f"0x{addr:08X}" for addr in member['direct_seeds']] + static_alias_ranges=member.get('static_alias_ranges'), + jump_table_proofs=member.get('jump_table_proofs'), + call_continuations=member.get('call_continuation_pcs')) + add_consensus_fallback(record,member) record['producer_name']=( f"{p}:member_{member['id']:04X}@{member['file_offset']:08X}") records.append(record) @@ -971,6 +1079,8 @@ def main(): f"@0x{base:08X}, region@0x{page_base:08X}, " f"trusted-fit={member['score']}:{member['runner']}, " f"direct seeds={len(member['direct_seeds'])}, " + f"framed roots={len(member.get('return_framed_seeds',()))}, " + f"static dispatch={len(member.get('static_dispatch_pcs',()))}, " f"normal seeds={member.get('normal_seed_count',0)}, " f"aliases={len(member.get('static_alias_ranges',()))}, " f"normal reject={member.get('normal_rejected','none')}, " @@ -990,8 +1100,12 @@ def main(): producer_lo=max(base,page_base) record=rec( page_base,region,member['seeds'], + dispatch_extra=member.get('static_dispatch_pcs'), producer_ranges=((producer_lo,base+len(body)),), - static_discovery=member['direct_seeds']) + static_discovery=member['direct_seeds'], + jump_table_proofs=member.get('jump_table_proofs'), + call_continuations=member.get('call_continuation_pcs')) + add_consensus_fallback(record,member) record['producer_name']=( f"{group['hed_path']}:entry_{member['id']:04X}" f"@logical_{member['file_offset']:08X}") @@ -1002,7 +1116,9 @@ def main(): f"logical+0x{member['file_offset']:X} @0x{base:08X}, " f"region@0x{page_base:08X}, " f"trusted-fit={member['score']}:{member['runner']}, " - f"direct seeds={len(member['direct_seeds'])}") + f"direct seeds={len(member['direct_seeds'])}, " + f"framed roots={len(member.get('return_framed_seeds',()))}, " + f"static dispatch={len(member.get('static_dispatch_pcs',()))}") # Runtime dirty-page capture coalesces directly adjacent producers into one # region keyed by the earlier page. Emit each provable two-file composition diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index 7631ca7e..26b0a0bf 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -414,6 +414,8 @@ def optional_enrichment_fallback_capture(cap: dict) -> dict | None: fallback['static_discovery_entry_pcs'] = encoded fallback['seeds'] = encoded fallback.pop('static_alias_ranges', None) + fallback.pop('static_jump_table_proofs', None) + fallback.pop('static_call_continuation_pcs', None) fallback.pop('_prior_aliases', None) fallback.pop(key, None) return fallback @@ -535,13 +537,217 @@ def _classify_cf(pc: int, word: int) -> tuple[str, int]: def _find_jump_table_targets(data: bytes, load_addr: int, size: int, entry: int, hard_cap: int, - jr_pc: int, jr_rs: int) -> set[int]: - """Fail closed until JR-table register/control dependence is proven. + jr_pc: int, jr_rs: int, + producer_range=None, proof_out=None) -> set[int]: + """Resolve one tightly bounded canonical ``sltiu``/``jr`` switch. - The former 40-word raw backscan could combine unrelated loads, constants, - and bounds checks across basic blocks into a synthetic table. + This is deliberately a structural reaching-definition proof, not the old + 40-word bag-of-opcodes backscan. The accepted suffix is:: + + sltiu guard,index,count; beq guard,zero,reject; nop + sll scaled,index,2; addu address,scaled,table_base + lw target,offset(address); [nop x 0..1]; jr target + + ``table_base`` must be a nearest-definition LUI, optionally followed by a + nearest-definition ADDIU. The lower instruction may rename the + register (``lui v0; addiu s0,v0,lo``), as used by Ape Escape. Any other + control flow between that constant and the suffix, an inbound edge that + skips the definition, a register mismatch/clobber, or one non-local table + entry rejects the entire candidate. """ - return set() + lo = load_addr + hi = load_addr + size + if not (entry <= jr_pc < hard_cap and 0 < jr_rs < 32): + return set() + jr_word = _word_at(data, load_addr, jr_pc) + if jr_word != ((jr_rs << 21) | 0x08): + return set() + + # Permit only the compiler's zero-or-one scheduling NOP between LW/JR. + lw_pc = jr_pc - 4 + for _ in range(1): + if _word_at(data, load_addr, lw_pc) != 0: + break + lw_pc -= 4 + lw_word = _word_at(data, load_addr, lw_pc) + if lw_word is None or ((lw_word >> 26) & 0x3F) != 0x23 \ + or ((lw_word >> 16) & 0x1F) != jr_rs: + return set() + address_reg = (lw_word >> 21) & 0x1F + if not address_reg: + return set() + lw_offset = lw_word & 0xFFFF + if lw_offset & 0x8000: + lw_offset -= 0x10000 + + addu_pc = lw_pc - 4 + addu_word = _word_at(data, load_addr, addu_pc) + if addu_word is None or ((addu_word >> 26) & 0x3F) != 0 \ + or (addu_word & 0x3F) != 0x21 \ + or ((addu_word >> 6) & 0x1F) != 0 \ + or ((addu_word >> 11) & 0x1F) != address_reg: + return set() + addu_inputs = ((addu_word >> 21) & 0x1F, + (addu_word >> 16) & 0x1F) + + sll_pc = addu_pc - 4 + sll_word = _word_at(data, load_addr, sll_pc) + if sll_word is None or ((sll_word >> 26) & 0x3F) != 0 \ + or (sll_word & 0x3F) != 0 \ + or ((sll_word >> 21) & 0x1F) != 0 \ + or ((sll_word >> 6) & 0x1F) != 2: + return set() + scaled_reg = (sll_word >> 11) & 0x1F + index_reg = (sll_word >> 16) & 0x1F + if not scaled_reg or not index_reg or scaled_reg not in addu_inputs: + return set() + table_reg = addu_inputs[1] if addu_inputs[0] == scaled_reg else addu_inputs[0] + if not table_reg or table_reg == scaled_reg: + return set() + + # The bound must guard this exact index on the sole fallthrough into SLL. + sltiu_pc = sll_pc - 12 + branch_pc = sll_pc - 8 + if _word_at(data, load_addr, sll_pc - 4) != 0: + return set() + sltiu_word = _word_at(data, load_addr, sltiu_pc) + branch_word = _word_at(data, load_addr, branch_pc) + if sltiu_word is None or ((sltiu_word >> 26) & 0x3F) != 0x0B \ + or ((sltiu_word >> 21) & 0x1F) != index_reg: + return set() + guard_reg = (sltiu_word >> 16) & 0x1F + table_count = sltiu_word & 0xFFFF + if not guard_reg or not (1 <= table_count < 512): + return set() + if branch_word is None or ((branch_word >> 26) & 0x3F) != 0x04: + return set() + branch_rs = (branch_word >> 21) & 0x1F + branch_rt = (branch_word >> 16) & 0x1F + if {branch_rs, branch_rt} != {0, guard_reg}: + return set() + reject_target = _branch_target(branch_pc, branch_word) + if (not (entry <= reject_target < hard_cap) or + sll_pc <= reject_target < jr_pc + 8): + return set() + + def nearest_writer(reg: int, before: int): + for pc in range(before - 4, max(entry, before - 0x80) - 1, -4): + word = _word_at(data, load_addr, pc) + if word is None: + return None + if _instruction_writes_gpr(word, reg): + return pc, word + return None + + table_writer = nearest_writer(table_reg, addu_pc) + if table_writer is None: + return set() + table_def_pc, table_def_word = table_writer + def_op = (table_def_word >> 26) & 0x3F + def_rs = (table_def_word >> 21) & 0x1F + def_rt = (table_def_word >> 16) & 0x1F + if def_op == 0x0F and def_rs == 0 and def_rt == table_reg: + constant_pc = table_def_pc + table_base = (table_def_word & 0xFFFF) << 16 + elif def_op == 0x09 and def_rt == table_reg and def_rs: + upper_writer = nearest_writer(def_rs, table_def_pc) + if upper_writer is None: + return set() + constant_pc, upper_word = upper_writer + if ((upper_word >> 26) & 0x3F) != 0x0F \ + or ((upper_word >> 21) & 0x1F) != 0 \ + or ((upper_word >> 16) & 0x1F) != def_rs: + return set() + low = table_def_word & 0xFFFF + upper = (upper_word & 0xFFFF) << 16 + if low & 0x8000: + low -= 0x10000 + table_base = (upper + low) & 0xFFFFFFFF + else: + return set() + + # Calls could clobber the proven constant. Other branches could enter the + # suffix through an unproved path. The one exact bounds branch is allowed. + for pc in range(constant_pc + 4, sll_pc, 4): + word = _word_at(data, load_addr, pc) + if word is None: + return set() + if _is_control_flow(word) and pc != branch_pc: + return set() + predecessor = _word_at(data, load_addr, constant_pc - 4) + if constant_pc >= entry + 4 and _is_control_flow(predecessor): + return set() + table_base = (table_base + lw_offset) & 0xFFFFFFFF + table_end = table_base + table_count * 4 + range_lo, range_hi = producer_range or (lo, hi) + if (table_base & 3) or not (range_lo <= table_base < table_end <= range_hi): + return set() + + targets = set() + for index in range(table_count): + target = _word_at(data, load_addr, table_base + index * 4) + if (target is None or (target & 3) or + not (entry <= target < hard_cap) or + not (range_lo <= target < range_hi) or + not _is_valid_mips_word(_word_at(data, load_addr, target))): + return set() + targets.add(target) + + # Establish which later sources are themselves reached only after the + # proven table dispatch. Such case blocks may safely loop to the suffix + # (Ape does this); an arbitrary earlier/later byte sequence may not jump + # into the protected definitions and manufacture domination. + case_reachable = set() + work = deque(targets) + while work and len(case_reachable) < 2048: + pc = work.popleft() + if (pc in case_reachable or not (entry <= pc < hard_cap) or + not (range_lo <= pc < range_hi)): + continue + word = _word_at(data, load_addr, pc) + if not _is_valid_mips_word(word): + continue + case_reachable.add(pc) + kind, target = _classify_cf(pc, word) + delay = pc + 4 + if kind == 'normal': + work.append(pc + 4) + elif kind == 'branch': + case_reachable.add(delay) + work.extend((pc + 8, target)) + elif kind == 'branch_never': + case_reachable.add(delay) + work.append(pc + 8) + elif kind == 'branch_never_likely': + work.append(pc + 8) + elif kind == 'j': + case_reachable.add(delay) + work.append(target) + elif kind in ('jal', 'jalr'): + case_reachable.add(delay) + work.append(pc + 8) + elif kind in ('jr', 'jr_ra'): + case_reachable.add(delay) + + for source in range(entry, hard_cap, 4): + source_word = _word_at(data, load_addr, source) + if source_word is None: + return set() + kind, target = _classify_cf(source, source_word) + if (kind in ('branch', 'j', 'jal') and + constant_pc < target <= jr_pc and + not (constant_pc <= source <= jr_pc) and + source not in case_reachable): + return set() + if proof_out is not None: + proof_out.append({ + 'host_entry': entry, + 'jr_pc': jr_pc, + 'table_base': table_base, + 'count': table_count, + 'targets': sorted(targets), + }) + return targets def _callable_legacy_seed(data: bytes, load_addr: int, addr: int) -> bool: @@ -748,8 +954,10 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, visited = set() direct_jals = set() static_indirect_targets = set() + call_continuations = set() branch_targets = set() jump_table_targets = set() + jump_table_proofs = [] forward_branch_targets = set() rejected_cross_producer_calls = set() accepted_cross_producer_calls = set() @@ -853,6 +1061,7 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, if in_function(pc + 8): work.append(pc + 8) branch_targets.add(pc + 8) + call_continuations.add(pc + 8) if (lo <= target < hi and (target & 3) == 0 and _is_valid_mips_word(_word_at(data, load_addr, target)) and callable_transfer_target(pc, target)): @@ -863,6 +1072,7 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, if in_function(pc + 8): work.append(pc + 8) branch_targets.add(pc + 8) + call_continuations.add(pc + 8) target_reg = (word >> 21) & 0x1F target = _resolve_constant_transfer(data, load_addr, entry, pc, target_reg) @@ -881,12 +1091,15 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, elif callable_transfer_target(pc, target): static_indirect_targets.add(target) else: + proofs = [] for jt in _find_jump_table_targets(data, load_addr, size, - entry, hard_cap, pc, jr_rs): + entry, hard_cap, pc, jr_rs, + entry_producer, proofs): jump_table_targets.add(jt) branch_targets.add(jt) if in_function(jt): work.append(jt) + jump_table_proofs.extend(proofs) elif kind == 'jr_ra': if in_function(delay): visited.add(delay) @@ -895,8 +1108,10 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, 'visited': visited, 'direct_jals': direct_jals, 'static_indirect_targets': static_indirect_targets, + 'call_continuations': call_continuations, 'branch_targets': branch_targets, 'jump_table_targets': jump_table_targets, + 'jump_table_proofs': jump_table_proofs, 'forward_branch_targets': forward_branch_targets, 'rejected_cross_producer_calls': rejected_cross_producer_calls, 'accepted_cross_producer_calls': accepted_cross_producer_calls, @@ -1347,6 +1562,9 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, 'static_alias_ranges': static_alias_ranges, 'rejected_cross_producer_calls': rejected_cross_producer_calls, 'accepted_cross_producer_calls': accepted_cross_producer_calls, + 'static_jump_table_proofs': cap.get('static_jump_table_proofs', []), + 'static_call_continuation_pcs': _parse_addr_list( + cap.get('static_call_continuation_pcs', [])), } # Interior entries carry the 'interior' marker so the recompiler emits # them as overlapping aliases, never as walk roots. Promoted kernel From 5a4b74f5d36da844f35b56b3609e0ff9d77ecd60 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sun, 19 Jul 2026 17:39:55 -0700 Subject: [PATCH 35/37] fix: report AOT recall with exact semantics --- docs/AOT_OVERLAY_PLAN.md | 82 +++++-- tools/aot_overlay_spike/README.md | 66 +++--- tools/aot_overlay_spike/coverage_report.py | 201 +++++++++++++----- .../aot_overlay_spike/test_coverage_report.py | 40 +++- 4 files changed, 281 insertions(+), 108 deletions(-) diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index 3544f3e8..9c4c8228 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -10,20 +10,46 @@ plus exact adjacent-producer composites (52 capture records total). The 29 base/ shards and the live A01 composite audit with zero unsupported instructions or bad targets under `cg5_d0a05489`. Direct/constant-register call discovery, raw/cross-file base recovery, and padded-return frameless-boundary retention raised full-playthrough -vault exact-entry recall from 1113/1856 (60.0%) to 1270/1856 (68.4%); entry+code_crc -is 1150/1856 (62.0%). Exact entry parity is intentionally no longer treated as the -uncovered-code count: the runtime vault contains fine-grained fragments, including -consecutive instruction entries, while static AOT emits broad functions. Compiled -code-range recall is 1811/1856 (97.6%), leaving exactly 45 true gaps, all in the -game-independent kernel/low-memory region. Runtime static-only validation loaded -the richer MAIN shard at 60 fps with `stale_blocked=0`, no range/manifest overflow, -and byte-matching candidates. The clean static-only attract soak then loaded the +vault exact-address recall from 1113/1856 (60.0%) to 1270/1856 (68.4%). Those +historical figures predate duplicate-VA CRC accounting; current reports separately +count unique entry addresses and exact `(entry, code_crc)` candidates, preserving +every observed code variant at a reused address. Exact-entry parity is intentionally +no longer treated as the uncovered-code count: the runtime vault contains +fine-grained fragments, including consecutive instruction entries, while static AOT +emits broad functions. Historical interval containment was 1811/1856 (97.6%), with +45 observed addresses outside every static interval, all in the game-independent +kernel/low-memory region. Interval containment is unverified potential, not proof +that the matching code variant can dispatch natively. Runtime static-only validation +loaded the richer MAIN shard at 60 fps with `stale_blocked=0`, no range/manifest +overflow, and byte-matching candidates. The clean static-only attract soak then loaded the 459-function A01 composite and ran the formerly ~14 fps lava demo at 65–67 fps at BelowNormal priority, with correct rendering and no range/manifest overflow. Misses continue to fail safely to the interpreter. --- +## 2026-07-19: strict indirect-dispatch enrichment and scoreboard contract + +- A framed-root candidate is accepted only at a return-adjacent boundary with a + bounded stack frame and an in-frame `ra` save before its first control transfer. +- Indirect switch recovery accepts only the canonical bounded form: `sltiu` guard + plus delay-slot NOP, `sll index,2`, producer-owned `lui`+`addiu` table address, + `addu`, `lw`, and the exact `jr`. A clobber, skipped definition, malformed or + foreign table, or invalid target rejects the whole table. Case targets and call + continuations remain dispatch-only interiors, never roots. Python discovery, + C++ analysis, and code generation use the same matcher and producer bounds. +- The emitted proof metadata is diagnostic. If generated-C audit or host compilation + rejects an enriched recipe, compilation strips framed roots, aliases, dispatch + PCs, and proof fields, then retries the exact producer bytes with direct-JAL roots + only. +- The scoreboard reports three distinct bounded metrics: exact entry address; exact + `(entry, code_crc)` candidate, retaining multiple variants per address; and + unverified interval containment potential. Vaults and capture histories are finite + observed needed sets, never exhaustive title inventories. A reported 100% means + only 100% of the named observed set under the named metric. + +--- + ## 2026-07-18: byte-proven archive producers - The generic extractor now recognizes strict 0x800-aligned `{id,size}` member @@ -37,24 +63,25 @@ continue to fail safely to the interpreter. generated-C audit or GCC rejection automatically retries the same byte-proven member with direct roots only. The final disposable MMX6 build compiled all 30 members plus the BIOS resident shard (`ok=31 failed=0 skipped=0`). Across 16 - legacy capture files it covers 636/677 entries by overlay code range (93.9%); - all 41 residual gaps are BIOS/kernel PCs, and the separately recompiled base - BIOS raises combined native code-range recall to 677/677 (100.0%). + legacy capture files it contains 636/677 observed entries in overlay intervals + (93.9%); all 41 observed addresses outside those intervals are BIOS/kernel PCs, + and the separately recompiled base BIOS raises combined interval containment to + 677/677 (100.0%) for that observed set. - Ape's two PS-X EXE mirrors are deduplicated in favor of the self-describing EXE producer. Its remaining 44 HED/BNS members plus two minigame EXEs and the BIOS recipe are also audit-clean in a disposable cache. - A fresh 47/47 clean Ape rebuild then completed a title-to-attract-to-title soak across four overlay generations. All 72 exercised game PCs lie in static compiled ranges; the other 35 PCs are covered by the separately generated - BIOS/kernel ranges, yielding 107/107 combined range coverage with zero true - gaps. Live-byte guards safely rejected 21 stale candidates, visuals remained - correct, and the dev history recorded 21 verified immutable snapshots with no - invalid or duplicate references. + BIOS/kernel ranges, yielding 107/107 combined interval containment for that + observed set. Live-byte guards safely rejected 21 stale candidates, visuals + remained correct, and the dev history recorded 21 verified immutable snapshots + with no invalid or duplicate references. - Normal PS-X discovery now exports its exact overlapping `F/R` alias recipes. Clean overlay builds therefore preserve decoder-proven indirect entries without promoting every entry to a hard root or borrowing a previous cache manifest. T2 compiled 53/53 disposable candidates with zero audit failures and restored - 100% combined BIOS code-range coverage for both vault and live history. + 100% combined BIOS interval containment for both observed sets. - Large archive reads are now linear rather than quadratic sector concatenation. These validations wrote only `%TEMP%` caches; title caches remain untouched. - DLL publication is now atomic: compilers target a unique sibling temporary, @@ -69,12 +96,12 @@ continue to fail safely to the interpreter. The clean 25-record corpus produces 17 unique shards with zero audit failures. Overlay ranges cover 820/823 historical-vault PCs and 410/450 live-history PCs; every residual is in the separately recompiled BIOS/kernel ranges, yielding - **100.0% combined code-range recall and zero true gaps** for both scoreboards. + **100.0% combined interval containment** for both observed scoreboards. - The resulting `cg5_060368b2` checkpoint was re-extracted from disc and tested against disposable caches on the other producers. Ape rebuilt 47/47 GCC shards - and retained 107/107 combined live-history range coverage. MMX6 rebuilt 31/31; + and retained 107/107 combined live-history interval containment. MMX6 rebuilt 31/31; two unsafe broad-discovery attempts failed their audit/compile gates and fell - back to conservative seeds, retaining 677/677 combined range coverage. Tomba 2 + back to conservative seeds, retaining 677/677 combined interval containment. Tomba 2 audited all 53 candidates with zero unsupported/bad targets, TCC-built the full 53/53 set, and GCC-built 11 representative shards including MAIN, large area variants, OPN/CRD, and BIOS. @@ -223,10 +250,17 @@ explicit assertion is required because an old gap manifest stores misses, not every previously covered entry; silently assuming monotonicity would make a regressed static set look healthier than it is. +All scoreboard inputs are finite observed needed sets. Reports distinguish exact +entry-address recall, exact `(entry, code_crc)` candidate recall (including multiple +CRC variants at one address), and interval containment potential. The last metric +does not prove that the owning variant or a valid compiled interior matches live +bytes; no vault result is evidence of an exhaustive title inventory. + The base BIOS recompiler is a separate static producer. When its generated dispatcher is available, pass `--bios-dispatch ` so the -scoreboard preserves overlay-cache-only recall and also reports combined native -coverage from the dispatch table plus guarded relocated-kernel body ranges. Do not +scoreboard preserves overlay-cache-only metrics and also reports combined exact +candidate and interval metrics from the dispatch table plus guarded relocated-kernel +body ranges. Do not manufacture overlay shards merely to duplicate those already-native BIOS bodies. Boot-installed RAM helpers that are not contiguous ROM relocations use a separate @@ -243,6 +277,9 @@ Tomba 2 kernel gaps without turning adjacent zeros/data into code. 1. Python does function-boundary discovery from the seeds (`classify_overlay_seeds`, `_walk_overlay_function`) with a strict callable-prologue gate; interior/ jump-table PCs become aliases, never walk roots. + Optional enrichment admits only return-adjacent framed roots and the canonical, + producer-bounded switch sequence described above. Its proof fields are persisted + for diagnosis, not treated as runtime trust. 2. Captured bytes wrapped in a **synthetic PS-X EXE** (`make_psxexe`, `:266-274`) and handed to the *same* recompiler binary the main EXE uses: `psxrecomp-game --seeds seeds.txt --overlay --ws-config game.toml`. @@ -259,6 +296,9 @@ Tomba 2 kernel gaps without turning adjacent zeros/data into code. A reachable direct branch that crosses a sibling-entry hard cap may add its target as a static root only when a bounded target-local CFG reaches a return without invalid words or sequential escape. + If audit or host compilation rejects enrichment, the tool retries the same bytes + after removing framed roots, aliases, dispatch PCs, and proof fields; only + direct-JAL roots remain. Exact-hash BIOS resident captures additionally receive a `.resident` sidecar; ordinary captures actively remove a stale sidecar for the same output stem. 5. `--static` mode instead namespaces the C and folds variants into diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index f1a2a7db..51fbbae6 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -10,6 +10,10 @@ for the full design, findings, and honest coverage numbers. pipeline. Disc-container discovery remains engine-specific; unsupported producers fail closed instead of being guessed as code. +Coverage reports compare against finite played/live observed sets, not exhaustive +title inventories. A reported 100% means only 100% of that named observed set under +the named metric. + ## What each does - `overlay_determinism.py` — corpus analysis across all games' captured overlays: @@ -195,19 +199,20 @@ that exact `00106000_0DC32D96` shard and ran the formerly ~14 fps lava-area attr demo at 65–67 fps while the runtime process remained BelowNormal. The scene rendered correctly, with no range-index or lazy-manifest overflow. -`coverage_report.py` reports both exact manifest-entry recall and compiled -code-range recall. The distinction matters: runtime/autocompile vaults may contain -one-entry fragments at consecutive instructions, while static AOT emits one broad -function with the same PCs in its guarded `R` ranges. Exact-entry parity is retained -for continuity; code-range recall is the useful uncovered-code roadmap. At the -current Tomba 2 checkpoint the play-free overlay cache plus the exact-hash -resident shard covers 97.6% of vault code ranges. Pass the generated base BIOS +`coverage_report.py` reports three separate bounded metrics: exact entry address; +exact `(entry, code_crc)` candidate, preserving every observed CRC variant at a +reused address; and unverified interval containment potential. The distinction +matters: runtime/autocompile vaults may contain one-entry fragments at consecutive +instructions, while static AOT emits one broad function with the same PCs in its +guarded `R` ranges. Interval containment does not prove that the owning variant or +a valid compiled interior matches live bytes. At the historical Tomba 2 checkpoint, +the play-free overlay cache plus exact-hash resident shard contained 97.6% of the +observed vault addresses in static intervals. Pass the generated base BIOS dispatcher with `--bios-dispatch generated/SCPH1001_dispatch.c` to report separate -combined metrics. The base recompiler covers relocated kernel bodies with a -live-byte guard; including it raises both the full-playthrough vault and the -verified append-only live-history needed set to **100% native code-range -coverage**. This avoids generating duplicate overlay shards for code that is -already native. +combined metrics. Including it reached 100% interval containment for the named +finite full-playthrough and verified live-history sets; that is not exhaustive +title coverage or proof of native dispatch. This avoids generating duplicate +overlay shards for code already provided by the BIOS producer. ## Indexed and companion archive recovery (2026-07-18) @@ -228,14 +233,25 @@ generated-C audit or host-compiler rejection retries the exact same bytes with direct-call roots only. A rejected recipe never contributes a DLL or a failure to the final cache. +The optional indirect-dispatch enrichment is deliberately narrow. A framed root +must be adjacent to a prior `jr ra`, establish a bounded stack frame, and save `ra` +in-frame before its first control transfer. A switch must match the canonical +producer-bounded `sltiu` guard + NOP, `sll`, `lui`+`addiu`, `addu`, `lw`, exact +`jr` sequence; clobbered or skipped definitions, malformed/foreign tables, and +invalid targets reject the whole table. Case targets and call continuations are +dispatch-only interiors, never roots. Python discovery, C++ analysis, and code +generation share those bounds. Proof metadata is diagnostic; an audit or compiler +failure strips framed roots, aliases, dispatch PCs, and proof fields before retrying +the exact bytes with direct-JAL roots only. + The validation compile produced `PSX_SHARD_RESULT ok=31 failed=0 skipped=0` (30 members + BIOS resident); unsafe enrichment fell back while clean members retained broad ranges. Against the union of 16 legacy MMX6 capture files, overlay -code-range recall rose from the direct-only **71.8%** baseline to **93.9%** +interval containment rose from the direct-only **71.8%** baseline to **93.9%** (636/677). Every one of the remaining 41 PCs is in the BIOS/kernel window; including the separately recompiled, live-byte-guarded base BIOS raises combined -native code-range recall to **100.0%** (677/677). Validation used only disposable -`%TEMP%` caches. +interval containment to **100.0%** (677/677) for that observed set. Validation +used only disposable `%TEMP%` caches. Tomba 1's header-table exports require a different conservative retry. Those authoritative addresses are commonly switch-case entries in the middle of a @@ -249,7 +265,7 @@ valid unreferenced frameless leaves. A clean 25-record corpus produces 17 unique shards with zero audit failures. Overlay ranges cover **820/823 (99.6%)** historical-vault PCs and **410/450 (91.1%)** live-history PCs; every residual is in the separately recompiled BIOS/kernel ranges, producing **100.0% combined -code-range recall and zero true gaps** on both scoreboards. The generated-C audit +interval containment** on both observed scoreboards. The generated-C audit remains the gate: rejected broad discovery is never installed. Ape Escape's `KKIIDDZZ.HED` encodes contiguous @@ -267,8 +283,8 @@ retaining its declared `0x80100C84` entry. All decoder-classified interior entries remain ordinary candidates, and the extractor serializes their `F/R` overlapping alias recipes instead of promoting them to hard call roots. A clean Tomba 2 disposable rebuild compiled 53/53 candidates with zero unsupported/bad -targets and, together with the base BIOS, reproduced **100% code-range coverage** -against both the full-playthrough vault and 765-entry append-only live history. +targets and, together with the base BIOS, reached **100% interval containment** +against the finite full-playthrough vault and 765-entry append-only live history. No prior MAIN manifest was present for the two regenerated shards. Position-fixed framed scans likewise recover Psy-Q's exact `lui R; load ...,off(R); addiu sp,sp,-N` entry after a previous return, replacing @@ -279,9 +295,9 @@ A fresh Ape validation rebuilt all **47/47** candidates with zero audit or host- compiler failures. A full title-to-attract-to-title cycle exercised four overlay generations and 107 dispatch PCs. The overlay cache covers all 72 exercised game PCs by compiled range; the remaining 35 are BIOS/kernel PCs, and the separately -generated base BIOS covers them all (**100.0% combined code-range recall, zero -true gaps**). Live-byte validation rejected 21 stale candidates without executing -them, while every loaded shard reported zero missing manifests and both loader +generated base BIOS covers them all (**100.0% combined interval containment** of +that observed set). Live-byte validation rejected 21 stale candidates without +executing them, while every loaded shard reported zero missing manifests and both loader indexes stayed within capacity. Screenshots verified the rainbow/falling attract scene and returned title screen rendered correctly. The dev persistence profile produced 21/21 signed immutable snapshots with zero invalid or duplicate records; @@ -296,9 +312,9 @@ final-named DLL that a later build mistakes for a valid shard, and a failed The `cg5_060368b2` discovery checkpoint was then regenerated from each disc and cross-title checked in disposable caches. Ape GCC-built **47/47** candidates and -retained **107/107 combined live-history code-range coverage**. MMX6 GCC-built -**31/31** candidates and retained **677/677 combined coverage**; broad normal-mode -attempts containing embedded data or an incomplete label graph were rejected and +retained **107/107 combined live-history interval containment**. MMX6 GCC-built +**31/31** candidates and retained **677/677 combined interval containment**; broad +normal-mode attempts containing embedded data or an incomplete label graph were rejected and the corresponding shards safely retried with direct seeds. Tomba 2's full set of **53/53** candidates passed the generated-C audit with zero unsupported/bad targets, and GCC built representative MAIN, large-area, OPN/CRD, and BIOS shards; @@ -330,7 +346,7 @@ so successful and interrupted compiles leave only final DLL/manifest pairs. final historical miss at `0x8010427C`. Stable-round exact discovery absorbs call-derived interior aliases into their existing host instead of using them as hard caps. Together these changes bring Tomba 1's combined historical and - live code-range scoreboards to **100.0% with zero true gaps**. + live observed-set scoreboards to **100.0% interval containment**. 2. ~~Header-table base recovery~~ — DONE (jal-fit, above). 3. ~~Recover weak-signal A09/A0J/GAME/OPN~~ — DONE. They are ordinary uncompressed MIPS, not archives. Counting only callable `jal` targets sharply resolves diff --git a/tools/aot_overlay_spike/coverage_report.py b/tools/aot_overlay_spike/coverage_report.py index 6755dc71..a1cb8f53 100644 --- a/tools/aot_overlay_spike/coverage_report.py +++ b/tools/aot_overlay_spike/coverage_report.py @@ -2,9 +2,10 @@ """AOT static-coverage recall report + gap manifest. Answers both "which played manifest entries did static extraction reproduce?" -and "which played entry PCs lie in compiled static code ranges?" The latter is -the effective code-coverage view when the played vault contains fine-grained -runtime fragments but static extraction emits broader functions. +and "which played entry PCs lie in any compiled static code interval?" The +latter is only an address-containment potential metric: it does not prove that +the containing manifest belongs to the live content variant, passes its whole- +function CRC guard, or can dispatch at that interior PC. Two ground-truth "needed" sources: * a played/coverage VAULT cache dir (.ranges from a full playthrough) — the most @@ -14,8 +15,8 @@ Two ground-truth "needed" sources: Recall is reported at three strengths: * entry-level : same function ENTRY address discovered (did we find it?) - * code-range : played entry PC lies in an R range compiled by static AOT - * entry+code_crc : same entry AND identical bytes (is our shard the real fn?) + * interval potential: played entry PC lies in an R range from some static AOT + * entry+code_crc : same entry AND same manifest code CRC candidate Usage: coverage_report.py --static --vault @@ -69,7 +70,11 @@ def parse_ranges_dir(d, with_audit=False): to quarantine them without needing the original capture bytes: F carries the range CRC and its following R carries the exact byte count. """ - ent = {} + # One virtual entry can legitimately have several content-keyed overlay + # variants. The runtime retains all of them and selects by the owning + # function's whole-range CRC, so collapsing this to entry -> one arbitrary + # CRC makes the scoreboard depend on glob traversal order. + ent = defaultdict(set) intervals = [] quarantined = set() for rf in glob.glob(os.path.join(d, '**', '*.ranges'), recursive=True): @@ -79,7 +84,7 @@ def parse_ranges_dir(d, with_audit=False): m = re.match(r'F\s+([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+)', line) if m: if pending is not None: - ent[pending[0]] = pending[1] + ent[pending[0]].add(pending[1]) pending = (norm(int(m.group(1), 16)), int(m.group(2), 16)) continue m = re.match(r'R\s+([0-9A-Fa-f]+)\s+([0-9A-Fa-f]+)', line) @@ -91,12 +96,12 @@ def parse_ranges_dir(d, with_audit=False): quarantined.add((pending[0], pending[1], size)) else: if pending is not None: - ent[pending[0]] = pending[1] + ent[pending[0]].add(pending[1]) intervals.append((lo, lo + size)) pending = None if pending is not None: - ent[pending[0]] = pending[1] - result = (ent, merge_intervals(intervals)) + ent[pending[0]].add(pending[1]) + result = (dict(ent), merge_intervals(intervals)) if not with_audit: return result audit = [ @@ -398,8 +403,23 @@ def parse_addendum_entries(path): audit['verified_snapshots'] += 1 return entries, audit +def _crc_set(value): + """Normalize legacy scalar CRC values and the current variant sets.""" + if value is None: + return set() + if isinstance(value, (set, frozenset, list, tuple)): + return {int(crc) for crc in value if crc is not None} + return {int(value)} + + def recall(needed_entries, static_ent, static_ranges, needed_crc=None): - """needed_entries: set; static_ent: {entry->crc}. Returns dict of metrics.""" + """Return exact-entry, exact-candidate, and interval-potential metrics. + + ``static_ent`` and ``needed_crc`` map an entry address to every known code + CRC variant at that VA. Scalar values remain accepted for older callers. + An R-interval hit is intentionally reported only as unverified potential: + ranges from different content variants are merged for this diagnostic. + """ covered = set(static_ent) hit = needed_entries & covered miss = needed_entries - covered @@ -412,14 +432,40 @@ def recall(needed_entries, static_ent, static_ranges, needed_crc=None): out = {'needed': len(needed_entries), 'covered_here': len(hit), 'missed': len(miss), 'recall_entry': (len(hit)/len(needed_entries) if needed_entries else 0.0), 'misses': sorted(miss), + 'covered_by_interval_potential': len(range_hit), + 'missed_interval_potential': len(range_miss), + 'recall_interval_potential': (len(range_hit)/len(needed_entries) if needed_entries else 0.0), + 'interval_potential_misses': sorted(range_miss), + 'interval_metric_semantics': 'unverified_address_containment_only', + # Backward-compatible JSON aliases. These do not imply a matching + # content variant or a dispatchable CPS continuation. 'covered_by_code_range': len(range_hit), 'missed_code_range': len(range_miss), 'recall_code_range': (len(range_hit)/len(needed_entries) if needed_entries else 0.0), 'range_misses': sorted(range_miss)} if needed_crc is not None: - crc_hit = sum(1 for e in hit if static_ent.get(e) == needed_crc.get(e)) - out['covered_entry_crc'] = crc_hit - out['recall_entry_crc'] = (crc_hit/len(needed_entries) if needed_entries else 0.0) + needed_candidates = { + (entry, crc) + for entry in needed_entries + for crc in _crc_set(needed_crc.get(entry)) + } + static_candidates = { + (entry, crc) + for entry, variants in static_ent.items() + for crc in _crc_set(variants) + } + crc_hits = needed_candidates & static_candidates + crc_misses = needed_candidates - static_candidates + crc_hit_addresses = {entry for entry, _ in crc_hits} + out['needed_entry_crc'] = len(needed_candidates) + out['covered_entry_crc'] = len(crc_hits) + out['missed_entry_crc'] = len(crc_misses) + out['recall_entry_crc'] = ( + len(crc_hits) / len(needed_candidates) if needed_candidates else 0.0) + out['covered_entry_crc_addresses'] = len(crc_hit_addresses) + out['recall_entry_crc_addresses'] = ( + len(crc_hit_addresses) / len(needed_entries) if needed_entries else 0.0) + out['entry_crc_misses'] = sorted(crc_misses) return out def group_by_region(addrs): @@ -451,15 +497,21 @@ def main(): static_ent, static_ranges, static_zero = parse_ranges_dir( a.static, with_audit=True) report = {'game': a.game, 'static_entries': len(static_ent), 'sources': {}} + report['static_entry_crc_candidates'] = sum( + len(_crc_set(variants)) for variants in static_ent.values()) report['static_quarantined_zero_ranges'] = static_zero - combined_ent = dict(static_ent) + combined_ent = { + entry: set(_crc_set(variants)) + for entry, variants in static_ent.items() + } combined_ranges = static_ranges if a.bios_dispatch: try: bios_ent, bios_ranges = parse_bios_dispatch(a.bios_dispatch) except (OSError, ValueError) as exc: ap.error(f'cannot parse --bios-dispatch: {exc}') - combined_ent.update({entry: None for entry in bios_ent}) + for entry in bios_ent: + combined_ent.setdefault(entry, set()).add(None) combined_ranges = merge_intervals(static_ranges + bios_ranges) report['bios_dispatch'] = a.bios_dispatch report['bios_static_entries'] = len(bios_ent) @@ -468,9 +520,12 @@ def main(): md = [] md.append(f'# AOT static-coverage recall — {a.game}\n') md.append('_How much of the played reference set did the play-free static ' - 'extractor reproduce, and how much lies in compiled static code?_\n') + 'extractor reproduce, and how much has unverified address overlap ' + 'with a compiled static interval?_\n') md.append(f'- Static shard cache: `{a.static}`') - md.append(f'- Static manifest entries: **{len(static_ent)}**\n') + md.append(f'- Static manifest entry addresses: **{len(static_ent)}**; ' + f'content-keyed entry+CRC candidates: ' + f'**{report["static_entry_crc_candidates"]}**\n') if static_zero: md.append(f'- Quarantined proven all-zero static ranges: ' f'**{len(static_zero)}** (excluded as data, not code)\n') @@ -485,10 +540,14 @@ def main(): report['vault_quarantined_zero_ranges'] = vault_zero r = recall(set(vault_ent), static_ent, static_ranges, vault_ent) report['sources']['vault'] = { - k: v for k, v in r.items() if k not in ('misses', 'range_misses')} + k: v for k, v in r.items() + if k not in ('misses', 'range_misses', + 'interval_potential_misses', 'entry_crc_misses')} report['vault_misses'] = [f'0x{x|0x80000000:08X}' for x in r['misses']] report['vault_range_misses'] = [ f'0x{x|0x80000000:08X}' for x in r['range_misses']] + report['vault_interval_potential_misses'] = list( + report['vault_range_misses']) combined = None if a.bios_dispatch: combined = recall(set(vault_ent), combined_ent, combined_ranges) @@ -496,6 +555,8 @@ def main(): k: v for k, v in combined.items() if k not in ('misses', 'range_misses')} report['vault_combined_range_misses'] = [ f'0x{x|0x80000000:08X}' for x in combined['range_misses']] + report['vault_combined_interval_potential_misses'] = list( + report['vault_combined_range_misses']) md.append('## vs played vault (most complete needed-set)\n') if vault_zero: md.append(f'- Quarantined proven all-zero played ranges: ' @@ -503,29 +564,32 @@ def main(): md.append(f'- Manifest entries in the full-playthrough vault: **{r["needed"]}**') md.append(f'- Discovered by static: **{r["covered_here"]}** ' f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') - md.append(f'- Covered by compiled static code ranges: ' - f'**{r["covered_by_code_range"]}** ' - f'(**{r["recall_code_range"]*100:.1f}%** code-range recall)') - md.append(' - Code-range recall answers whether the played entry PC is in ' - 'byte-guarded native code. Exact-entry recall is stricter manifest ' - 'granularity; runtime fragment caches can contain one entry per ' - 'instruction, so it substantially understates broad static shards.') - md.append(f'- Byte-identical (entry+code_crc): **{r.get("covered_entry_crc",0)}** ' + md.append(f'- Contained in some compiled static interval (unverified ' + f'potential): **{r["covered_by_interval_potential"]}** ' + f'(**{r["recall_interval_potential"]*100:.1f}%**)') + md.append(' - Interval containment alone does not prove native dispatch: ' + 'the owning content variant must pass its whole-function CRC guard, ' + 'and an interior PC must be a valid CPS continuation.') + md.append(f'- Exact manifest candidates (entry+code_crc): ' + f'**{r.get("covered_entry_crc",0)}/' + f'{r.get("needed_entry_crc",0)}** ' f'(**{r.get("recall_entry_crc",0)*100:.1f}%**) ' - f'_(cg-version differences lower this vs entry-level)_') + f'_(cg-version differences can lower this vs address-level recall)_') md.append(f'- **MISSED exact entries: {r["missed"]}**') - md.append(f'- **TRUE CODE-RANGE GAPS: {r["missed_code_range"]}** played ' - 'entry PCs outside all compiled static ranges\n') + md.append(f'- **OUTSIDE ALL STATIC INTERVALS: ' + f'{r["missed_interval_potential"]}** played entry PCs ' + '(potential gaps; interval membership is not dispatch proof)\n') if combined: md.append('### Combined with base recompiled BIOS\n') md.append(f'- Exact native dispatch entries: **{combined["covered_here"]}** ' f'(**{combined["recall_entry"]*100:.1f}%**)') - md.append(f'- Covered by native code ranges: ' - f'**{combined["covered_by_code_range"]}** ' - f'(**{combined["recall_code_range"]*100:.1f}%**)') - md.append(f'- **COMBINED CODE-RANGE GAPS: ' - f'{combined["missed_code_range"]}**\n') - md.append('#### Combined gaps grouped by region\n') + md.append(f'- Contained in an overlay/BIOS static interval ' + f'(unverified potential): ' + f'**{combined["covered_by_interval_potential"]}** ' + f'(**{combined["recall_interval_potential"]*100:.1f}%**)') + md.append(f'- **OUTSIDE ALL COMBINED STATIC INTERVALS: ' + f'{combined["missed_interval_potential"]}**\n') + md.append('#### Addresses outside all combined static intervals\n') grouped_combined = group_by_region(combined['range_misses']) if not grouped_combined: md.append('- None.') @@ -538,7 +602,7 @@ def main(): f'{x|0x80000000:08X}' for x in addrs[i:i+8])) md.append(' ```') md.append('') - md.append('### Code-range gaps grouped by overlay region\n') + md.append('### Addresses outside all static intervals, grouped by region\n') for reg, addrs in group_by_region(r['range_misses']).items(): md.append(f'- region `0x{reg|0x80000000:08X}`: {len(addrs)} misses') md.append(' ```') @@ -566,10 +630,13 @@ def main(): if live_sources: r = recall(live, static_ent, static_ranges) report['sources']['live_session'] = { - k: v for k, v in r.items() if k not in ('misses', 'range_misses')} + k: v for k, v in r.items() + if k not in ('misses', 'range_misses', 'interval_potential_misses')} report['live_misses'] = [f'0x{x|0x80000000:08X}' for x in r['misses']] report['live_range_misses'] = [ f'0x{x|0x80000000:08X}' for x in r['range_misses']] + report['live_interval_potential_misses'] = list( + report['live_range_misses']) combined = None if a.bios_dispatch: combined = recall(live, combined_ent, combined_ranges) @@ -577,6 +644,8 @@ def main(): k: v for k, v in combined.items() if k not in ('misses', 'range_misses')} report['live_combined_range_misses'] = [ f'0x{x|0x80000000:08X}' for x in combined['range_misses']] + report['live_combined_interval_potential_misses'] = list( + report['live_combined_range_misses']) md.append('## vs live capture history\n') md.append(f'- Sources: **{" + ".join(live_sources)}**') if a.addendum and 'live_addendum_audit' in report: @@ -594,17 +663,17 @@ def main(): md.append(f'- Dispatch entries exercised: **{r["needed"]}**') md.append(f'- Discovered by static: **{r["covered_here"]}** ' f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') - md.append(f'- Covered by compiled static code ranges: ' - f'**{r["covered_by_code_range"]}** ' - f'(**{r["recall_code_range"]*100:.1f}%** code-range recall)') - md.append(f'- Overlay-only true code-range gaps: ' - f'**{r["missed_code_range"]}**') + md.append(f'- Contained in some compiled static interval (unverified ' + f'potential): **{r["covered_by_interval_potential"]}** ' + f'(**{r["recall_interval_potential"]*100:.1f}%**)') + md.append(f'- Outside all overlay static intervals (potential gaps): ' + f'**{r["missed_interval_potential"]}**') if combined: - md.append(f'- Including base BIOS native code ranges: ' - f'**{combined["covered_by_code_range"]}** ' - f'(**{combined["recall_code_range"]*100:.1f}%**)') - md.append(f'- Combined true code-range gaps: ' - f'**{combined["missed_code_range"]}**') + md.append(f'- Including base BIOS static intervals (unverified ' + f'potential): **{combined["covered_by_interval_potential"]}** ' + f'(**{combined["recall_interval_potential"]*100:.1f}%**)') + md.append(f'- Outside all combined static intervals (potential gaps): ' + f'**{combined["missed_interval_potential"]}**') md.append(f'- Exact-entry misses (diagnostic; may be interior fragments): ' f'**{r["missed"]}**\n') if a.prior_report: @@ -636,6 +705,11 @@ def main(): covered = needed - len(miss) r = {'needed': needed, 'covered_here': covered, 'missed': len(miss), 'recall_entry': (covered/needed if needed else 0.0), + 'covered_by_interval_potential': needed - len(range_miss), + 'missed_interval_potential': len(range_miss), + 'recall_interval_potential': ( + (needed - len(range_miss))/needed if needed else 0.0), + 'interval_metric_semantics': 'unverified_address_containment_only', 'covered_by_code_range': needed - len(range_miss), 'missed_code_range': len(range_miss), 'recall_code_range': ((needed - len(range_miss))/needed if needed else 0.0)} @@ -648,6 +722,8 @@ def main(): f'0x{x|0x80000000:08X}' for x in sorted(miss)] report[f'{prior_prefix}_range_misses'] = [ f'0x{x|0x80000000:08X}' for x in sorted(range_miss)] + report[f'{prior_prefix}_interval_potential_misses'] = list( + report[f'{prior_prefix}_range_misses']) combined = None if a.bios_dispatch: combined_miss = old_miss - set(combined_ent) @@ -663,6 +739,11 @@ def main(): 'covered_here': combined_covered, 'missed': len(combined_miss), 'recall_entry': (combined_covered/needed if needed else 0.0), + 'covered_by_interval_potential': needed - len(combined_range_miss), + 'missed_interval_potential': len(combined_range_miss), + 'recall_interval_potential': ( + (needed - len(combined_range_miss))/needed if needed else 0.0), + 'interval_metric_semantics': 'unverified_address_containment_only', 'covered_by_code_range': needed - len(combined_range_miss), 'missed_code_range': len(combined_range_miss), 'recall_code_range': ( @@ -673,6 +754,8 @@ def main(): report['sources'][combined_key] = combined report[f'{prior_prefix}_combined_range_misses'] = [ f'0x{x|0x80000000:08X}' for x in sorted(combined_range_miss)] + report[f'{prior_prefix}_combined_interval_potential_misses'] = list( + report[f'{prior_prefix}_combined_range_misses']) heading = ('## vs pre-history persisted live-session gaps ' '(separate monotonic roll-forward)\n' if live_sources else '## vs persisted live-session gaps (monotonic roll-forward)\n') @@ -684,17 +767,17 @@ def main(): md.append(f'- Dispatch entries the persisted session exercised: **{needed}**') md.append(f'- Discovered by current static: **{covered}** ' f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') - md.append(f'- Covered by current static code ranges: ' - f'**{r["covered_by_code_range"]}** ' - f'(**{r["recall_code_range"]*100:.1f}%** code-range recall)') - md.append(f'- Overlay-only true code-range gaps: ' - f'**{r["missed_code_range"]}**') + md.append(f'- Contained in a current static interval (unverified ' + f'potential): **{r["covered_by_interval_potential"]}** ' + f'(**{r["recall_interval_potential"]*100:.1f}%**)') + md.append(f'- Outside all overlay static intervals (potential gaps): ' + f'**{r["missed_interval_potential"]}**') if combined: - md.append(f'- Including base BIOS native code ranges: ' - f'**{combined["covered_by_code_range"]}** ' - f'(**{combined["recall_code_range"]*100:.1f}%**)') - md.append(f'- Combined true code-range gaps: ' - f'**{combined["missed_code_range"]}**') + md.append(f'- Including base BIOS static intervals (unverified ' + f'potential): **{combined["covered_by_interval_potential"]}** ' + f'(**{combined["recall_interval_potential"]*100:.1f}%**)') + md.append(f'- Outside all combined static intervals (potential gaps): ' + f'**{combined["missed_interval_potential"]}**') md.append(f'- Exact-entry misses (diagnostic; may be interior fragments): ' f'**{len(miss)}**') md.append('- Provenance: caller explicitly asserted that the current static ' @@ -713,7 +796,7 @@ def main(): for src, m in report['sources'].items(): print(f' {src}: recall_entry={m.get("recall_entry",0)*100:.1f}% ' f'({m.get("covered_here")}/{m.get("needed")}), ' - f'recall_code_range={m.get("recall_code_range",0)*100:.1f}%, ' + f'interval_potential={m.get("recall_interval_potential",0)*100:.1f}%, ' f'missed={m.get("missed")}') if __name__ == '__main__': diff --git a/tools/aot_overlay_spike/test_coverage_report.py b/tools/aot_overlay_spike/test_coverage_report.py index 484d0b3b..2dd2f125 100644 --- a/tools/aot_overlay_spike/test_coverage_report.py +++ b/tools/aot_overlay_spike/test_coverage_report.py @@ -38,11 +38,45 @@ const PsxNativeStub psx_bios_native_stubs[1] = { self.assertEqual(entries, {0xA0, 0x500, 0x1FC00180}) self.assertEqual(ranges, [(0xA0, 0xB0), (0x500, 0x520)]) - def test_code_range_recall_is_distinct_from_exact_entry_recall(self): + def test_interval_potential_is_distinct_from_exact_entry_recall(self): result = coverage_report.recall( - {0x100, 0x104}, {0x100: 0x1234}, [(0x100, 0x108)]) + {0x100, 0x104}, {0x100: {0x1234}}, [(0x100, 0x108)]) self.assertEqual(result['covered_here'], 1) + self.assertEqual(result['covered_by_interval_potential'], 2) self.assertEqual(result['covered_by_code_range'], 2) + self.assertEqual( + result['interval_metric_semantics'], + 'unverified_address_containment_only') + + def test_exact_candidate_recall_preserves_duplicate_va_crc_variants(self): + result = coverage_report.recall( + {0x100}, {0x100: {0xBBBB, 0xCCCC}}, [(0x100, 0x108)], + {0x100: {0xAAAA, 0xBBBB}}) + self.assertEqual(result['covered_here'], 1) + self.assertEqual(result['needed_entry_crc'], 2) + self.assertEqual(result['covered_entry_crc'], 1) + self.assertEqual(result['missed_entry_crc'], 1) + self.assertEqual(result['recall_entry_crc'], 0.5) + self.assertEqual(result['entry_crc_misses'], [(0x100, 0xAAAA)]) + + def test_interval_containment_does_not_count_as_crc_candidate(self): + result = coverage_report.recall( + {0x100}, {0x100: {0xBBBB}}, [(0x100, 0x108)], + {0x100: {0xAAAA}}) + self.assertEqual(result['covered_here'], 1) + self.assertEqual(result['covered_by_interval_potential'], 1) + self.assertEqual(result['covered_entry_crc'], 0) + self.assertEqual(result['recall_entry_crc'], 0.0) + + def test_range_parser_retains_all_crc_variants_for_duplicate_va(self): + with tempfile.TemporaryDirectory() as tmp: + with open(os.path.join(tmp, 'a.ranges'), 'w', encoding='utf-8') as out: + out.write('F 80000100 AAAAAAAA\nR 80000100 8\n') + with open(os.path.join(tmp, 'b.ranges'), 'w', encoding='utf-8') as out: + out.write('F 80000100 BBBBBBBB\nR 80000100 C\n') + entries, ranges = coverage_report.parse_ranges_dir(tmp) + self.assertEqual(entries, {0x100: {0xAAAAAAAA, 0xBBBBBBBB}}) + self.assertEqual(ranges, [(0x100, 0x10C)]) def test_all_zero_compiled_range_is_quarantined(self): with tempfile.TemporaryDirectory() as tmp: @@ -56,7 +90,7 @@ const PsxNativeStub psx_bios_native_stubs[1] = { out.write('R 80000100 10\n') entries, ranges, audit = coverage_report.parse_ranges_dir( tmp, with_audit=True) - self.assertEqual(entries, {0x100: 0x12345678}) + self.assertEqual(entries, {0x100: {0x12345678}}) self.assertEqual(ranges, [(0x100, 0x110)]) self.assertEqual(audit, [{ 'entry': '0x8000EE7C', 'code_crc': '%08X' % zero_crc, From 156d9b9b3430e56f9f35daeb536cd1f85266e301 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Mon, 20 Jul 2026 00:00:22 -0700 Subject: [PATCH 36/37] Add self-contained media-to-source CLI release --- .github/workflows/cli-release.yml | 58 ++++ .gitignore | 1 + README.md | 70 +++++ recompiler/CMakeLists.txt | 62 ++++- recompiler/src/main_bios.cpp | 12 + recompiler/src/main_cli.cpp | 424 ++++++++++++++++++++++++++++++ recompiler/src/main_psx.cpp | 24 +- runtime/runtime.cmake | 11 +- tools/build_cli.py | 133 ++++++++++ 9 files changed, 773 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/cli-release.yml create mode 100644 recompiler/src/main_cli.cpp create mode 100644 tools/build_cli.py diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml new file mode 100644 index 00000000..0b8b95e8 --- /dev/null +++ b/.github/workflows/cli-release.yml @@ -0,0 +1,58 @@ +name: CLI release + +on: + pull_request: + push: + branches: [master] + workflow_dispatch: + release: + types: [published] + +permissions: + contents: read + +jobs: + build: + runs-on: windows-latest + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive + - name: Install MinGW toolchain (Windows) + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: true + install: >- + mingw-w64-x86_64-gcc + mingw-w64-x86_64-cmake + mingw-w64-x86_64-ninja + mingw-w64-x86_64-python + - name: Build CLI package + shell: msys2 {0} + run: python tools/build_cli.py release + - name: Smoke test packaged CLI + shell: msys2 {0} + run: dist/psxrecomp-cli-windows-x86_64/psxrecomp.exe --help + - uses: actions/upload-artifact@v6 + with: + name: psxrecomp-cli-windows-x86_64 + path: dist/psxrecomp-cli-*.zip + if-no-files-found: error + + publish-release-assets: + if: github.event_name == 'release' + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/download-artifact@v6 + with: + pattern: psxrecomp-cli-* + merge-multiple: true + - name: Attach CLI packages to release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + run: gh release upload "$RELEASE_TAG" psxrecomp-cli-*.zip --clobber diff --git a/.gitignore b/.gitignore index cdd34f56..31806705 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ recompiler/build/ recompiler/build-asan/ recompiler/build-asan2/ recompiler/build-checked/ +recompiler/build-cli/ recompiler/build-t2/ runtime/build/ runtime/build-msvc/ diff --git a/README.md b/README.md index 5e467dab..109ec742 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,76 @@ Background on the original prototype: [![PSXRecomp demo](https://img.youtube.com/vi/CID9oVhgCyY/maxresdefault.jpg)](https://www.youtube.com/watch?v=CID9oVhgCyY) +## How to use PSXRecomp + +PSXRecomp takes a PlayStation disc image and BIOS and creates a recompilation +project containing C source and build scripts. + +### Generate a project with the released CLI + +1. Download `psxrecomp-cli-windows-x86_64.zip` from + [Releases](https://github.com/mstan/psxrecomp/releases/latest). +2. Extract the whole zip to a folder. Keep its contents together. +3. Open PowerShell in that folder and run: + +```powershell +.\psxrecomp.exe build ` + --disc "C:\Games\My Game\game.cue" ` + --bios "C:\BIOS\SCPH1001.BIN" ` + --output "C:\Projects\MyGameRecomp" +``` + +Use the `.cue` file when a game has one, and keep its `.bin` track files beside +it. Single-file `.bin` and `.iso` images are also accepted. + +The output folder contains: + +- generated C source for the game and BIOS; +- `game.toml`, which you can edit for game-specific settings; +- `CMakeLists.txt` and build scripts; and +- a local copy of the PSXRecomp runtime source needed by the project. + +The downloaded CLI is self-contained. You do not need to install Python or +build this repository to generate a project. + +### Build the generated project + +Install CMake, Ninja, a C/C++ compiler, and SDL2 development files. Then run: + +```powershell +powershell -ExecutionPolicy Bypass -File "C:\Projects\MyGameRecomp\build.ps1" +``` + +The generated project also includes a shell build script for macOS and Linux: + +```sh +sh /path/to/MyGameRecomp/build.sh +``` + +The ready-made CLI release is currently for 64-bit Windows. You can build the +CLI from source on another operating system using the instructions below. + +The generated project is a practical starting point, not a promise that every +game works without game-specific fixes. PSX games can load extra code and use +hardware in ways that require additional configuration or development. + +Use only disc and BIOS files you obtained legally. PSXRecomp does not include +them. Generated game and BIOS source is derived from those files, so do not +redistribute it. + +### Build the CLI from source + +You need Git, Python 3, CMake, Ninja, and a C++20 compiler. + +```sh +git clone --recurse-submodules https://github.com/mstan/psxrecomp.git +cd psxrecomp +python tools/build_cli.py release +``` + +The ready-to-use CLI archive is written to `dist/`. To package debug binaries +instead, run `python tools/build_cli.py debug`. + > **Where the project is headed.** Development so far has been **breadth-first**: > stand up as many games as possible and get them into a playable alpha, proving > the framework generalizes. That phase has largely delivered — seven titles now diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index 6f5dca2e..59af8789 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -87,23 +87,43 @@ target_include_directories(psxrecomp-game PRIVATE lib/ELFIO ) target_link_libraries(psxrecomp-game PRIVATE fmt rabbitizer) -if(MINGW OR WIN32) - target_link_options(psxrecomp-game PRIVATE -Wl,--stack,16777216) -endif() +if(MINGW) + target_link_options(psxrecomp-game PRIVATE -Wl,--stack,16777216) +elseif(MSVC) + target_link_options(psxrecomp-game PRIVATE /STACK:16777216) +endif() # psxrecomp-toml — PS-X EXE → game.toml generator. # Standalone tool that analyses a PS-X EXE and generates a complete game.toml # config with auto-detected fields (load_address, entry_pc, text_size, # stack_base) and optional JAL-target seed generation. -add_executable(psxrecomp-toml - src/ps1_exe_parser.cpp - src/main_toml.cpp -) +add_executable(psxrecomp-toml + src/ps1_exe_parser.cpp + src/main_toml.cpp +) target_include_directories(psxrecomp-toml PRIVATE include lib/fmt/include ) -target_link_libraries(psxrecomp-toml PRIVATE fmt) +target_link_libraries(psxrecomp-toml PRIVATE fmt) + +# psxrecomp — user-facing media-to-source project generator. +add_executable(psxrecomp + src/main_cli.cpp + src/ps1_exe_parser.cpp + ../runtime/src/iso_reader.cpp +) +target_include_directories(psxrecomp PRIVATE + include + ../runtime/include + lib/fmt/include +) +target_link_libraries(psxrecomp PRIVATE fmt) +if(MINGW) + target_link_options(psxrecomp PRIVATE -Wl,--stack,16777216) +elseif(MSVC) + target_link_options(psxrecomp PRIVATE /STACK:16777216) +endif() # ---- baked codegen hash (stale-binary guard) -------------------------------- # Bake the SAME emitter-source hash the runtime folds into its overlay cache @@ -145,9 +165,11 @@ target_include_directories(PSXRecomp PRIVATE target_link_libraries(PSXRecomp PRIVATE fmt rabbitizer) # Increase stack size for large binaries (1447+ functions) -if(MINGW OR WIN32) - target_link_options(PSXRecomp PRIVATE -Wl,--stack,16777216) -endif() +if(MINGW) + target_link_options(PSXRecomp PRIVATE -Wl,--stack,16777216) +elseif(MSVC) + target_link_options(PSXRecomp PRIVATE /STACK:16777216) +endif() # psxrecomp-bios — Phase 1a BIOS boot slice recompiler. # Strict, fail-loud, ingests bios/SCPH1001.BIN as a flat ROM at 0xBFC00000. @@ -174,9 +196,21 @@ target_include_directories(psxrecomp-bios PRIVATE lib/toml11 ) target_link_libraries(psxrecomp-bios PRIVATE fmt rabbitizer) -if(MINGW OR WIN32) - target_link_options(psxrecomp-bios PRIVATE -Wl,--stack,16777216) -endif() +if(MINGW) + target_link_options(psxrecomp-bios PRIVATE -Wl,--stack,16777216) +elseif(MSVC) + target_link_options(psxrecomp-bios PRIVATE /STACK:16777216) +endif() + +# Released Windows CLI archives must run without a separate MinGW install. +# Keep this opt-in so ordinary developer/debug builds retain their normal +# linker behavior on every platform. +option(PSXRECOMP_STATIC_CLI "Statically link MinGW runtime libraries into CLI tools" OFF) +if(PSXRECOMP_STATIC_CLI AND MINGW) + foreach(_cli_target psxrecomp psxrecomp-game psxrecomp-toml psxrecomp-bios) + target_link_options(${_cli_target} PRIVATE -static -static-libgcc -static-libstdc++) + endforeach() +endif() # L1 decoder conformance test — compares MipsDecoder against Rabbitizer # oracle for every instruction in SCPH1001.BIN code regions. diff --git a/recompiler/src/main_bios.cpp b/recompiler/src/main_bios.cpp index 9f410a9f..e94d3f52 100644 --- a/recompiler/src/main_bios.cpp +++ b/recompiler/src/main_bios.cpp @@ -912,6 +912,18 @@ int run_discover(const fs::path& bios_path, const fs::path& out_dir, int main(int argc, char** argv) { try { + for (int i = 1; i < argc; ++i) { + const std::string a = argv[i]; + if (a == "-h" || a == "--help") { + std::fprintf(stdout, + "usage: psxrecomp-bios --config \n" + " psxrecomp-bios [--cc ]\n" + " psxrecomp-bios --discover \n" + " psxrecomp-bios --emit-full \n"); + return 0; + } + } + // ── --config short-form ──────────────────────────── // If --config is the first (or only) flag, all paths come from the // TOML. This is the going-forward invocation; the positional form diff --git a/recompiler/src/main_cli.cpp b/recompiler/src/main_cli.cpp new file mode 100644 index 00000000..bb1fe472 --- /dev/null +++ b/recompiler/src/main_cli.cpp @@ -0,0 +1,424 @@ +// psxrecomp — self-contained developer front-end. +// +// Disc image + BIOS in; generated C project + build scripts out. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#include +#endif + +#include "fmt/format.h" +#include "iso_reader.h" +#include "ps1_exe_parser.h" + +namespace fs = std::filesystem; + +namespace { + +struct Options { + fs::path disc; + fs::path bios; + fs::path output; + std::string name; +}; + +void usage(const char* program) { + fmt::print( + "Usage:\n" + " {} build --disc --bios " + "--output [--name ]\n\n" + "The output contains generated game/BIOS C, game.toml, CMakeLists.txt,\n" + "and build scripts. No compiler toolchain is bundled.\n", + program); +} + +std::string toml_string(std::string value) { + std::string out; + out.reserve(value.size()); + for (char c : value) { + if (c == '\\') out += '/'; + else if (c == '"') out += "\\\""; + else out += c; + } + return out; +} + +std::string trim(std::string value) { + while (!value.empty() && std::isspace((unsigned char)value.back())) value.pop_back(); + size_t first = 0; + while (first < value.size() && std::isspace((unsigned char)value[first])) first++; + return value.substr(first); +} + +std::string parse_boot_path(const std::string& cnf) { + std::string lower = cnf; + std::transform(lower.begin(), lower.end(), lower.begin(), + [](unsigned char c) { return (char)std::tolower(c); }); + size_t pos = lower.find("cdrom:"); + if (pos == std::string::npos) pos = lower.find("cdrom0:"); + if (pos == std::string::npos) return {}; + pos = lower.find(':', pos) + 1; + while (pos < cnf.size() && (cnf[pos] == '\\' || cnf[pos] == '/')) pos++; + std::string path; + while (pos < cnf.size()) { + char c = cnf[pos++]; + if (c == ';' || c == '\r' || c == '\n' || c == '\0' || + std::isspace((unsigned char)c)) break; + path += c; + } + return path; +} + +std::string serial_from_boot(const std::string& boot) { + std::string base = fs::path(boot).filename().string(); + std::string clean; + for (char c : base) { + if (std::isalnum((unsigned char)c)) clean += (char)std::toupper((unsigned char)c); + } + size_t split = 0; + while (split < clean.size() && std::isalpha((unsigned char)clean[split])) split++; + if (split && split < clean.size()) clean.insert(split, "-"); + return clean.empty() ? "PSX-GAME" : clean; +} + +std::string safe_stem(std::string value) { + std::string out; + for (char c : value) { + if (std::isalnum((unsigned char)c)) out += c; + else if (!out.empty() && out.back() != '_') out += '_'; + } + while (!out.empty() && out.back() == '_') out.pop_back(); + return out.empty() ? "PSXGameRecomp" : out; +} + +void write_file(const fs::path& path, const std::string& contents) { + fs::create_directories(path.parent_path()); + std::ofstream out(path, std::ios::binary | std::ios::trunc); + if (!out) throw std::runtime_error("cannot write " + path.string()); + out << contents; + if (!out) throw std::runtime_error("write failed for " + path.string()); +} + +void write_bytes(const fs::path& path, const std::vector<uint8_t>& contents) { + fs::create_directories(path.parent_path()); + std::ofstream out(path, std::ios::binary | std::ios::trunc); + if (!out) throw std::runtime_error("cannot write " + path.string()); + out.write((const char*)contents.data(), (std::streamsize)contents.size()); + if (!out) throw std::runtime_error("write failed for " + path.string()); +} + +fs::path executable_dir(const char* argv0) { + std::error_code ec; + fs::path path = fs::absolute(argv0, ec); + if (ec) path = argv0; + path = fs::weakly_canonical(path, ec); + return (ec ? fs::path(argv0) : path).parent_path(); +} + +fs::path find_helper(const fs::path& exe_dir, const std::string& name) { +#ifdef _WIN32 + const std::string filename = name + ".exe"; +#else + const std::string filename = name; +#endif + for (const fs::path& candidate : { + exe_dir / "libexec" / filename, + exe_dir / filename, + }) { + if (fs::is_regular_file(candidate)) return fs::absolute(candidate); + } + throw std::runtime_error("missing packaged helper: " + filename); +} + +fs::path find_framework(const fs::path& exe_dir) { + for (const fs::path& candidate : { + exe_dir / "framework", + exe_dir.parent_path().parent_path(), + }) { + if (fs::is_regular_file(candidate / "runtime" / "runtime.cmake")) + return fs::absolute(candidate); + } + throw std::runtime_error("packaged framework sources are missing"); +} + +fs::path find_bios_seeds(const fs::path& exe_dir) { + for (const fs::path& candidate : { + exe_dir / "share" / "phase2_ghidra_seeds.json", + exe_dir.parent_path() / "seeds" / "phase2_ghidra_seeds.json", + }) { + if (fs::is_regular_file(candidate)) return fs::absolute(candidate); + } + throw std::runtime_error("packaged BIOS seed data is missing"); +} + +int run_process(const std::vector<std::string>& args) { + std::vector<char*> raw; + raw.reserve(args.size() + 1); + for (const std::string& arg : args) raw.push_back(const_cast<char*>(arg.c_str())); + raw.push_back(nullptr); +#ifdef _WIN32 + intptr_t result = _spawnv(_P_WAIT, args[0].c_str(), raw.data()); + return result < 0 ? -1 : (int)result; +#else + pid_t pid = fork(); + if (pid < 0) return -1; + if (pid == 0) { + execv(args[0].c_str(), raw.data()); + _exit(127); + } + int status = 0; + if (waitpid(pid, &status, 0) < 0) return -1; + return WIFEXITED(status) ? WEXITSTATUS(status) : 128; +#endif +} + +class CurrentPathGuard { +public: + explicit CurrentPathGuard(const fs::path& path) : original_(fs::current_path()) { + fs::current_path(path); + } + ~CurrentPathGuard() { + std::error_code ignored; + fs::current_path(original_, ignored); + } + + CurrentPathGuard(const CurrentPathGuard&) = delete; + CurrentPathGuard& operator=(const CurrentPathGuard&) = delete; + +private: + fs::path original_; +}; + +void copy_framework(const fs::path& source, const fs::path& destination) { + fs::create_directories(destination); + for (const auto& entry : fs::directory_iterator(source)) { + fs::copy(entry.path(), destination / entry.path().filename(), + fs::copy_options::recursive | fs::copy_options::overwrite_existing); + } +} + +Options parse_options(int argc, char** argv) { + Options options; + for (int i = 2; i < argc; ++i) { + std::string arg = argv[i]; + auto value = [&](const char* flag) -> std::string { + if (i + 1 >= argc) throw std::runtime_error(std::string(flag) + " needs a value"); + return argv[++i]; + }; + if (arg == "--disc") options.disc = value("--disc"); + else if (arg == "--bios") options.bios = value("--bios"); + else if (arg == "--output" || arg == "-o") options.output = value("--output"); + else if (arg == "--name") options.name = value("--name"); + else if (arg == "-h" || arg == "--help") { + usage(argv[0]); + std::exit(0); + } else { + throw std::runtime_error("unknown argument: " + arg); + } + } + if (options.disc.empty() || options.bios.empty() || options.output.empty()) + throw std::runtime_error("--disc, --bios, and --output are required"); + options.disc = fs::absolute(options.disc); + options.bios = fs::absolute(options.bios); + options.output = fs::absolute(options.output); + return options; +} + +int build_project(const Options& options, const fs::path& exe_dir) { + if (!fs::is_regular_file(options.disc)) + throw std::runtime_error("disc image not found: " + options.disc.string()); + if (!fs::is_regular_file(options.bios)) + throw std::runtime_error("BIOS not found: " + options.bios.string()); + if (fs::file_size(options.bios) != 512 * 1024) + throw std::runtime_error("BIOS must be an uncompressed 512 KiB PlayStation BIOS dump"); + if (fs::exists(options.output)) { + if (!fs::is_directory(options.output)) + throw std::runtime_error("output path is not a directory: " + options.output.string()); + if (!fs::is_empty(options.output)) + throw std::runtime_error("output directory is not empty: " + options.output.string()); + } + + PS1::ISOReader disc; + if (!disc.Open(options.disc.string())) + throw std::runtime_error("could not open the disc image or its CUE tracks"); + size_t cnf_size = disc.GetFileSize("SYSTEM.CNF"); + if (!cnf_size) throw std::runtime_error("SYSTEM.CNF was not found on the disc"); + std::vector<uint8_t> cnf_bytes(cnf_size + 1, 0); + size_t cnf_read = disc.ReadFile("SYSTEM.CNF", cnf_bytes.data(), cnf_size); + std::string boot = parse_boot_path(std::string((char*)cnf_bytes.data(), cnf_read)); + if (boot.empty()) throw std::runtime_error("SYSTEM.CNF does not contain a BOOT executable"); + size_t exe_size = disc.GetFileSize(boot); + if (!exe_size) throw std::runtime_error("boot executable was not found on disc: " + boot); + std::vector<uint8_t> exe_bytes(exe_size); + if (disc.ReadFile(boot, exe_bytes.data(), exe_bytes.size()) != exe_bytes.size()) + throw std::runtime_error("failed to read the complete boot executable"); + if (exe_bytes.size() < 2048 || std::string((char*)exe_bytes.data(), 8) != "PS-X EXE") + throw std::runtime_error("the disc BOOT file is not a PS-X EXE"); + + fs::create_directories(options.output); + // Mark the generated folder as its own project before config_loader walks + // upward looking for a project root. This matters when the output folder + // happens to live inside another checkout. + write_file(options.output / ".gitignore", + "build/\ninput/\ngenerated/\nbios-generated/\n*.mcr\nsaves/\n"); + const std::string boot_file = fs::path(boot).filename().string(); + const fs::path local_exe = options.output / "input" / boot_file; + write_bytes(local_exe, exe_bytes); + + std::string parse_error; + auto parsed = PSXRecomp::PS1ExeParser::parse_file(local_exe, parse_error); + if (!parsed) throw std::runtime_error("failed to parse boot executable: " + parse_error); + const auto& image = *parsed; + const std::string serial = serial_from_boot(boot); + const std::string title = options.name.empty() + ? trim(disc.GetVolumeID()) + : options.name; + const std::string game_name = title.empty() ? serial : title; + const std::string project_name = safe_stem(game_name) + "Recomp"; + + std::set<uint32_t> seeds = {image.entry_point()}; + for (uint32_t address = image.load_address(); address + 4 <= image.end_address(); address += 4) { + auto word = image.read_word(address); + if (!word || ((*word >> 26) & 0x3F) != 0x03) continue; + uint32_t target = (address & 0xF0000000u) | ((*word & 0x03FFFFFFu) << 2); + if (target >= image.load_address() && target < image.end_address()) seeds.insert(target); + } + std::string seed_text = fmt::format("# Auto-generated JAL targets for {}\n", serial); + for (uint32_t seed : seeds) seed_text += fmt::format("0x{:08X}\n", seed); + write_file(options.output / "seeds" / "functions.txt", seed_text); + + uint32_t text_size = image.code_size(); + if (text_size & 0xFFF) text_size = (text_size + 0xFFF) & ~0xFFFu; + const uint32_t stack = image.header.stack_base ? image.header.stack_base : 0x801FFFF0u; + const std::string config = fmt::format( + "[game]\n" + "name = \"{}\"\n" + "id = \"{}\"\n" + "exe = \"input/{}\"\n" + "disc = \"{}\"\n" + "load_address = \"0x{:08X}\"\n" + "entry_pc = \"0x{:08X}\"\n" + "text_size = \"0x{:X}\"\n" + "stack_base = \"0x{:08X}\"\n\n" + "[recompiler]\n" + "seeds = \"seeds/functions.txt\"\n" + "strict = true\n" + "out_dir = \"generated\"\n\n" + "[runtime]\n" + "window_title = \"{} Recompiled\"\n" + "bios_hle = false\n" + "overlay_cache = true\n", + toml_string(game_name), serial, toml_string(boot_file), + toml_string(options.disc.generic_string()), image.load_address(), + image.entry_point(), text_size, stack, toml_string(game_name)); + write_file(options.output / "game.toml", config); + + const fs::path game_tool = find_helper(exe_dir, "psxrecomp-game"); + const fs::path bios_tool = find_helper(exe_dir, "psxrecomp-bios"); + fmt::print("[1/4] Recompiling game executable...\n"); + { + CurrentPathGuard project_directory(options.output); + if (run_process({game_tool.string(), "--config", "game.toml"})) + throw std::runtime_error("game recompilation failed"); + } + fmt::print("[2/4] Recompiling BIOS...\n"); + fs::create_directories(options.output / "bios-generated"); + if (run_process({bios_tool.string(), options.bios.string(), + (options.output / "bios-generated").string(), "--emit-full", + find_bios_seeds(exe_dir).string()})) + throw std::runtime_error("BIOS recompilation failed"); + + fmt::print("[3/4] Copying build framework...\n"); + copy_framework(find_framework(exe_dir), options.output / "psxrecomp"); + + const std::string cmake = fmt::format( + "cmake_minimum_required(VERSION 3.20)\n" + "project({} C CXX)\n" + "set(CMAKE_C_STANDARD 99)\n" + "set(CMAKE_CXX_STANDARD 17)\n" + "set(PSXRECOMP_ROOT \"${{CMAKE_CURRENT_SOURCE_DIR}}/psxrecomp\")\n" + "set(PSX_LAUNCHER OFF CACHE BOOL \"\" FORCE)\n" + "include(\"${{PSXRECOMP_ROOT}}/runtime/runtime.cmake\")\n" + "file(GLOB GAME_FULL CONFIGURE_DEPENDS\n" + " \"${{CMAKE_CURRENT_SOURCE_DIR}}/generated/*_full.c\"\n" + " \"${{CMAKE_CURRENT_SOURCE_DIR}}/generated/*_full_*.c\")\n" + "file(GLOB GAME_DISPATCH \"${{CMAKE_CURRENT_SOURCE_DIR}}/generated/*_dispatch.c\")\n" + "psxrecomp_add_runtime_target(psx-runtime\n" + " GAME_GENERATED_FULL_C \"${{GAME_FULL}}\"\n" + " GAME_GENERATED_DISPATCH_C \"${{GAME_DISPATCH}}\"\n" + " BIOS_GENERATED_FULL_C \"${{CMAKE_CURRENT_SOURCE_DIR}}/bios-generated/SCPH1001_full.c\"\n" + " BIOS_GENERATED_DISPATCH_C \"${{CMAKE_CURRENT_SOURCE_DIR}}/bios-generated/SCPH1001_dispatch.c\"\n" + " WINDOW_TITLE \"{} Recompiled\"\n" + " DEFAULT_GAME_CONFIG_PATH \"game.toml\"\n" + ")\n", + project_name, game_name); + write_file(options.output / "CMakeLists.txt", cmake); + + write_file(options.output / "build.ps1", + "$ErrorActionPreference = 'Stop'\n" + "$Root = Split-Path -Parent $MyInvocation.MyCommand.Path\n" + "cmake -S $Root -B (Join-Path $Root 'build') -G Ninja -DCMAKE_BUILD_TYPE=Release -DPSX_LAUNCHER=OFF\n" + "if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }\n" + "cmake --build (Join-Path $Root 'build') --config Release --parallel\n" + "exit $LASTEXITCODE\n"); + write_file(options.output / "build.sh", + "#!/usr/bin/env sh\n" + "set -eu\n" + "ROOT=$(CDPATH= cd -- \"$(dirname -- \"$0\")\" && pwd)\n" + "cmake -S \"$ROOT\" -B \"$ROOT/build\" -G Ninja -DCMAKE_BUILD_TYPE=Release -DPSX_LAUNCHER=OFF\n" + "cmake --build \"$ROOT/build\" --config Release --parallel\n"); + write_file(options.output / ".gitignore", + "build/\ninput/\ngenerated/\nbios-generated/\n*.mcr\nsaves/\n"); + write_file(options.output / "README.md", fmt::format( + "# {}\n\n" + "Generated locally by PSXRecomp from your own disc and BIOS.\n\n" + "## Build\n\n" + "Install CMake, Ninja, a C/C++ compiler, and SDL2 development files.\n" + "Then run `sh build.sh` on macOS/Linux or `.\\build.ps1` in PowerShell.\n\n" + "The executable is written under `build/`. Keep your original disc image\n" + "at the path stored in `game.toml`, or update that path before running.\n\n" + "The `input/`, `generated/`, and `bios-generated/` folders contain data\n" + "derived from copyrighted files you supplied. Do not redistribute them.\n", + game_name)); + fmt::print("[4/4] Wrote project files.\n\nReady: {}\nBuild with: {}\n", + options.output.string(), +#ifdef _WIN32 + (options.output / "build.ps1").string() +#else + (options.output / "build.sh").string() +#endif + ); + return 0; +} + +} // namespace + +int main(int argc, char** argv) { + try { + if (argc < 2 || std::string(argv[1]) == "-h" || std::string(argv[1]) == "--help") { + usage(argv[0]); + return argc < 2 ? 2 : 0; + } + if (std::string(argv[1]) != "build") { + usage(argv[0]); + throw std::runtime_error("unknown command: " + std::string(argv[1])); + } + return build_project(parse_options(argc, argv), executable_dir(argv[0])); + } catch (const std::exception& error) { + fmt::print(stderr, "psxrecomp: error: {}\n", error.what()); + return 1; + } +} diff --git a/recompiler/src/main_psx.cpp b/recompiler/src/main_psx.cpp index d97ab2d0..77a9168d 100644 --- a/recompiler/src/main_psx.cpp +++ b/recompiler/src/main_psx.cpp @@ -81,6 +81,23 @@ void materialize_alias_groups(PSXRecomp::FunctionAnalysisResult& result, } // namespace int main(int argc, char** argv) { + const auto print_usage = [&]() { + fmt::print("Usage: {} --config <game.toml>\n", argv[0]); + fmt::print(" {} <PS1-EXE file> [--seeds <file>] [--out-dir <dir>] [--strict] [--inspect]\n", argv[0]); + fmt::print("Example: {} SCUS_942.36 --seeds seeds/functions.txt --out-dir generated --strict\n", argv[0]); + }; + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "-h" || arg == "--help") { + print_usage(); + return 0; + } + } + if (argc < 2) { + print_usage(); + return 0; + } + /* --codegen-hash: print the baked emitter-source hash and exit. Handled * before the banner so the output is machine-parseable (one hex line). * compile_overlays.py compares this to the hash from --runtime-include and @@ -213,13 +230,6 @@ int main(int argc, char** argv) { fmt::print(" ws_tag_funcs= {}\n", ws_tag_funcs.size()); fmt::print("\n"); } else { - if (argc < 2) { - fmt::print("Usage: {} --config <game.toml> # going-forward\n", argv[0]); - fmt::print(" {} <PS1-EXE file> [--seeds <file>] [--out-dir <dir>] [--strict] [--inspect]\n", argv[0]); - fmt::print("Example: {} SCUS_942.36 --seeds seeds/ghidra_funcs.txt --out-dir generated --strict\n\n", argv[0]); - return 0; - } - exe_path = argv[1]; for (int i = 2; i < argc; i++) { std::string arg = argv[i]; diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index a4c2dce4..195fb55f 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -265,6 +265,8 @@ function(psxrecomp_add_runtime_target target) set(oneValueArgs GAME_GENERATED_DISPATCH_C GAME_OVERLAY_STATIC_C + BIOS_GENERATED_FULL_C + BIOS_GENERATED_DISPATCH_C DEBUG_PORT WINDOW_TITLE DEFAULT_BIOS_PATH @@ -309,7 +311,14 @@ function(psxrecomp_add_runtime_target target) set(PSXRT_DEFAULT_GAME_CONFIG_PATH "") endif() - set(generated_sources ${PSXRECOMP_BIOS_GENERATED}) + if(PSXRT_BIOS_GENERATED_FULL_C AND PSXRT_BIOS_GENERATED_DISPATCH_C) + set(generated_sources + "${PSXRT_BIOS_GENERATED_FULL_C}" + "${PSXRT_BIOS_GENERATED_DISPATCH_C}") + set_source_files_properties(${generated_sources} PROPERTIES GENERATED TRUE) + else() + set(generated_sources ${PSXRECOMP_BIOS_GENERATED}) + endif() if(PSXRT_GAME_GENERATED_FULL_C) foreach(_full_src IN LISTS PSXRT_GAME_GENERATED_FULL_C) set_source_files_properties("${_full_src}" PROPERTIES GENERATED TRUE) diff --git a/tools/build_cli.py b/tools/build_cli.py new file mode 100644 index 00000000..294d3f08 --- /dev/null +++ b/tools/build_cli.py @@ -0,0 +1,133 @@ +"""Build and package the PSXRecomp developer command-line tools.""" + +from __future__ import annotations + +import argparse +import os +import pathlib +import platform +import shutil +import subprocess +import sys + + +ROOT = pathlib.Path(__file__).resolve().parent.parent +TOOLS = ("psxrecomp", "psxrecomp-toml", "psxrecomp-game", "psxrecomp-bios") + + +def run(command: list[str]) -> None: + print("+", subprocess.list2cmdline(command), flush=True) + subprocess.run(command, cwd=ROOT, check=True) + + +def platform_name() -> str: + systems = {"Windows": "windows", "Linux": "linux", "Darwin": "macos"} + machines = { + "amd64": "x86_64", + "x86_64": "x86_64", + "arm64": "arm64", + "aarch64": "arm64", + } + raw_system = platform.system() + if raw_system.startswith(("MSYS", "MINGW", "CYGWIN")): + raw_system = "Windows" + system = systems.get(raw_system, raw_system.lower()) + machine = machines.get(platform.machine().lower(), platform.machine().lower()) + return f"{system}-{machine}" + + +def find_tool(build_dir: pathlib.Path, name: str, config: str) -> pathlib.Path: + suffix = ".exe" if os.name == "nt" else "" + filename = name + suffix + candidates = (build_dir / filename, build_dir / config / filename) + for candidate in candidates: + if candidate.is_file(): + return candidate + checked = ", ".join(str(path) for path in candidates) + raise FileNotFoundError(f"{filename} was not built; checked {checked}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="build a redistributable PSXRecomp CLI archive") + parser.add_argument( + "configuration", nargs="?", choices=("release", "debug"), + default="release", help="build configuration (default: release)") + parser.add_argument( + "--build-dir", default="recompiler/build-cli", + help="CMake build directory (default: recompiler/build-cli)") + parser.add_argument( + "--dist-dir", default="dist", + help="package output directory (default: dist)") + parser.add_argument( + "--skip-build", action="store_true", + help="package tools already present in --build-dir") + args = parser.parse_args() + + config = "Release" if args.configuration == "release" else "Debug" + build_dir = (ROOT / args.build_dir).resolve() + dist_dir = (ROOT / args.dist_dir).resolve() + + if not args.skip_build: + if shutil.which("cmake") is None: + parser.error("CMake is not installed or is not on PATH") + source_arg = os.path.relpath(ROOT / "recompiler", ROOT) + build_arg = os.path.relpath(build_dir, ROOT) + configure = [ + "cmake", "-S", source_arg, "-B", build_arg, + f"-DCMAKE_BUILD_TYPE={config}", + "-DPSXRECOMP_STATIC_CLI=ON", + ] + if not (build_dir / "CMakeCache.txt").exists() and shutil.which("ninja"): + configure += ["-G", "Ninja"] + run(configure) + jobs = str(os.cpu_count() or 2) + run([ + "cmake", "--build", build_arg, "--config", config, + "--target", *TOOLS, "--parallel", jobs, + ]) + + package = f"psxrecomp-cli-{platform_name()}" + stage = dist_dir / package + archive = dist_dir / f"{package}.zip" + dist_dir.mkdir(parents=True, exist_ok=True) + if stage.exists(): + shutil.rmtree(stage) + if archive.exists(): + archive.unlink() + stage.mkdir() + + shutil.copy2(find_tool(build_dir, "psxrecomp", config), stage) + libexec = stage / "libexec" + libexec.mkdir() + for name in TOOLS[1:]: + shutil.copy2(find_tool(build_dir, name, config), libexec) + share = stage / "share" + share.mkdir() + shutil.copy2( + ROOT / "recompiler" / "seeds" / "phase2_ghidra_seeds.json", share) + + framework = stage / "framework" + source_ignore = shutil.ignore_patterns( + "build", "build-*", "tests", "__pycache__", "*.pyc", "*.pyo", + "*.exe", "*.obj", "*.pdb", "*.lib") + shutil.copytree(ROOT / "runtime", framework / "runtime", ignore=source_ignore) + shutil.copytree(ROOT / "recompiler", framework / "recompiler", ignore=source_ignore) + shutil.copy2(ROOT / "LICENSE", framework) + shutil.copy2(ROOT / "THIRD_PARTY_ATTRIBUTION.md", framework) + shutil.copy2(ROOT / "README.md", stage) + shutil.copy2(ROOT / "LICENSE", stage) + shutil.copy2(ROOT / "THIRD_PARTY_ATTRIBUTION.md", stage) + + shutil.make_archive(str(archive.with_suffix("")), "zip", dist_dir, package) + print(f"PSXRecomp CLI package: {archive}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except subprocess.CalledProcessError as exc: + print(f"build_cli: command failed with exit code {exc.returncode}", + file=sys.stderr) + raise SystemExit(exc.returncode) From 82beda79eed3bb163cab474d6c87e20808a403eb Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Mon, 20 Jul 2026 00:07:20 -0700 Subject: [PATCH 37/37] Fix release asset upload repository --- .github/workflows/cli-release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml index 0b8b95e8..0da8c6ac 100644 --- a/.github/workflows/cli-release.yml +++ b/.github/workflows/cli-release.yml @@ -55,4 +55,6 @@ jobs: env: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ github.event.release.tag_name }} - run: gh release upload "$RELEASE_TAG" psxrecomp-cli-*.zip --clobber + run: >- + gh release upload "$RELEASE_TAG" psxrecomp-cli-*.zip + --repo "$GITHUB_REPOSITORY" --clobber