From 43cd37624abc2bd35fdcff73f8f9c2f788f804c6 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Fri, 17 Jul 2026 22:58:09 -0700 Subject: [PATCH] aot: discover frameless static call targets --- docs/AOT_OVERLAY_PLAN.md | 10 +- recompiler/src/function_analysis.cpp | 169 +++++++++++++++--- recompiler/src/main_psx.cpp | 17 +- recompiler/tests/reachable_discovery_test.cpp | 58 +++++- .../tests/test_aot_overlay_discovery.py | 70 ++++++++ tools/aot_overlay_spike/README.md | 18 +- tools/aot_overlay_spike/coverage_report.py | 11 +- tools/compile_overlays.py | 117 ++++++++++-- 8 files changed, 414 insertions(+), 56 deletions(-) create mode 100644 recompiler/tests/test_aot_overlay_discovery.py diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index 5caa4848..c71e6c10 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -1,10 +1,18 @@ # AOT Overlay Sharding — Spike Findings & Plan (Tomba-first) -Status: SPIKE / DESIGN (2026-07-17). Not yet executed. +Status: ENHANCEMENT SPIKE (2026-07-17). Foundation executed and live-proven. Author: investigation via 4 parallel agents + adversarial source verification. Scope: can we move overlay sharding from *runtime discovery* to *build-time (AOT)*, starting with Tomba (SCUS-94236)? +Current Tomba 2 checkpoint: generic play-free extraction emits 23 clean regions; +all 23 compile under `cg5_d0a05489`. Direct-call frameless-leaf discovery raised +full-playthrough-vault entry recall from 1113/1856 (60.0%) to 1187/1856 (64.0%), +and entry+code_crc recall from 1012 to 1084. All 74 newly covered vault entries are +in MAIN.EXE. The remaining manifest has 669 misses: 268 MAIN.EXE, 356 overlay-window, +and 45 kernel/low-memory entries. Runtime static-only smoke loaded the new tag with +`stale_blocked=0`; misses continue to fail safely to the interpreter via CRC guards. + --- ## ⭐ SESSION-2 BREAKTHROUGH (2026-07-17, VERIFIED FROM DISC + CORPUS) diff --git a/recompiler/src/function_analysis.cpp b/recompiler/src/function_analysis.cpp index 33d7584f..3ac817c0 100644 --- a/recompiler/src/function_analysis.cpp +++ b/recompiler/src/function_analysis.cpp @@ -392,6 +392,19 @@ uint32_t FunctionAnalyzer::find_function_start(uint32_t return_addr) { // from the previous function's trailing gap. return skip_leading_padding_nops(exe_, search_addr + 8, return_addr); } + + // Packed Psy-Q BIOS thunks tail-jump through A0/B0/C0 and therefore do + // not contain `jr $ra`. Treat their JR delay slot as a real boundary + // while scanning backward, otherwise the following frameless routine + // is swallowed into the whole preceding thunk run. + if (search_addr >= exe_.header.load_address + 4u) { + uint32_t thunk_jr = 0; + if (is_bios_dispatch_thunk(search_addr - 4u, thunk_jr) && + thunk_jr == search_addr) { + return skip_leading_padding_nops(exe_, search_addr + 8u, + return_addr); + } + } } // Couldn't find clear start, assume max search distance @@ -506,6 +519,96 @@ static uint32_t exact_jump_target(uint32_t pc, uint32_t instr) { return ((pc + 4u) & 0xF0000000u) | ((instr & 0x03FFFFFFu) << 2); } +static bool exact_instruction_writes_gpr(uint32_t instr, uint32_t reg) { + if (reg == 0u) return false; + + uint32_t opcode = (instr >> 26) & 0x3Fu; + uint32_t rs = (instr >> 21) & 0x1Fu; + uint32_t rt = (instr >> 16) & 0x1Fu; + uint32_t rd = (instr >> 11) & 0x1Fu; + uint32_t funct = instr & 0x3Fu; + + if (opcode == 0x00u) { + // SPECIAL instructions with a GPR destination use rd. JR, stores to + // HI/LO, multiply/divide, syscall, and break have no GPR result. + switch (funct) { + case 0x08u: case 0x0Cu: case 0x0Du: + case 0x11u: case 0x13u: + case 0x18u: case 0x19u: case 0x1Au: case 0x1Bu: + return false; + default: + return rd == reg; + } + } + if (opcode == 0x03u) return reg == 31u; // JAL + if (opcode == 0x01u && (rt == 0x10u || rt == 0x11u)) return reg == 31u; + + // Immediate ALU ops, LUI, and loads write rt. Branches and stores do not. + if ((opcode >= 0x08u && opcode <= 0x0Fu) || + (opcode >= 0x20u && opcode <= 0x26u) || + opcode == 0x30u || opcode == 0x32u) { + return rt == reg; + } + // MFC/CFC from COP0/COP2 write rt; MTC/CTC do not. + if ((opcode == 0x10u || opcode == 0x12u) && (rs == 0u || rs == 2u)) { + return rt == reg; + } + return false; +} + +// Resolve the common absolute indirect-transfer idiom within one basic block: +// +// lui rN, hi(target) +// addiu/ori rN, rN, lo(target) +// jr/jalr rN +// +// The backward scan stops at control flow or any intervening write to rN, so a +// stale constant from another path cannot be mistaken for the transfer target. +static bool exact_resolve_constant_transfer(const PS1Executable& exe, + uint32_t entry, + uint32_t transfer_pc, + uint32_t target_reg, + uint32_t& target_out) { + bool have_low = false; + bool low_is_addiu = false; + uint16_t low = 0; + + for (uint32_t count = 1; count <= 16u; count++) { + uint32_t bytes = count * 4u; + if (transfer_pc < entry + bytes) break; + uint32_t pc = transfer_pc - bytes; + auto word_opt = exe.read_word(pc); + if (!word_opt.has_value()) break; + uint32_t instr = *word_opt; + uint32_t opcode = (instr >> 26) & 0x3Fu; + uint32_t rs = (instr >> 21) & 0x1Fu; + uint32_t rt = (instr >> 16) & 0x1Fu; + + if (!have_low && (opcode == 0x09u || opcode == 0x0Du) && + rs == target_reg && rt == target_reg) { + have_low = true; + low_is_addiu = opcode == 0x09u; + low = static_cast(instr & 0xFFFFu); + continue; + } + if (opcode == 0x0Fu && rt == target_reg) { + uint32_t upper = (instr & 0xFFFFu) << 16; + if (!have_low) { + target_out = upper; + } else if (low_is_addiu) { + target_out = upper + static_cast( + static_cast(static_cast(low))); + } else { + target_out = upper | low; + } + return true; + } + if (exact_instruction_writes_gpr(instr, target_reg)) break; + if (FunctionAnalyzer::is_branch_or_jump(instr)) break; + } + return false; +} + static ExactCf exact_classify_cf(uint32_t pc, uint32_t instr) { ExactCf cf; uint32_t opcode = (instr >> 26) & 0x3Fu; @@ -557,22 +660,11 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries(const std::vector auto callable_direct_jal_target = [&](uint32_t addr) { auto word_opt = exe_.read_word(addr); - if (!word_opt.has_value() || !exact_is_valid_mips_word(*word_opt)) return false; - if (addr == exe_.header.load_address) return true; - - if (addr >= exe_.header.load_address + 8u) { - auto prev8_opt = exe_.read_word(addr - 8u); - if (prev8_opt.has_value() && exact_is_jr_ra_word(*prev8_opt)) return true; - } - - if (exact_is_addiu_sp_neg(*word_opt)) { - if (addr < exe_.header.load_address + 4u) return true; - auto prev_opt = exe_.read_word(addr - 4u); - if (!prev_opt.has_value()) return true; - return exact_classify_cf(addr - 4u, *prev_opt).kind == ExactCfKind::Normal; - } - - return false; + // A JAL encountered by the reachable CFG walk is direct call evidence. + // Requiring a stack prologue here discarded legitimate frameless leaf + // functions and tiny library wrappers. The target still has to decode + // as a valid PS1 instruction and lie inside the verified image bound. + return word_opt.has_value() && exact_is_valid_mips_word(*word_opt); }; auto find_jump_table_targets = [&](uint32_t entry, uint32_t hard_cap, @@ -703,6 +795,15 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries(const std::vector case ExactCfKind::Jalr: if (in_function(delay)) wr.visited.insert(delay); if (in_function(pc + 8u)) work.push(pc + 8u); + { + uint32_t target = 0; + uint32_t target_reg = (instr >> 21) & 0x1Fu; + if (exact_resolve_constant_transfer(exe_, entry, pc, + target_reg, target) && + in_exe(target)) { + wr.direct_jal_targets.insert(target); + } + } break; case ExactCfKind::JrRa: if (in_function(delay)) wr.visited.insert(delay); @@ -712,9 +813,23 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries(const std::vector if (in_function(delay)) wr.visited.insert(delay); { uint32_t jr_rs = (instr >> 21) & 0x1Fu; - for (uint32_t target : find_jump_table_targets(entry, hard_cap, pc, jr_rs)) { - wr.jump_table_targets.insert(target); - if (in_function(target)) work.push(target); + uint32_t target = 0; + if (exact_resolve_constant_transfer(exe_, entry, pc, jr_rs, + target)) { + if (in_function(target)) { + wr.jump_table_targets.insert(target); + work.push(target); + } else if (in_exe(target)) { + // An absolute JR out of the current function is a + // statically proven tail-call entry. + wr.direct_jal_targets.insert(target); + } + } else { + for (uint32_t jt : find_jump_table_targets(entry, hard_cap, + pc, jr_rs)) { + wr.jump_table_targets.insert(jt); + if (in_function(jt)) work.push(jt); + } } } break; @@ -1286,13 +1401,21 @@ FunctionAnalysisResult FunctionAnalyzer::analyze() { break; case ExactCfKind::JrOther: { if (in_fn(delay)) visited.insert(delay); - std::vector jt; uint32_t jr_rs = (instr >> 21) & 0x1Fu; - if (resolve_jt_p3(entry, hard_cap, pc, jr_rs, jt)) { + uint32_t constant_target = 0; + if (exact_resolve_constant_transfer(exe_, entry, pc, jr_rs, + constant_target)) { resolved_jt = true; - for (uint32_t t : jt) if (in_fn(t)) work.push(t); + if (in_fn(constant_target)) work.push(constant_target); + else reached_exit = true; } else { - unresolved_indirect = true; // computed jump we can't bound + std::vector jt; + if (resolve_jt_p3(entry, hard_cap, pc, jr_rs, jt)) { + resolved_jt = true; + for (uint32_t t : jt) if (in_fn(t)) work.push(t); + } else { + unresolved_indirect = true; // computed jump we can't bound + } } break; } diff --git a/recompiler/src/main_psx.cpp b/recompiler/src/main_psx.cpp index 340a9287..c2459eca 100644 --- a/recompiler/src/main_psx.cpp +++ b/recompiler/src/main_psx.cpp @@ -420,7 +420,9 @@ int main(int argc, char** argv) { * 0xCF0): the static recompiler's install-slot hooks tail-dispatch into * exactly these PCs, so they are real execution roots even though no * prologue / preceding jr $ra exists. They are trusted walk roots and - * exempt from the overlay-mode boundary re-check below. + * exempt from the overlay-mode boundary re-check below. Lines of the + * form `call_root 0xXXXXXXXX` carry the same mechanical trust for a + * statically proven direct or constant-register call/tail-call target. * * Seeds are accepted within the loaded image's own bounds — overlay mode * wraps arbitrary regions (kernel RAM at 0x80000000, overlays at @@ -429,6 +431,7 @@ int main(int argc, char** argv) { std::vector file_seeds; std::vector interior_seeds; std::set trusted_root_seeds; + std::set trusted_call_root_seeds; if (extra_funcs_path) { std::ifstream ef(extra_funcs_path); if (ef.is_open()) { @@ -439,6 +442,7 @@ int main(int argc, char** argv) { if (line.empty() || line[0] == '#') continue; bool interior = false; bool trusted_root = false; + bool trusted_call_root = false; const char* p = line.c_str(); if (line.rfind("interior", 0) == 0) { interior = true; @@ -446,6 +450,9 @@ int main(int argc, char** argv) { } else if (line.rfind("dispatch_root", 0) == 0) { trusted_root = true; p += 13; + } else if (line.rfind("call_root", 0) == 0) { + trusted_call_root = true; + p += 9; } uint32_t addr = (uint32_t)std::strtoul(p, nullptr, 16); if (addr >= seed_lo && addr < seed_hi) { @@ -453,15 +460,17 @@ int main(int argc, char** argv) { interior_seeds.push_back(addr); } else { if (trusted_root) trusted_root_seeds.insert(addr); + if (trusted_call_root) trusted_call_root_seeds.insert(addr); file_seeds.push_back(addr); exact_entries.push_back(addr); } } } fmt::print("Loaded {} extra function addresses ({} interior, " - "{} dispatch-root) from {}\n", + "{} dispatch-root, {} call-root) from {}\n", file_seeds.size() + interior_seeds.size(), interior_seeds.size(), trusted_root_seeds.size(), + trusted_call_root_seeds.size(), extra_funcs_path); } else { fmt::print("WARNING: Cannot open extra-funcs file: {}\n", extra_funcs_path); @@ -507,6 +516,10 @@ int main(int argc, char** argv) { fmt::print(" seed 0x{:08X} accepted as dispatch root " "(install-slot class, no boundary evidence)\n", a); roots.insert(a); + } else if (trusted_call_root_seeds.count(a)) { + fmt::print(" seed 0x{:08X} accepted as static call root " + "(direct/constant-register target)\n", a); + roots.insert(a); } else if (callable_boundary(a)) { roots.insert(a); } else { diff --git a/recompiler/tests/reachable_discovery_test.cpp b/recompiler/tests/reachable_discovery_test.cpp index 64429fd2..5ea74b2b 100644 --- a/recompiler/tests/reachable_discovery_test.cpp +++ b/recompiler/tests/reachable_discovery_test.cpp @@ -60,16 +60,18 @@ int main() { auto image = make_exe_buffer(0x2000); const size_t text = 2048; - // Root: direct call to a callable target, unresolved jalr, direct call - // beyond the eventual bound, then return. + // Root: direct call to a framed target, direct call to a frameless leaf, + // unresolved jalr, direct call beyond the eventual bound, then return. put32(image, text + 0x00, jal(kLoad + 0x100)); put32(image, text + 0x04, 0x00000000u); - put32(image, text + 0x08, 0x0320F809u); // jalr $ra,$t9 (unresolved) + put32(image, text + 0x08, jal(kLoad + 0x300)); put32(image, text + 0x0C, 0x00000000u); - put32(image, text + 0x10, jal(kLoad + 0x1100)); + put32(image, text + 0x10, 0x0320F809u); // jalr $ra,$t9 (unresolved) put32(image, text + 0x14, 0x00000000u); - put32(image, text + 0x18, 0x03E00008u); + put32(image, text + 0x18, jal(kLoad + 0x1100)); put32(image, text + 0x1C, 0x00000000u); + put32(image, text + 0x20, 0x03E00008u); + put32(image, text + 0x24, 0x00000000u); // Callable direct target. put32(image, text + 0x100, 0x27BDFFF0u); @@ -81,6 +83,12 @@ int main() { put32(image, text + 0x204, 0x03E00008u); put32(image, text + 0x208, 0x27BD0010u); + // A direct JAL is sufficient callable-boundary evidence even when the + // target is a tiny frameless leaf. + put32(image, text + 0x300, 0x24020001u); // addiu $v0,$zero,1 + put32(image, text + 0x304, 0x03E00008u); + put32(image, text + 0x308, 0x00000000u); + // Callable target outside the verified static bound. put32(image, text + 0x1100, 0x27BDFFF0u); put32(image, text + 0x1104, 0x03E00008u); @@ -95,13 +103,49 @@ int main() { PSXRecomp::FunctionAnalyzer analyzer(exe); const auto reachable = starts(analyzer.analyze_exact_entries({kLoad})); - CHECK(reachable == std::set({kLoad, kLoad + 0x100}), - "reachable analysis follows direct callable JAL only"); + CHECK(reachable == std::set({kLoad, kLoad + 0x100, + kLoad + 0x300}), + "reachable analysis follows framed and frameless direct JAL targets"); CHECK(!reachable.count(kLoad + 0x200), "unseen indirect callback fails closed"); CHECK(!reachable.count(kLoad + 0x1100), "direct target beyond verified bound fails closed"); + // A constant-register JR out of the current function is a statically + // proven tail call. Resolve the common lui/addiu/jr sequence without an + // execution-derived seed. + auto indirect_image = make_exe_buffer(0x1000, kLoad + 0x500); + put32(indirect_image, text + 0x400, 0x24020001u); + put32(indirect_image, text + 0x404, 0x03E00008u); + put32(indirect_image, text + 0x408, 0x00000000u); + put32(indirect_image, text + 0x500, 0x3C088001u); // lui $t0,0x8001 + put32(indirect_image, text + 0x504, 0x25080400u); // addiu $t0,$t0,0x400 + put32(indirect_image, text + 0x508, 0x01000008u); // jr $t0 + put32(indirect_image, text + 0x50C, 0x00000000u); + auto indirect_exe = parse(indirect_image); + PSXRecomp::FunctionAnalyzer indirect_analyzer(indirect_exe); + const auto indirect = starts( + indirect_analyzer.analyze_exact_entries({kLoad + 0x500})); + CHECK(indirect.count(kLoad + 0x400) && indirect.count(kLoad + 0x500), + "constant-register JR discovers its frameless tail-call target"); + + // A packed BIOS thunk is also a backward-scan boundary. The frameless + // leaf after its delay slot must not be swallowed into the thunk run. + auto thunk_image = make_exe_buffer(0x1000); + put32(thunk_image, text + 0x600, 0x240A00B0u); + put32(thunk_image, text + 0x604, 0x01400008u); + put32(thunk_image, text + 0x608, 0x24090008u); + put32(thunk_image, text + 0x60C, 0x00000000u); + put32(thunk_image, text + 0x610, 0x24020001u); + put32(thunk_image, text + 0x614, 0x03E00008u); + put32(thunk_image, text + 0x618, 0x00000000u); + auto thunk_exe = parse(thunk_image); + PSXRecomp::FunctionAnalyzer thunk_analyzer(thunk_exe); + const auto thunk_starts = starts(thunk_analyzer.analyze()); + CHECK(thunk_starts.count(kLoad + 0x600) && + thunk_starts.count(kLoad + 0x610), + "packed BIOS thunk bounds the following frameless leaf"); + PSXRecomp::FunctionAnalyzer seeded_analyzer(exe); const auto seeded = starts( seeded_analyzer.analyze_exact_entries({kLoad, kLoad + 0x200})); diff --git a/recompiler/tests/test_aot_overlay_discovery.py b/recompiler/tests/test_aot_overlay_discovery.py new file mode 100644 index 00000000..fb3d253a --- /dev/null +++ b/recompiler/tests/test_aot_overlay_discovery.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Focused regression checks for play-free overlay call-target discovery.""" +import importlib.util +import pathlib +import struct +import sys + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "compile_overlays", ROOT / "tools" / "compile_overlays.py") +MOD = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MOD) + +LOAD = 0x80010000 + + +def put(data, offset, word): + struct.pack_into("> 2) & 0x03FFFFFF) + + +def main(): + data = bytearray(0x200) + + # A framed root directly calls a frameless leaf. JAL is the boundary + # proof; the leaf deliberately has neither a prologue nor a preceding + # return. + put(data, 0x00, 0x27BDFFF0) + put(data, 0x04, jal(LOAD + 0x40)) + put(data, 0x08, 0x00000000) + put(data, 0x0C, 0x03E00008) + put(data, 0x10, 0x27BD0010) + put(data, 0x40, 0x24020001) + put(data, 0x44, 0x03E00008) + put(data, 0x48, 0x00000000) + + cap = { + "schema": "psxrecomp overlay capture v2", + "load_addr": f"0x{LOAD:08X}", + "size": len(data), + "executed_pcs": [], + "dispatch_entry_pcs": [f"0x{LOAD:08X}"], + "function_entry_pcs": [f"0x{LOAD:08X}"], + "seeds": [f"0x{LOAD:08X}"], + } + seeds, audit = MOD.classify_overlay_seeds( + cap, bytes(data), LOAD, len(data), 0, {}) + assert audit["included_reasons"][LOAD + 0x40] == "DIRECT_JAL_TARGET" + assert f"call_root 0x{LOAD + 0x40:08X}" in seeds + + # A constant-register JR tail-calls backward to a frameless leaf. + put(data, 0x100, 0x27BDFFF0) + put(data, 0x104, 0x3C088001) # lui $t0,0x8001 + put(data, 0x108, 0x25080040) # addiu $t0,$t0,0x40 + put(data, 0x10C, 0x01000008) # jr $t0 + put(data, 0x110, 0x27BD0010) + walk = MOD._walk_overlay_function( + bytes(data), LOAD, len(data), LOAD + 0x100, LOAD + len(data)) + assert walk["static_indirect_targets"] == {LOAD + 0x40} + + print("ALL PASS") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index efb5db0e..6a81a92f 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -28,10 +28,12 @@ discovery quality, not to be run as-is in a build. - **Tomba 1:** 90% of played functions reproduced play-free + ~1964 unplayed extra; live-validated in-game (Dwarf Village went native, no runtime compile). -- **Tomba 2:** 661 functions byte-identical (entry+code_crc) to the shipped/played - vault — provably real — from the MAIN.EXE producer. The BIN header-table producer - now recovers a correct, self-consistent common base (jal-fit, below) — 21 A*.BIN - overlays converge on region 0x80108000 (+3996), 4 weak-signal files safely skipped. +- **Tomba 2:** the clean play-free cache now covers 1187/1856 played entries + (**64.0%**) and byte-matches 1084 (**58.4% entry+code_crc**). Direct-call + provenance for frameless leaves added 74 MAIN.EXE hits over the 60.0% baseline. + The BIN header-table producer recovers a correct, self-consistent common base + (jal-fit, below): 21 A*.BIN overlays converge on region 0x80108000 (+3996), with + 4 weak-signal files safely skipped. ## Correctness guarantee (why partial coverage is safe) @@ -82,8 +84,12 @@ misses, production autocompile (tcc/gcc) self-heals on first visit. ## Next to raise coverage (future session) -1. Frameless-leaf + indirect-call-table seed discovery (helps every game; would - lift MAIN.EXE past ~62% and Tomba 1 past 90%). +1. ~~Direct-call frameless-leaf discovery~~ — DONE. Reachable `jal` targets are + emitted as trusted `call_root` seeds; same-basic-block constant-register + `jalr`/tail-`jr` targets are resolved with clobber checks. Tomba 2 vault recall + rose 60.0% -> 64.0% (+74 MAIN.EXE entries), with 23/23 shards compiling cleanly. + Continue with statically proven function-pointer/jump-table consumers; the + first corpus sweep correctly rejected apparent constants clobbered by `lw`. 2. ~~Header-table base recovery~~ — DONE (jal-fit, above). 3. Recover the weak-signal skipped overlays (Tomba 2 A09/A0J/GAME/OPN): likely compressed/archived or too few intra-overlay calls; may need a decompressor or diff --git a/tools/aot_overlay_spike/coverage_report.py b/tools/aot_overlay_spike/coverage_report.py index 04ce8a02..6a25b878 100644 --- a/tools/aot_overlay_spike/coverage_report.py +++ b/tools/aot_overlay_spike/coverage_report.py @@ -128,9 +128,14 @@ def main(): f'(**{r["recall_entry"]*100:.1f}%** entry-level recall)') md.append(f'- **MISSED live: {r["missed"]}**\n') - os.makedirs(os.path.dirname(a.out_md), exist_ok=True) - open(a.out_md, 'w', encoding='utf-8').write('\n'.join(md)) - json.dump(report, open(a.out_json, 'w'), indent=1) + for output_path in (a.out_md, a.out_json): + output_dir = os.path.dirname(os.path.abspath(output_path)) + os.makedirs(output_dir, exist_ok=True) + with open(a.out_md, 'w', encoding='utf-8', newline='\n') as out_md: + out_md.write('\n'.join(md).rstrip('\n') + '\n') + with open(a.out_json, 'w', encoding='utf-8', newline='\n') as out_json: + json.dump(report, out_json, indent=1) + out_json.write('\n') print(f'wrote {a.out_md}') print(f'wrote {a.out_json}') for src, m in report['sources'].items(): diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index ca446a2a..a39f0000 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -281,6 +281,7 @@ def make_psxexe(load_addr: int, entry_pc: int, data: bytes) -> bytes: INCLUDE_REASONS = { 'DISPATCH_ENTRY', 'DIRECT_JAL_TARGET', + 'STATIC_INDIRECT_TARGET', 'FUNCTION_POINTER_TARGET', 'TOML_DECLARED_ENTRY', # Dispatch-proven PC with no callable boundary (mid-function code reached @@ -504,6 +505,64 @@ def _callable_direct_jal_target(data: bytes, load_addr: int, addr: int) -> bool: return _callable_legacy_seed(data, load_addr, addr) +def _instruction_writes_gpr(word: int, reg: int) -> bool: + if not reg: + return False + op = (word >> 26) & 0x3F + rs = (word >> 21) & 0x1F + rt = (word >> 16) & 0x1F + rd = (word >> 11) & 0x1F + fn = word & 0x3F + if op == 0: + if fn in (0x08, 0x0C, 0x0D, 0x11, 0x13, + 0x18, 0x19, 0x1A, 0x1B): + return False + return rd == reg + if op == 0x03: + return reg == 31 + if op == 0x01 and rt in (0x10, 0x11): + return reg == 31 + if op in range(0x08, 0x10) or op in range(0x20, 0x27) \ + or op in (0x30, 0x32): + return rt == reg + if op in (0x10, 0x12) and rs in (0, 2): + return rt == reg + return False + + +def _resolve_constant_transfer(data: bytes, load_addr: int, + entry: int, transfer_pc: int, + target_reg: int): + """Resolve a same-basic-block lui + addiu/ori constant feeding jr/jalr.""" + low = None + low_is_addiu = False + for count in range(1, 17): + pc = transfer_pc - count * 4 + if pc < entry: + break + word = _word_at(data, load_addr, pc) + if word is None: + break + op = (word >> 26) & 0x3F + rs = (word >> 21) & 0x1F + rt = (word >> 16) & 0x1F + if low is None and op in (0x09, 0x0D) \ + and rs == target_reg and rt == target_reg: + low = word & 0xFFFF + low_is_addiu = op == 0x09 + continue + if op == 0x0F and rt == target_reg: + upper = (word & 0xFFFF) << 16 + if low is None: + return upper + if low_is_addiu and low & 0x8000: + low -= 0x10000 + return (upper + low) & 0xFFFFFFFF if low_is_addiu else upper | low + if _instruction_writes_gpr(word, target_reg) or _is_control_flow(word): + break + return None + + def _walk_overlay_function(data: bytes, load_addr: int, size: int, entry: int, hard_cap: int) -> dict: lo = load_addr @@ -515,6 +574,7 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, work = deque([entry]) visited = set() direct_jals = set() + static_indirect_targets = set() branch_targets = set() jump_table_targets = set() @@ -554,7 +614,7 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, work.append(pc + 8) branch_targets.add(pc + 8) if (lo <= target < hi and (target & 3) == 0 and - _callable_direct_jal_target(data, load_addr, target)): + _is_valid_mips_word(_word_at(data, load_addr, target))): direct_jals.add(target) elif kind == 'jalr': if in_function(delay): @@ -562,16 +622,29 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, if in_function(pc + 8): work.append(pc + 8) branch_targets.add(pc + 8) + target_reg = (word >> 21) & 0x1F + target = _resolve_constant_transfer(data, load_addr, entry, pc, + target_reg) + if target is not None and lo <= target < hi and (target & 3) == 0: + static_indirect_targets.add(target) elif kind == 'jr': if in_function(delay): visited.add(delay) jr_rs = (word >> 21) & 0x1F - for jt in _find_jump_table_targets(data, load_addr, size, - entry, hard_cap, pc, jr_rs): - jump_table_targets.add(jt) - branch_targets.add(jt) - if in_function(jt): - work.append(jt) + target = _resolve_constant_transfer(data, load_addr, entry, pc, jr_rs) + if target is not None and lo <= target < hi and (target & 3) == 0: + if in_function(target): + branch_targets.add(target) + work.append(target) + else: + static_indirect_targets.add(target) + else: + for jt in _find_jump_table_targets(data, load_addr, size, + entry, hard_cap, pc, jr_rs): + jump_table_targets.add(jt) + branch_targets.add(jt) + if in_function(jt): + work.append(jt) elif kind == 'jr_ra': if in_function(delay): visited.add(delay) @@ -579,6 +652,7 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int, return { 'visited': visited, 'direct_jals': direct_jals, + 'static_indirect_targets': static_indirect_targets, 'branch_targets': branch_targets, 'jump_table_targets': jump_table_targets, } @@ -679,14 +753,19 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, not _callable_legacy_seed(data, load_addr, addr)): included.setdefault(addr, 'DISPATCH_INTERIOR') return - elif (reason != 'TOML_DECLARED_ENTRY' and addr in jump_table_targets and + elif (reason not in ('DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET', + 'TOML_DECLARED_ENTRY') and + addr in jump_table_targets and not _callable_legacy_seed(data, load_addr, addr)): excluded[addr] = 'BRANCH_TARGET_ONLY' return if reason in FATAL_SEED_REASONS: raise RuntimeError(f'BUG: refusing to include 0x{addr:08X} as {reason}') old = included.get(addr) - if old is None or old in ('FUNCTION_POINTER_TARGET', 'DISPATCH_INTERIOR'): + if reason in ('DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET'): + # Static call-edge evidence outranks heuristic capture metadata. + included[addr] = reason + elif old is None or old in ('FUNCTION_POINTER_TARGET', 'DISPATCH_INTERIOR'): included[addr] = reason for addr in dispatch_entry_pcs: @@ -725,11 +804,18 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, all_branch_targets.update(walk['branch_targets']) all_branch_targets.update(walk['jump_table_targets']) for target in sorted(walk['direct_jals']): - if target not in known: - include(target, 'DIRECT_JAL_TARGET') - if target in included: - known.add(target) - pending.append(target) + # Always re-include: a target may already be present as a + # heuristic FUNCTION_POINTER_TARGET. Reachable JAL evidence + # upgrades it to a trusted call root. + include(target, 'DIRECT_JAL_TARGET') + if target in included and target not in known: + known.add(target) + pending.append(target) + for target in sorted(walk['static_indirect_targets']): + include(target, 'STATIC_INDIRECT_TARGET') + if target in included and target not in known: + known.add(target) + pending.append(target) if not kernel_window: break @@ -810,6 +896,8 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, return f'interior 0x{addr:08X}' if r == 'DISPATCH_ROOT': return f'dispatch_root 0x{addr:08X}' + if r in ('DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET'): + return f'call_root 0x{addr:08X}' return f'0x{addr:08X}' seeds = [seed_line(addr) for addr in sorted(included)] return seeds, audit @@ -822,6 +910,7 @@ def print_seed_audit(audit: dict) -> None: print(f'dispatch_entry_pcs: {len(audit["dispatch_entry_pcs"])}') print(f'function_entry_pcs: {len(audit["function_entry_pcs"])}') print(f'direct_jal_targets_included: {audit["counts"].get("DIRECT_JAL_TARGET", 0)}') + print(f'static_indirect_targets_included: {audit["counts"].get("STATIC_INDIRECT_TARGET", 0)}') print(f'function_pointer_targets_included: {audit["counts"].get("FUNCTION_POINTER_TARGET", 0)}') print(f'toml_entries_included: {audit["counts"].get("TOML_DECLARED_ENTRY", 0)}') print(f'dispatch_interior_included: {audit["counts"].get("DISPATCH_INTERIOR", 0)}')