diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index 17a7499f..2ae2b088 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -493,6 +493,26 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_overlay_cross_page_delay_codegen.py --recompiler $ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) + # The overlay capture format's trailing delay-slot guard word: legal + # to READ (a branch at the run's last word emits its slot from it), + # illegal to ANALYSE (the word after it does not exist). Also pins + # that the mandatory-delay-slot throw still fires on an untagged + # image, and that no range manifest claims a byte past the image end. + # Bead beads-eio.3.100. + add_test( + NAME overlay_guard_word_analysis_bound + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_overlay_guard_word_analysis_bound.py + --recompiler $ + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) + # Producer half of the same contract: the capture writer declares its + # guard bytes and compile_overlays puts them in the PS-EXE tag the + # recompiler reads. + add_test( + NAME overlay_guard_bytes_declaration + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/../tools/tests/test_overlay_guard_bytes_declaration.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) add_test( NAME fmv_cycle_batch_codegen COMMAND ${Python3_EXECUTABLE} diff --git a/recompiler/include/control_flow.h b/recompiler/include/control_flow.h index 858a2f23..b29d8845 100644 --- a/recompiler/include/control_flow.h +++ b/recompiler/include/control_flow.h @@ -89,6 +89,14 @@ public: private: const PS1Executable& exe_; + // One past the last address that DISCOVERY and BLOCK CONSTRUCTION may + // reach for `func`: the function's own end, clamped to the image's + // analysis bound (PS1Executable::analysis_end_address). The clamp keeps a + // trailing delay-slot guard word — readable on purpose, but with no word + // after it — from ever becoming a block leader or a block's exit + // instruction. See the exe_tag comment in ps1_exe_parser.h. + uint32_t analysis_walk_hi(const Function& func) const; + // Find all basic block boundaries in a function std::set find_block_boundaries(const Function& func); diff --git a/recompiler/include/ps1_exe_parser.h b/recompiler/include/ps1_exe_parser.h index ae941104..aa563033 100644 --- a/recompiler/include/ps1_exe_parser.h +++ b/recompiler/include/ps1_exe_parser.h @@ -55,19 +55,96 @@ struct PS1ExeHeader { static_assert(sizeof(PS1ExeHeader) == 2048, "PS1ExeHeader must be exactly 2048 bytes"); +// -------------------------------------------------------------------------- +// Analysis-bound tag: how many TRAILING bytes of the payload are guard words +// -------------------------------------------------------------------------- +// +// Overlay capture DELIBERATELY appends a coherent guard instruction past the +// end of a dirty-page run (runtime/src/overlay_capture.c, write_json_window: +// `size += 4u`, pinned by runtime/tests/test_interpreter_perf_guards.py). A +// MIPS branch sitting at the run's final word (...FFC) always executes its +// delay slot in the NEXT page (...000); supplying that one word lets overlay +// codegen emit exact semantics and lets the range manifest hash/watch it. +// +// So the guard word is a legal DELAY-SLOT SOURCE but an ILLEGAL BLOCK LEADER: +// the word AFTER it does not exist in the image, so a control transfer AT the +// guard word could never have its mandatory delay slot emitted. The +// recompiler must therefore carry two distinct bounds — a READ bound (the +// whole payload, so a delay slot at the guard word resolves) and an ANALYSIS +// bound (short of the guard words, so discovery and block construction can +// never lead there). +// +// The producer that wrapped the capture states the count explicitly, in a +// magic-tagged field in the otherwise-zero tail of the 2048-byte PS-X EXE +// header (tools/compile_overlays.py, make_psxexe). The recompiler is TOLD +// rather than inferring the count from the payload size: an inference from +// `size % 4096 == 4` fits every capture written by the current format but +// would silently shift underneath any future change to capture granularity, +// and pre-2026-07-25 captures are page-exact with no guard word at all. +// +// An absent or malformed tag means ZERO guard bytes, which is exactly what +// every genuine PS-X EXE (the BIOS, a retail main executable) is: those images +// have no guard word, their analysis bound equals their read bound, and their +// analysis is bit-for-bit unchanged by this mechanism. +namespace exe_tag { +// Offsets are into the 2048-byte header. Retail headers use only +// [0x00, 0x4C) plus an ASCII region string ending well before 0x100; the tail +// is zero-filled. The 8-byte magic makes a false positive on a real EXE +// effectively impossible even if some image did carry bytes there. +inline constexpr uint32_t kGuardMagicOffset = 0x7E0u; +inline constexpr uint32_t kGuardCountOffset = 0x7E8u; // uint32, little-endian +inline constexpr char kGuardMagic[8] = + {'P', 'S', 'X', 'R', 'G', 'R', 'D', '1'}; +// A guard run exists to cover branch delay slots at a page edge; one word is +// what the capture format appends. Cap the accepted value at a page so a +// corrupt tag can never shrink analysis to nothing. +inline constexpr uint32_t kMaxGuardBytes = 4096u; +} // namespace exe_tag + +// Decode the analysis-bound tag. Returns 0 when the tag is absent, when the +// magic does not match, or when the declared count is not a sane, strictly +// interior, instruction-aligned trailer of `file_size` bytes of payload. +uint32_t decode_analysis_guard_bytes(const PS1ExeHeader& header, + uint32_t file_size); + // Parsed PS1 executable with code data class PS1Executable { public: PS1ExeHeader header; std::vector code_data; // Raw binary (file_size bytes) + // Trailing bytes of code_data that exist ONLY to supply architectural + // delay slots to instructions inside the analysis bound. Read-legal, + // analysis-illegal. See the exe_tag comment above. Zero for every image + // that is not an overlay capture wrapped by compile_overlays. + uint32_t analysis_guard_bytes = 0; + // Computed properties uint32_t load_address() const { return header.load_address; } uint32_t entry_point() const { return header.initial_pc; } uint32_t code_size() const { return header.file_size; } + + // READ bound: one past the last byte the image physically supplies. Used + // by read_word so a mandatory delay slot living at a guard word resolves. uint32_t end_address() const { return header.end_address(); } - // Access code as 32-bit words (for MIPS disassembly) + // ANALYSIS bound: one past the last byte that may be DISCOVERED, made a + // block leader, or made a block's exit instruction. Equal to + // end_address() unless the producer declared trailing guard words. + uint32_t analysis_end_address() const { + const uint32_t hi = end_address(); + return (analysis_guard_bytes < hi - load_address()) + ? hi - analysis_guard_bytes + : hi; + } + + bool has_analysis_guard() const { return analysis_guard_bytes != 0; } + + // Access code as 32-bit words (for MIPS disassembly). + // + // Deliberately bounded by end_address(), NOT analysis_end_address(): the + // entire point of a guard word is that it is readable so that a transfer + // at the last analysable word can emit its delay slot. std::optional read_word(uint32_t address) const { if (address < load_address() || address >= end_address()) { return std::nullopt; // Out of range diff --git a/recompiler/src/analysis_db.cpp b/recompiler/src/analysis_db.cpp index 49243774..7ae51fc2 100644 --- a/recompiler/src/analysis_db.cpp +++ b/recompiler/src/analysis_db.cpp @@ -269,7 +269,7 @@ bool recover_jump_table(const PS1Executable& exe, const std::unordered_map& fn_index, RecoveredTable& out) { const uint32_t img_lo = exe.load_address(); - const uint32_t img_hi = exe.end_address(); + const uint32_t img_hi = exe.analysis_end_address(); // jr $rD <- lw $rD, off($rB) DecodedInstruction lw; @@ -508,7 +508,7 @@ AnalysisDb build_analysis_db(const PS1Executable& exe, db.image_size = exe.code_size(); const uint32_t lo = exe.load_address(); - const uint32_t hi = exe.end_address(); + const uint32_t hi = exe.analysis_end_address(); auto read_w = [&](uint32_t a) -> uint32_t { auto w = exe.read_word(a); diff --git a/recompiler/src/basic_block.cpp b/recompiler/src/basic_block.cpp index 310f5dd8..0779419e 100644 --- a/recompiler/src/basic_block.cpp +++ b/recompiler/src/basic_block.cpp @@ -1,3 +1,20 @@ +// --------------------------------------------------------------------------- +// BasicBlockAnalyzer is NOT on any live path. +// +// Nothing in the tree constructs it: the live basic-block walker is +// ControlFlowAnalyzer::analyze_function (control_flow.cpp), built at +// main_psx.cpp:932/954/1138 and rebuilt in +// CodeGenerator::generate_file's mid-function split pre-pass. This file is +// still compiled and therefore still hashed into the codegen tag, so a change +// here invalidates caches without changing any emitted byte. +// +// It is kept, and kept CORRECT, rather than deleted: it already bounded its +// walk properly, and its bounds are now the analysis bound +// (PS1Executable::analysis_end_address) so wiring it up could never +// reintroduce the guard-word-as-block-leader defect (bead beads-eio.3.100). +// Do not "fix" a discovery bug by editing this file — measure which walker +// runs first. +// --------------------------------------------------------------------------- #include "basic_block.h" #include @@ -24,19 +41,19 @@ std::vector BasicBlockAnalyzer::find_leaders() { // The instruction after the delay slot is a leader (fall-through target). // Delay slot is at i+1, so i+2 is the next block's start. uint32_t fall_through = instr.address + 8; // skip instr + delay slot - if (fall_through < exe_.end_address()) { + if (fall_through < exe_.analysis_end_address()) { leaders.push_back(fall_through); } // The branch/jump target is also a leader. if (instr.is_branch && instr.branch_target != 0 && instr.branch_target >= exe_.load_address() && - instr.branch_target < exe_.end_address()) { + instr.branch_target < exe_.analysis_end_address()) { leaders.push_back(instr.branch_target); } if (instr.is_jump && instr.jump_target != 0 && instr.jump_target >= exe_.load_address() && - instr.jump_target < exe_.end_address()) { + instr.jump_target < exe_.analysis_end_address()) { leaders.push_back(instr.jump_target); } } @@ -125,27 +142,27 @@ BasicBlock BasicBlockAnalyzer::build_block(uint32_t start_addr) { if (branch_instr->is_branch) { // Conditional: fall-through + taken target - if (fall_through < exe_.end_address()) + if (fall_through < exe_.analysis_end_address()) block.successors.push_back(fall_through); if (branch_instr->branch_target >= exe_.load_address() && - branch_instr->branch_target < exe_.end_address()) + branch_instr->branch_target < exe_.analysis_end_address()) block.successors.push_back(branch_instr->branch_target); } else if (branch_instr->is_jump) { if (branch_instr->is_jr_ra) { // Function return — no successor within this function } else if (branch_instr->opcode == 0x03) { // JAL — control returns here; fall-through is successor - if (fall_through < exe_.end_address()) + if (fall_through < exe_.analysis_end_address()) block.successors.push_back(fall_through); } else if (branch_instr->jump_target >= exe_.load_address() && - branch_instr->jump_target < exe_.end_address()) { + branch_instr->jump_target < exe_.analysis_end_address()) { block.successors.push_back(branch_instr->jump_target); } } } else if (term_it) { // Straight-line end: fall through to next instruction uint32_t fall_through = block.end_addr + 4; - if (fall_through < exe_.end_address()) + if (fall_through < exe_.analysis_end_address()) block.successors.push_back(fall_through); } } @@ -180,7 +197,7 @@ std::vector BasicBlockAnalyzer::analyze() { uint32_t start = leaders[li]; uint32_t next_leader = (li + 1 < leaders.size()) ? leaders[li + 1] - : exe_.end_address(); + : exe_.analysis_end_address(); // Only create a block if start_addr is in the decoded range if (addr_to_idx_.count(start) == 0) continue; @@ -258,20 +275,20 @@ std::vector BasicBlockAnalyzer::analyze() { if (branch_instr->is_branch) { // Conditional: two successors - if (after_branch < exe_.end_address()) + if (after_branch < exe_.analysis_end_address()) block.successors.push_back(after_branch); if (branch_instr->branch_target >= exe_.load_address() && - branch_instr->branch_target < exe_.end_address()) + branch_instr->branch_target < exe_.analysis_end_address()) block.successors.push_back(branch_instr->branch_target); } else { // jump if (branch_instr->is_jr_ra) { // Return: no intra-function successors } else if (branch_instr->opcode == 0x03) { // JAL // JAL: fall-through after delay slot - if (after_branch < exe_.end_address()) + if (after_branch < exe_.analysis_end_address()) block.successors.push_back(after_branch); } else if (branch_instr->jump_target >= exe_.load_address() && - branch_instr->jump_target < exe_.end_address()) { + branch_instr->jump_target < exe_.analysis_end_address()) { block.successors.push_back(branch_instr->jump_target); } else if (branch_instr->opcode == 0x08 || branch_instr->opcode == 0x09) { // JR/JALR to register — dynamic target, can't statically determine @@ -280,10 +297,10 @@ std::vector BasicBlockAnalyzer::analyze() { } else { // Straight-line: fall through uint32_t fall_through = block.end_addr + 4; - if (fall_through < exe_.end_address() && fall_through < next_leader) { + if (fall_through < exe_.analysis_end_address() && fall_through < next_leader) { // Only add if it's in range (typically the next block) } - if (fall_through < exe_.end_address()) + if (fall_through < exe_.analysis_end_address()) block.successors.push_back(fall_through); } diff --git a/recompiler/src/code_generator.cpp b/recompiler/src/code_generator.cpp index 4194abee..e9e9a946 100644 --- a/recompiler/src/code_generator.cpp +++ b/recompiler/src/code_generator.cpp @@ -3441,7 +3441,9 @@ std::string CodeGenerator::generate_file( if (config_.split_mid_function_targets) { std::set cfgs_to_scan; // Which CFGs to scan (empty = all) uint32_t exe_start = exe_.header.load_address; - uint32_t exe_end = exe_.end_address(); + // Analysis bound: a split at a trailing delay-slot guard word would + // mint a function whose first instruction has no successor word. + uint32_t exe_end = exe_.analysis_end_address(); int total_new = 0; // Safety cap only — the loop must run to convergence. Unconverged // targets emit `call_by_address(mid-func); return;` which misses the @@ -3675,6 +3677,27 @@ std::string CodeGenerator::generate_ranges_manifest( blk.exit_instr.address == blk.end_addr && hi <= UINT32_MAX - 4u) { hi += 4u; } + /* A validity range may never claim a byte the shard never saw. + * The candidate CRC and the page-generation watch are computed + * over these ranges, so a range past the image end would validate + * the shard against bytes that were not part of its input — the + * cache would then "confirm" garbage. + * + * Two ways the +4 above can overrun, both real: + * - a block leading at a trailing guard word (the defect this + * manifest is a second-order victim of; fixed upstream by the + * analysis bound, clamped here as well so the invariant does + * not depend on that fix staying in place), and + * - a branch-likely opcode (0x14-0x17, RESERVED on the R3000A) + * as the image's very last word. translate_basic_block + * short-circuits those into an inline RI raise BEFORE reading + * any delay slot, so the mandatory-delay-slot throw never + * fires and generation succeeds with hi == image_end + 4. + * Clamp to the READ bound, not the analysis bound: a guard word + * genuinely IS compiled into the shard when it serves as a delay + * slot, and must participate in the CRC. */ + const uint32_t image_hi = exe_.end_address(); + if (hi > image_hi) hi = image_hi; if (hi > lo) iv.emplace_back(lo, hi); } if (iv.empty()) continue; diff --git a/recompiler/src/control_flow.cpp b/recompiler/src/control_flow.cpp index 68e45234..833db11e 100644 --- a/recompiler/src/control_flow.cpp +++ b/recompiler/src/control_flow.cpp @@ -175,15 +175,33 @@ ControlFlowInstr ControlFlowAnalyzer::analyze_instruction(uint32_t addr, uint32_ return cf; } +uint32_t ControlFlowAnalyzer::analysis_walk_hi(const Function& func) const { + const uint32_t analysis_hi = exe_.analysis_end_address(); + const uint32_t walk_lo = + func.alias_walk_lo ? func.alias_walk_lo : func.start_addr; + // Only clamp when the clamp still leaves a non-empty walk. A function that + // begins at or past the analysis bound is a degenerate discovery artefact; + // leave its range alone so the emitter's mandatory-delay-slot check keeps + // failing closed on it rather than silently producing an empty CFG. + if (func.end_addr > analysis_hi && analysis_hi > walk_lo) return analysis_hi; + return func.end_addr; +} + +// NOTE: find_block_boundaries / build_basic_blocks / link_basic_blocks / +// detect_loops below are NOT on the live path — analyze_function does its own +// single-pass walk (see the comment there). They are kept in sync with the +// analysis bound anyway so wiring them up can never reintroduce the +// guard-word-as-block-leader defect. std::set ControlFlowAnalyzer::find_block_boundaries(const Function& func) { std::set boundaries; + const uint32_t walk_hi = analysis_walk_hi(func); // Function start is always a boundary boundaries.insert(func.start_addr); // Scan all instructions in function uint32_t addr = func.start_addr; - while (addr < func.end_addr) { + while (addr < walk_hi) { auto instr_opt = exe_.read_word(addr); if (!instr_opt.has_value()) { break; @@ -195,7 +213,7 @@ std::set ControlFlowAnalyzer::find_block_boundaries(const Function& fu ControlFlowInstr cf = analyze_instruction(addr, instr); // If branch/jump has a target, target is a boundary - if (cf.target != 0 && cf.target >= func.start_addr && cf.target < func.end_addr) { + if (cf.target != 0 && cf.target >= func.start_addr && cf.target < walk_hi) { boundaries.insert(cf.target); } @@ -204,7 +222,7 @@ std::set ControlFlowAnalyzer::find_block_boundaries(const Function& fu cf.type == ControlFlowType::JumpLink || cf.type == ControlFlowType::JumpLinkReg) { uint32_t fall_through = addr + 8; // After delay slot - if (fall_through < func.end_addr) { + if (fall_through < walk_hi) { boundaries.insert(fall_through); } } @@ -215,7 +233,7 @@ std::set ControlFlowAnalyzer::find_block_boundaries(const Function& fu cf.type == ControlFlowType::JumpRegister || cf.type == ControlFlowType::Return) { uint32_t after_delay = addr + 8; - if (after_delay < func.end_addr) { + if (after_delay < walk_hi) { boundaries.insert(after_delay); } } @@ -232,6 +250,7 @@ std::map ControlFlowAnalyzer::build_basic_blocks( const std::set& boundaries) { std::map blocks; + const uint32_t walk_hi = analysis_walk_hi(func); std::vector boundary_vec(boundaries.begin(), boundaries.end()); std::sort(boundary_vec.begin(), boundary_vec.end()); @@ -242,7 +261,7 @@ std::map ControlFlowAnalyzer::build_basic_blocks( if (i + 1 < boundary_vec.size()) { block.end_addr = boundary_vec[i + 1] - 4; } else { - block.end_addr = func.end_addr - 4; + block.end_addr = walk_hi - 4; } if (block.end_addr < block.start_addr) continue; @@ -357,8 +376,14 @@ ControlFlowGraph ControlFlowAnalyzer::analyze_function(const Function& func) { // all stay intra-function. The emitted alias enters via goto to its // start_addr block; walk_lo is the host's start. uint32_t walk_lo = func.alias_walk_lo ? func.alias_walk_lo : func.start_addr; + // Discovery/block-construction bound. Normally exactly func.end_addr; on + // an overlay capture that carries trailing delay-slot guard words it stops + // short of them, so the guard word can never become a block leader or a + // block's exit instruction (its own delay slot does not exist). It stays + // READABLE: a transfer at walk_hi - 4 still emits its slot from the guard. + const uint32_t walk_hi = analysis_walk_hi(func); cfg.function_start = walk_lo; - cfg.function_end = func.end_addr; + cfg.function_end = walk_hi; cfg.producer_lo = func.producer_lo; cfg.producer_hi = func.producer_hi; @@ -372,17 +397,17 @@ ControlFlowGraph ControlFlowAnalyzer::analyze_function(const Function& func) { // aliases produce identical CFGs. boundary_vec.push_back(func.start_addr); for (uint32_t e : func.alias_group_entries) { - if (e >= walk_lo && e < func.end_addr) boundary_vec.push_back(e); + if (e >= walk_lo && e < walk_hi) boundary_vec.push_back(e); } } auto add_boundary = [&](uint32_t target) { - if (target >= walk_lo && target < func.end_addr) { + if (target >= walk_lo && target < walk_hi) { boundary_vec.push_back(target); } }; // Scan instructions for branches/jumps to find block starts - for (uint32_t addr = walk_lo; addr < func.end_addr; addr += 4) { + for (uint32_t addr = walk_lo; addr < walk_hi; addr += 4) { auto instr_opt = exe_.read_word(addr); if (!instr_opt) continue; uint32_t instr = *instr_opt; @@ -390,13 +415,13 @@ ControlFlowGraph ControlFlowAnalyzer::analyze_function(const Function& func) { if (is_control_flow(instr)) { ControlFlowInstr cf = analyze_instruction(addr, instr); - if (cf.target != 0 && cf.target >= walk_lo && cf.target < func.end_addr) { + if (cf.target != 0 && cf.target >= walk_lo && cf.target < walk_hi) { add_boundary(cf.target); } // After delay slot is a new block uint32_t after_delay = addr + 8; - if (after_delay < func.end_addr) { + if (after_delay < walk_hi) { add_boundary(after_delay); } } @@ -408,7 +433,7 @@ ControlFlowGraph ControlFlowAnalyzer::analyze_function(const Function& func) { for (size_t i = 0; i < boundary_vec.size(); i++) { BasicBlock block; block.start_addr = boundary_vec[i]; - block.end_addr = (i + 1 < boundary_vec.size()) ? boundary_vec[i + 1] - 4 : func.end_addr - 4; + block.end_addr = (i + 1 < boundary_vec.size()) ? boundary_vec[i + 1] - 4 : walk_hi - 4; if (block.end_addr < block.start_addr) continue; @@ -445,7 +470,7 @@ ControlFlowGraph ControlFlowAnalyzer::analyze_function(const Function& func) { // Add successors — only targets within this function's block set auto in_func = [&](uint32_t addr) { return addr >= walk_lo && - addr < func.end_addr && + addr < walk_hi && std::binary_search(boundary_vec.begin(), boundary_vec.end(), addr); }; if (block.exit_instr.type == ControlFlowType::Branch) { diff --git a/recompiler/src/function_analysis.cpp b/recompiler/src/function_analysis.cpp index bfb86862..ede40acc 100644 --- a/recompiler/src/function_analysis.cpp +++ b/recompiler/src/function_analysis.cpp @@ -11,7 +11,7 @@ FunctionAnalyzer::FunctionAnalyzer(const PS1Executable& exe) : exe_(exe) {} void FunctionAnalyzer::add_forced_entry(uint32_t addr) { // Validate address is within the EXE range - if (addr >= exe_.header.load_address && addr < exe_.end_address()) { + if (addr >= exe_.header.load_address && addr < exe_.analysis_end_address()) { forced_entry_points_.push_back(addr); } } @@ -782,7 +782,7 @@ bool resolve_exact_bounded_jump_table( }; if (producer_lo == 0u && producer_hi == 0u) { producer_lo = exe.header.load_address; - producer_hi = exe.end_address(); + producer_hi = exe.analysis_end_address(); } if (producer_lo >= producer_hi || (producer_lo & 3u) != 0u || (producer_hi & 3u) != 0u) { @@ -1119,8 +1119,13 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( fmt::print("\n=== Exact-Entry Function Analysis ===\n\n"); + // Discovery bound, NOT the read bound. An overlay capture deliberately + // carries trailing delay-slot guard words (ps1_exe_parser.h exe_tag): + // readable so a branch at the last analysable word can emit its slot, but + // never admissible as code themselves, because the word after a guard + // word does not exist in the image. auto in_exe = [&](uint32_t addr) { - return addr >= exe_.header.load_address && addr < exe_.end_address() && (addr & 3u) == 0; + return addr >= exe_.header.load_address && addr < exe_.analysis_end_address() && (addr & 3u) == 0; }; auto producer_for = [&](uint32_t addr) -> int { @@ -1265,7 +1270,7 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( } else { ExactJumpTable table; uint32_t producer_lo = exe_.header.load_address; - uint32_t producer_hi = exe_.end_address(); + uint32_t producer_hi = exe_.analysis_end_address(); if (entry_producer >= 0) { producer_lo = producer_ranges[entry_producer].first; producer_hi = producer_ranges[entry_producer].second; @@ -1337,7 +1342,7 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( std::map>> candidate_evidence; for (size_t i = 0; i < round_entries.size(); i++) { uint32_t hard_cap = (i + 1 < round_entries.size()) - ? round_entries[i + 1] : exe_.end_address(); + ? round_entries[i + 1] : exe_.analysis_end_address(); ExactWalkResult wr = walk(round_entries[i], hard_cap); candidate_targets.insert(wr.direct_jal_targets.begin(), wr.direct_jal_targets.end()); @@ -1372,7 +1377,7 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( auto owner_it = std::prev(target_it); auto next_it = std::next(target_it); uint32_t hard_cap = next_it != known_entries.end() - ? *next_it : exe_.end_address(); + ? *next_it : exe_.analysis_end_address(); ExactWalkResult owner_walk = walk(*owner_it, hard_cap); if (owner_walk.visited.count(target)) { known_entries.erase(target_it); @@ -1392,7 +1397,11 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( for (size_t i = 0; i < starts_vec.size(); i++) { uint32_t entry = starts_vec[i]; - uint32_t hard_cap = (i + 1 < starts_vec.size()) ? starts_vec[i + 1] : exe_.end_address(); + // The final entry's cap is the ANALYSIS end, not the image end, so a + // walk that falls through the last real word of an overlay capture + // stops there instead of absorbing the trailing guard word into + // func.end_addr (which made the guard word the last block's leader). + uint32_t hard_cap = (i + 1 < starts_vec.size()) ? starts_vec[i + 1] : exe_.analysis_end_address(); ExactWalkResult wr = walk(entry, hard_cap); if (wr.visited.empty()) continue; @@ -1437,7 +1446,7 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries( for (size_t i = 0; i < starts_vec.size(); i++) { uint32_t host_start = starts_vec[i]; uint32_t hard_cap = (i + 1 < starts_vec.size()) - ? starts_vec[i + 1] : exe_.end_address(); + ? starts_vec[i + 1] : exe_.analysis_end_address(); ExactWalkResult wr = walk(host_start, hard_cap); if (wr.visited.empty()) continue; uint32_t host_end = *wr.visited.rbegin() + 4u; @@ -1483,7 +1492,7 @@ FunctionAnalysisResult FunctionAnalyzer::analyze() { std::vector return_addresses; uint32_t current_addr = exe_.header.load_address; - uint32_t end_addr = exe_.end_address(); + uint32_t end_addr = exe_.analysis_end_address(); fmt::print("Scanning {} KB of code for function returns...\n", (end_addr - current_addr) / 1024); @@ -1537,7 +1546,7 @@ FunctionAnalysisResult FunctionAnalyzer::analyze() { fmt::print("Following JAL call targets to discover additional functions...\n"); uint32_t exe_start = exe_.header.load_address; - uint32_t exe_end = exe_.end_address(); + uint32_t exe_end = exe_.analysis_end_address(); for (uint32_t addr = exe_start; addr < exe_end; addr += 4) { auto word_opt = exe_.read_word(addr); @@ -1832,7 +1841,7 @@ FunctionAnalysisResult FunctionAnalyzer::analyze() { // tables minted as function starts (those never walk as clean code: // ~44% of random data words carry an invalid primary opcode). auto in_exe_p3 = [&](uint32_t a) { - return a >= exe_.header.load_address && a < exe_.end_address() && (a & 3u) == 0; + return a >= exe_.header.load_address && a < exe_.analysis_end_address() && (a & 3u) == 0; }; // Same primary-opcode validity table as is_likely_data_section so the two // agree on what "clean code" means (COP2/GTE, LWC2/SWC2 are valid PS1 ops). diff --git a/recompiler/src/main_cli.cpp b/recompiler/src/main_cli.cpp index 69e85273..ec14078b 100644 --- a/recompiler/src/main_cli.cpp +++ b/recompiler/src/main_cli.cpp @@ -312,11 +312,11 @@ int build_project(const Options& options, const fs::path& exe_dir) { find_bios_profile(framework_source, bios_profile); std::set seeds = {image.entry_point()}; - for (uint32_t address = image.load_address(); address + 4 <= image.end_address(); address += 4) { + for (uint32_t address = image.load_address(); address + 4 <= image.analysis_end_address(); address += 4) { auto word = image.read_word(address); if (!word || ((*word >> 26) & 0x3F) != 0x03) continue; uint32_t target = (address & 0xF0000000u) | ((*word & 0x03FFFFFFu) << 2); - if (target >= image.load_address() && target < image.end_address()) seeds.insert(target); + if (target >= image.load_address() && target < image.analysis_end_address()) seeds.insert(target); } std::string seed_text = fmt::format("# Auto-generated JAL targets for {}\n", serial); for (uint32_t seed : seeds) seed_text += fmt::format("0x{:08X}\n", seed); diff --git a/recompiler/src/main_psx.cpp b/recompiler/src/main_psx.cpp index bd17a055..bf276000 100644 --- a/recompiler/src/main_psx.cpp +++ b/recompiler/src/main_psx.cpp @@ -514,6 +514,13 @@ int main(int argc, char** argv) { fmt::print(" Code Size: {} bytes ({} KB)\n", exe->header.file_size, exe->header.file_size / 1024); fmt::print(" End Address: 0x{:08X}\n", exe->end_address()); + if (exe->has_analysis_guard()) { + // Only printed for a producer-tagged capture, so a non-overlay image's + // log stays byte-identical to the untagged build. + fmt::print(" Analysis End: 0x{:08X} ({} trailing delay-slot guard " + "byte(s), readable but not discoverable)\n", + exe->analysis_end_address(), exe->analysis_guard_bytes); + } fmt::print(" Global Pointer: 0x{:08X}\n", exe->header.initial_gp); fmt::print(" Stack Pointer: 0x{:08X}\n\n", exe->header.initial_sp); @@ -656,7 +663,7 @@ int main(int argc, char** argv) { std::ifstream ef(extra_funcs_path); if (ef.is_open()) { const uint32_t seed_lo = exe->header.load_address; - const uint32_t seed_hi = exe->end_address(); + const uint32_t seed_hi = exe->analysis_end_address(); std::string line; while (std::getline(ef, line)) { if (line.empty() || line[0] == '#') continue; @@ -822,7 +829,7 @@ int main(int argc, char** argv) { for (uint32_t i = 0; i < 3u; i++) { auto value = exe->read_word(a + i * 4u); if (!value.has_value() || (*value & 3u) != 0u || - *value < exe_lo || *value >= exe->end_address()) { + *value < exe_lo || *value >= exe->analysis_end_address()) { return false; } } @@ -1006,7 +1013,7 @@ int main(int argc, char** argv) { std::set forced; forced.insert(exe->header.initial_pc); const uint32_t exe_lo = exe->header.load_address; - const uint32_t exe_hi = exe->end_address(); + const uint32_t exe_hi = exe->analysis_end_address(); auto read_w = [&](uint32_t a) -> uint32_t { auto w = exe->read_word(a); return w.has_value() ? *w : 0u; diff --git a/recompiler/src/main_toml.cpp b/recompiler/src/main_toml.cpp index 309a3d45..21cf0691 100644 --- a/recompiler/src/main_toml.cpp +++ b/recompiler/src/main_toml.cpp @@ -93,7 +93,7 @@ static std::string id_from_exe(const fs::path& exe_path) { static std::set find_jal_targets(const PSXRecomp::PS1Executable& exe) { std::set targets; const uint32_t base = exe.load_address(); - const uint32_t end = exe.end_address(); + const uint32_t end = exe.analysis_end_address(); for (uint32_t addr = base; addr + 4 <= end; addr += 4) { auto word = exe.read_word(addr); @@ -116,7 +116,7 @@ static std::set find_jal_targets(const PSXRecomp::PS1Executable& exe) static std::set find_after_return(const PSXRecomp::PS1Executable& exe) { std::set after_ret; const uint32_t base = exe.load_address(); - const uint32_t end = exe.end_address(); + const uint32_t end = exe.analysis_end_address(); for (uint32_t addr = base; addr + 8 <= end; addr += 4) { auto word = exe.read_word(addr); diff --git a/recompiler/src/ps1_exe_parser.cpp b/recompiler/src/ps1_exe_parser.cpp index 62fa3e14..196097b8 100644 --- a/recompiler/src/ps1_exe_parser.cpp +++ b/recompiler/src/ps1_exe_parser.cpp @@ -6,6 +6,27 @@ namespace PSXRecomp { +uint32_t decode_analysis_guard_bytes(const PS1ExeHeader& header, + uint32_t file_size) { + const uint8_t* raw = reinterpret_cast(&header); + if (std::memcmp(raw + exe_tag::kGuardMagicOffset, exe_tag::kGuardMagic, + sizeof(exe_tag::kGuardMagic)) != 0) { + return 0; // no tag: a genuine PS-X EXE, or an untagged producer + } + uint32_t declared = 0; + std::memcpy(&declared, raw + exe_tag::kGuardCountOffset, sizeof(declared)); + if (declared == 0) return 0; + // A guard trailer must be instruction-aligned, sane in size, and leave at + // least one analysable instruction behind. Anything else is a malformed + // tag: fall back to "no guard", which is the fail-closed answer (the + // emitter's mandatory-delay-slot check then still refuses to emit a + // transfer it cannot complete). + if ((declared & 3u) != 0) return 0; + if (declared > exe_tag::kMaxGuardBytes) return 0; + if (file_size < declared + 4u) return 0; + return declared; +} + bool apply_static_analysis_bound(PS1Executable& exe, uint32_t configured_size, std::string& error_msg) { @@ -73,6 +94,10 @@ bool apply_static_analysis_bound(PS1Executable& exe, exe.header.file_size = configured_size; exe.code_data.resize(configured_size); + // A title-configured truncation cuts at a page boundary inside real + // payload; whatever guard trailer the producer declared no longer sits at + // the new end. Drop it so the analysis bound is the truncation itself. + exe.analysis_guard_bytes = 0; return true; } @@ -288,6 +313,13 @@ std::optional PS1ExeParser::parse_buffer( exe.header.file_size ); + // Analysis-bound tag: how many trailing bytes are delay-slot guard words + // that must stay readable but must never be discovered as code. Absent on + // every genuine PS-X EXE, so this is a no-op for the BIOS and for retail + // main executables. + exe.analysis_guard_bytes = + decode_analysis_guard_bytes(exe.header, exe.header.file_size); + // Final validation if (!exe.validate(error_msg)) { return std::nullopt; diff --git a/recompiler/tests/test_overlay_guard_word_analysis_bound.py b/recompiler/tests/test_overlay_guard_word_analysis_bound.py new file mode 100644 index 00000000..1474b3ea --- /dev/null +++ b/recompiler/tests/test_overlay_guard_word_analysis_bound.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""The overlay delay-slot guard word is READ-legal but ANALYSIS-illegal. + +An overlay capture deliberately carries one coherent guard instruction past +the end of a dirty-page run (runtime/src/overlay_capture.c write_json_window, +`size += 4u`), so that a MIPS branch sitting at the run's final word (...FFC) +has its architectural delay slot (...000) available. That makes the guard word +a legal DELAY-SLOT SOURCE and an illegal BLOCK LEADER: the word AFTER it does +not exist in the image, so a control transfer AT the guard word could never +emit its own mandatory delay slot. + +The producer states the guard-byte count in the PS-EXE analysis-bound tag +(tools/compile_overlays.py make_psxexe -> recompiler/include/ps1_exe_parser.h +namespace exe_tag). The recompiler must then: + + A. TAGGED, guard word decodes as control flow, word before it does not. + This is the exact shape that failed 17 of 67 Ape Escape (SCUS-94423) + shards at overlay 0x80136000 (bead beads-eio.3.100): discovery walked + into the guard word, block construction made it the last block's LEADER, + the "also check last instruction itself" promotion made it that block's + exit_instr, and the emitter correctly refused to emit a transfer whose + delay slot it could not read. Generation must now SUCCEED, the guard word + must not be a block leader, and control must leave the unit by publishing + the guard-word PC for the runtime to dispatch. + + B. UNTAGGED, byte-for-byte identical payload. The producer says there is no + guard word, so the last word IS ordinary code with a missing mandatory + delay slot, and generation must still FAIL CLOSED with the delay-slot + diagnostic. Emitting a control transfer without its architectural delay + slot silently corrupts guest state; that throw is correct and must + survive any fix to (A). A and B differ ONLY in the header tag, so this + pair proves the tag is what changed behaviour and that nothing was + weakened. + + C. TAGGED, with a real branch at the last analysable word. The guard word + must still be readable AS a delay slot, must be emitted, and must be + covered by the range manifest so its bytes join the candidate CRC and the + page-generation watch. + + D. No range in the manifest may ever claim a byte past the image end. The + +4 extension in generate_ranges_manifest fires for any block whose exit + instruction IS its last word; a branch-likely opcode (0x14-0x17, RESERVED + on the R3000A) as the image's very last word reaches that extension with + generation SUCCEEDING, because translate_basic_block short-circuits those + into an inline RI raise before any delay slot is read. Unclamped, the + manifest then hashes 4 bytes the shard never saw and the cache validates + against garbage. + +Usage: test_overlay_guard_word_analysis_bound.py [--recompiler ] +Exit 0 = PASS. +""" + +import argparse +import os +import struct +import subprocess +import sys +import tempfile + +LOAD = 0x80010000 +PAGE_BYTES = 4096 + +# Analysis-bound tag; must stay in lockstep with ps1_exe_parser.h exe_tag and +# with tools/compile_overlays.py. +GUARD_TAG_MAGIC_OFFSET = 0x7E0 +GUARD_TAG_COUNT_OFFSET = 0x7E8 +GUARD_TAG_MAGIC = b"PSXRGRD1" + +NOP = 0x00000000 +ADDIU_V0_0x2000 = 0x24022000 # addiu $v0, $zero, 0x2000 — NOT control flow +BEQ_V1_V0_FWD = 0x1062005A # beq $v1, $v0, +0x5A — IS control flow +BEQ_ZERO_FWD = 0x10000002 # beq $zero, $zero, +2 — IS control flow +BEQL_RESERVED = 0x50800008 # beql — RESERVED on the R3000A (opcode 0x14) + + +def words(seq): + return b"".join(struct.pack(" bytes: + header = bytearray(2048) + header[0:8] = b"PS-X EXE" + struct.pack_into(" int: + here = os.path.dirname(os.path.abspath(__file__)) + default_recompiler = os.path.normpath( + os.path.join(here, "..", "build", "psxrecomp-game.exe")) + parser = argparse.ArgumentParser() + parser.add_argument("--recompiler", default=default_recompiler) + args = parser.parse_args() + if not os.path.isfile(args.recompiler): + raise SystemExit(f"recompiler not found: {args.recompiler}") + + failures = [] + payload = guard_shape_payload() + image_end = LOAD + len(payload) # 0x80012004 + guard_pc = image_end - 4 # 0x80012000 + last_analysable = guard_pc - 4 # 0x80011FFC + + with tempfile.TemporaryDirectory() as root: + # ---- A: tagged guard word must not be analysed as code ------------- + proc, out_dir = run_codegen(args.recompiler, root, "tagged_guard", + payload, 4) + combined = (proc.stdout or "") + (proc.stderr or "") + if proc.returncode != 0: + failures.append( + "tagged guard word still fails generation: " + + next((l for l in combined.splitlines() if "what()" in l), + combined[-400:])) + else: + source, manifest = read_generated(out_dir) + if f"0x{last_analysable:08X}" not in source: + failures.append( + "tagged guard image did not emit its last analysable " + f"word 0x{last_analysable:08X}") + # The guard word must never be emitted as an executed instruction. + # It may only appear as a published PC handed back to the runtime. + for line in source.splitlines(): + if f"0x{guard_pc:08X}" not in line: + continue + if "cpu->pc =" in line or "psx_check_interrupts_at" in line: + continue + failures.append( + "guard word was analysed as code, not just published as a " + f"PC: {line.strip()}") + if f"cpu->pc = 0x{guard_pc:08X}u" not in source: + failures.append( + "tagged guard image does not publish the guard-word PC " + "for runtime dispatch at the unit edge") + for lo, hi in manifest_ranges(manifest): + if lo <= guard_pc < hi: + failures.append( + f"range [0x{lo:08X},0x{hi:08X}) claims the guard word " + "even though no emitted instruction uses it") + + # ---- B: identical bytes, no tag -> the throw must still fire ------- + proc_b, _ = run_codegen(args.recompiler, root, "untagged_same_bytes", + payload, 0) + combined_b = (proc_b.stdout or "") + (proc_b.stderr or "") + if proc_b.returncode == 0: + failures.append( + "untagged image with a missing mandatory delay slot generated " + "a shard instead of failing closed") + elif "mandatory delay slot" not in combined_b: + failures.append( + "untagged image failed without the delay-slot diagnostic: " + + combined_b[-400:]) + elif f"0x{guard_pc:08X}" not in combined_b: + failures.append( + "delay-slot diagnostic did not name the offending transfer PC " + f"0x{guard_pc:08X}: " + combined_b[-400:]) + + # ---- B2: a malformed declaration must fail closed, not open ------- + # 6 is not instruction-aligned, so decode_analysis_guard_bytes must + # reject it and fall back to "no guard" -- i.e. behave exactly like B. + proc_b2, _ = run_codegen(args.recompiler, root, "malformed_tag", + payload, 6) + combined_b2 = (proc_b2.stdout or "") + (proc_b2.stderr or "") + if proc_b2.returncode == 0: + failures.append( + "a misaligned guard-byte declaration widened analysis instead " + "of being rejected") + elif "mandatory delay slot" not in combined_b2: + failures.append( + "misaligned guard-byte declaration failed for the wrong " + "reason: " + combined_b2[-400:]) + + # ---- C: the guard word must still work AS a delay slot ------------ + branch_body = [NOP] * ((0x2000 // 4) - 1) + [BEQ_ZERO_FWD] + branch_payload = words(branch_body + [ADDIU_V0_0x2000]) + proc_c, out_c = run_codegen(args.recompiler, root, "guard_is_slot", + branch_payload, 4) + if proc_c.returncode != 0: + failures.append( + "branch at the last analysable word could not use the guard " + "word as its delay slot: " + + ((proc_c.stderr or "") + (proc_c.stdout or ""))[-400:]) + else: + source_c, manifest_c = read_generated(out_c) + if f"0x{ADDIU_V0_0x2000:08X}" not in source_c: + failures.append( + "guard word was not emitted as the branch's delay slot") + covered = any(lo <= guard_pc and guard_pc + 4 <= hi + for lo, hi in manifest_ranges(manifest_c)) + if not covered: + failures.append( + "range manifest does not hash the guard word that is " + "compiled in as a delay slot") + + # ---- D: no range may exceed the captured image end ----------------- + # beql as the very last word: RESERVED on the R3000A, so + # translate_basic_block emits an inline RI raise and never reads a + # delay slot -- generation succeeds and the +4 range extension fires. + beql_payload = words([NOP] * 6 + [BEQL_RESERVED]) + beql_end = LOAD + len(beql_payload) + proc_d, out_d = run_codegen(args.recompiler, root, "beql_tail", + beql_payload, 0) + if proc_d.returncode != 0: + failures.append( + "reserved branch-likely tail aborted generation: " + + ((proc_d.stderr or "") + (proc_d.stdout or ""))[-400:]) + else: + _, manifest_d = read_generated(out_d) + ranges_d = manifest_ranges(manifest_d) + if not ranges_d: + failures.append("reserved branch-likely tail emitted no ranges") + for lo, hi in ranges_d: + if hi > beql_end: + failures.append( + f"range [0x{lo:08X},0x{hi:08X}) claims " + f"{hi - beql_end} byte(s) past the image end " + f"0x{beql_end:08X}") + + if failures: + for failure in failures: + print("FAIL:", failure) + return 1 + print("PASS: guard word is a delay-slot source and never a block leader; " + "the mandatory-delay-slot throw still fires on an untagged image; " + "no range manifest claims a byte past the captured image end") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/runtime/src/overlay_capture.c b/runtime/src/overlay_capture.c index 78ae757a..0ca510e7 100644 --- a/runtime/src/overlay_capture.c +++ b/runtime/src/overlay_capture.c @@ -356,9 +356,12 @@ static void write_json_window(FILE *f, uint32_t win_lo_page, * stabilize region keys, but they are not MIPS execution barriers. * Adjacent dirty pages were already folded into this run. */ const uint32_t ram_size = 2u * 1024u * 1024u; + uint32_t guard_bytes = 0u; if (phys <= ram_size && size <= ram_size - phys && - phys + size <= ram_size - 4u) + phys + size <= ram_size - 4u) { size += 4u; + guard_bytes = 4u; + } uint32_t virt = 0x80000000u | (phys & 0x1FFFFFu); in_run = 0; @@ -395,6 +398,15 @@ static void write_json_window(FILE *f, uint32_t win_lo_page, fprintf(f, " \"schema\": \"psxrecomp overlay capture v2\",\n"); fprintf(f, " \"load_addr\": \"0x%08X\",\n", virt); fprintf(f, " \"size\": %u,\n", size); + /* How many of the trailing bytes above are the delay-slot guard, + * i.e. NOT part of the dirty-page run. The consumer must keep + * them READABLE (a branch at the run's last word emits its slot + * from here) while never discovering them as code: the word after + * a guard word is not in this region at all. Stated explicitly so + * the recompiler is TOLD rather than inferring it from `size` + * (tools/compile_overlays.py capture_guard_bytes -> + * recompiler/include/ps1_exe_parser.h exe_tag). */ + fprintf(f, " \"guard_bytes\": %u,\n", guard_bytes); fprintf(f, " \"bytes_b64\": \""); write_b64(f, ram_base + phys, size); fprintf(f, "\",\n"); diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index c36f0169..a15ab324 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -328,14 +328,92 @@ class ShardStats: # PS-EXE fake header # --------------------------------------------------------------------------- -def make_psxexe(load_addr: int, entry_pc: int, data: bytes) -> bytes: - """Wrap raw overlay bytes in a minimal PS-EXE header.""" +# Analysis-bound tag read by the recompiler (recompiler/include/ps1_exe_parser.h, +# namespace exe_tag). Offsets are into the 2048-byte PS-EXE header, whose tail +# is zero-filled in every retail image; the 8-byte magic makes a false positive +# on a genuine EXE effectively impossible. +GUARD_TAG_MAGIC_OFFSET = 0x7E0 +GUARD_TAG_COUNT_OFFSET = 0x7E8 +GUARD_TAG_MAGIC = b'PSXRGRD1' +CAPTURE_PAGE_BYTES = 4096 + + +def capture_guard_bytes(cap: dict, size: int, label: str = '') -> int: + """Trailing bytes of a captured region that are guard words ONLY. + + overlay_capture.c write_json_window appends one coherent guard instruction + past the end of a dirty-page run so that a MIPS branch at the run's final + word (...FFC) has its architectural delay slot (...000) available. Those + bytes must stay READABLE by the recompiler and must NEVER be discovered as + code: the word after a guard word is not in the image, so a control + transfer AT the guard word could not emit its own delay slot. The + recompiler is TOLD the count (see make_psxexe) rather than inferring it. + + Precedence: + 1. the capture's explicit ``guard_bytes`` field — the writer stating its + own intent, which survives any future change to capture granularity; + 2. for captures recorded BEFORE that field existed, reconstruct from the + format invariant write_json_window has always obeyed: a dirty-page run + is a whole number of 4 KiB pages, plus one 4-byte guard word whenever + the next word is still inside RAM. ``size % 4096 == 4`` is exactly + that signature and means one guard word. + + Every OTHER shape reconstructs to zero, and that is ordinary traffic rather + than an anomaly, so this stays quiet: + * ``size % 4096 == 0`` — a page run whose guard word was suppressed + because it would have left RAM, or a pre-2026-07-25 page-exact capture. + * any other residue — a STATIC extraction, not a RAM page run. Those + records carry an exact file extent (tools/aot_overlay_spike/ + extract_generic.py; they are the ones with static_dispatch_entry_pcs / + producer_ranges) and append no guard word. MEASURED: 52 of the 946 + records in the SCUS-94454 vault are this shape, with residues from 112 + to 3996 bytes and one that is not even 4-byte aligned. + + Zero is also the fail-closed answer. If some future producer ever appends a + guard word in a shape this cannot recognise, that word is analysed as code + and the emitter's mandatory-delay-slot check refuses the transfer LOUDLY at + generation time; it never silently emits a transfer without its slot. The + way to opt in is the explicit field, not a wider guess here. + """ + declared = cap.get('guard_bytes') if cap else None + if declared is not None: + try: + guard = int(declared) + except (TypeError, ValueError): + guard = -1 + if guard in (0, 4) and guard < size: + return guard + # A malformed DECLARATION is a producer bug, unlike the shapes above. + print(f' WARNING: {label or "capture"} declares guard_bytes=' + f'{declared!r} with size {size}; ignoring (treating as 0)') + return 0 + if size % CAPTURE_PAGE_BYTES == 4 and size > 4: + return 4 + return 0 + + +def make_psxexe(load_addr: int, entry_pc: int, data: bytes, *, + guard_bytes: int) -> bytes: + """Wrap raw overlay bytes in a minimal PS-EXE header. + + ``guard_bytes`` is keyword-only and has NO default on purpose: every + producer of an overlay image must state how many of its trailing bytes are + delay-slot guard words (see capture_guard_bytes). A new call site that + forgets is a TypeError, not a silently mis-analysed shard. + """ + if guard_bytes < 0 or guard_bytes % 4 or guard_bytes >= len(data): + raise ValueError( + f'invalid guard_bytes={guard_bytes} for {len(data)}-byte image') header = bytearray(2048) header[0:8] = b'PS-X EXE' struct.pack_into(' str: + toml_doc: dict, guard_bytes: int = 0) -> str: """Stable identity of every input to deterministic fragment generation.""" recipe = { - 'schema': 'psxrecomp fragment fail key v2', + # v3 adds guard_bytes: the analysis bound is an input to generation, so + # a memo recorded under the old (guard-word-as-code) analysis must not + # suppress a retry under the corrected one. + 'schema': 'psxrecomp fragment fail key v3', 'phys_addr': phys_addr, 'interior': interior & 0x1FFFFFFF, 'load_addr': load_addr, 'size': size, + 'guard_bytes': guard_bytes, 'data_sha256': hashlib.sha256(data).hexdigest(), 'producer_ranges': [list(pair) for pair in producer_ranges], 'cross_call_allow': sorted(set(cross_call_allow)), @@ -3114,12 +3196,14 @@ def append_interior_fail_memo(cache_dir: str, key: str, reason: str) -> None: def generate_interior_fragment_static(interior: int, data: bytes, load_addr: int, size: int, - phys_addr: int, args): + phys_addr: int, args, *, + guard_bytes: int): """Generate one isolated, exact-range-gated static interior shard.""" with tempfile.TemporaryDirectory() as tmp: psx = os.path.join(tmp, 'frag.psx') with open(psx, 'wb') as f: - f.write(make_psxexe(load_addr, interior, data)) + f.write(make_psxexe(load_addr, interior, data, + guard_bytes=guard_bytes)) seeds_path = os.path.join(tmp, 'seeds.txt') with open(seeds_path, 'w') as f: f.write(f'dispatch_root 0x{interior:08X}\n') @@ -3371,6 +3455,11 @@ def make_interior_fragment_job(phys_addr: int, load_addr: int, size: int, 'load_addr': load_addr, 'size': size, 'data': data, + # Trailing delay-slot guard words in `data` (readable, not + # discoverable). Stated by the capture writer where the field exists, + # reconstructed from the page-run format otherwise. + 'guard_bytes': capture_guard_bytes( + capture, size, f'region 0x{load_addr:08X}'), 'candidates': interiors | dispatch_roots | static_demands | forced, 'executed': executed, 'static_demands': static_demands, @@ -3921,7 +4010,8 @@ def compile_fragment_batch(requested_entries, data: bytes, load_addr: int, args, sub_env: dict, toml_doc: dict, producer_ranges=(), cross_call_allow=(), hosted_owners: dict | None = None, - manifest_provenance: str | None = None): + manifest_provenance: str | None = None, + *, guard_bytes: int): """Compile an ISOLATED interior-entry 'island' fragment that ENTERS at an executed orphan DISPATCH_INTERIOR PC (a host that static analysis never discovered, e.g. an FMV driver reached via a computed jump) and covers the @@ -3964,7 +4054,8 @@ def compile_fragment_batch(requested_entries, data: bytes, load_addr: int, with tempfile.TemporaryDirectory() as tmp: psx = os.path.join(tmp, 'frag.psx') with open(psx, 'wb') as f: - f.write(make_psxexe(load_addr, first_entry, data)) + f.write(make_psxexe(load_addr, first_entry, data, + guard_bytes=guard_bytes)) seeds_path = os.path.join(tmp, 'seeds.txt') with open(seeds_path, 'w') as f: for range_lo, range_hi in producer_ranges: @@ -4156,7 +4247,8 @@ def compile_fragment_batch(requested_entries, data: bytes, load_addr: int, def compile_interior_fragment(interior: int, data: bytes, load_addr: int, size: int, phys_addr: int, cache_dir: str, args, sub_env: dict, toml_doc: dict, - producer_ranges=(), cross_call_allow=()): + producer_ranges=(), cross_call_allow=(), + *, guard_bytes: int): """Compile one isolated uncertain interior entry. Executed, operator-forced, and interval-only aliases deliberately use this @@ -4166,7 +4258,8 @@ def compile_interior_fragment(interior: int, data: bytes, load_addr: int, return compile_fragment_batch( {interior}, data, load_addr, size, phys_addr, cache_dir, args, sub_env, toml_doc, producer_ranges, cross_call_allow, - manifest_provenance=ORPHAN_MANIFEST_PROVENANCE) + manifest_provenance=ORPHAN_MANIFEST_PROVENANCE, + guard_bytes=guard_bytes) def fragment_shard_key(func_ids: list, @@ -5587,13 +5680,17 @@ def main(): crc32 = binascii.crc32(data) & 0xFFFFFFFF phys_addr = (load_addr & 0x1FFFFFFF) _label = f'overlay 0x{load_addr:08X} crc {crc32:08X}' + # Trailing delay-slot guard words appended by the capture writer. + # Declared to the recompiler through the PS-EXE analysis-bound tag so + # they stay readable but are never discovered as code. + guard_bytes = capture_guard_bytes(cap, size, _label) if args.static: for captured_entry in _parse_addr_list( cap.get('dispatch_entry_pcs', [])): entry = ((captured_entry & 0x1FFFFFFF) | 0x80000000) static_requested_entries.add(entry) static_entry_sources[entry] = ( - data, load_addr, size, phys_addr) + data, load_addr, size, phys_addr, guard_bytes) # --force-interior is an explicit operator assertion that a live # dispatch entry was observed even if the retained capture lost its # classifier provenance. Static mode must honor it just like DLL @@ -5606,7 +5703,7 @@ def main(): entry = forced_phys | 0x80000000 static_requested_entries.add(entry) static_entry_sources[entry] = ( - data, load_addr, size, phys_addr) + data, load_addr, size, phys_addr, guard_bytes) # Reclassify prior F entry addresses from an identical image. Callable # entries may become current roots; other entries re-enter as dispatch @@ -5666,7 +5763,9 @@ def main(): if not args.static: dll_path = os.path.join(cache_dir, f'{phys_addr:08X}_{crc32:08X}{overlay_ext()}') - print(f'Overlay load=0x{load_addr:08X} size={size} crc32=0x{crc32:08X}') + print(f'Overlay load=0x{load_addr:08X} size={size} crc32=0x{crc32:08X}' + + (f' guard={guard_bytes}B (delay-slot only, not analysed)' + if guard_bytes else '')) if args.static: print(f' seeds: {len(seeds)} mode: static -> {static_out}') else: @@ -5721,7 +5820,8 @@ def main(): entry_pc = int(root_seeds[0].split()[-1], 16) psx_path = os.path.join(tmp, f'overlay_{load_addr:08X}.psx') with open(psx_path, 'wb') as f: - f.write(make_psxexe(load_addr, entry_pc, data)) + f.write(make_psxexe(load_addr, entry_pc, data, + guard_bytes=guard_bytes)) # Write seeds file seeds_path = os.path.join(tmp, 'seeds.txt') @@ -6081,7 +6181,8 @@ def main(): job['phys_addr'], entry, job['data'], job['load_addr'], job['size'], job['producer_ranges'], job['cross_call_allow'], - expected_abi, args.cps, toml) + expected_abi, args.cps, toml, + job['guard_bytes']) if key in fail_memo: memo_entries[entry] = fail_memo[key] if not memo_entries: @@ -6146,7 +6247,8 @@ def main(): key = _interior_fail_key( phys_addr, entry, data, load_addr, size, job['producer_ranges'], job['cross_call_allow'], - expected_abi, args.cps, toml) + expected_abi, args.cps, toml, + job['guard_bytes']) memo_action = interior_fail_memo_action( entry, job, args.force) if key in fail_memo and memo_action != 'retry': @@ -6195,7 +6297,8 @@ def main(): return compile_fragment_batch( entries, data, load_addr, size, phys_addr, cache_dir, args, frag_env, toml, job['producer_ranges'], - job['cross_call_allow']) + job['cross_call_allow'], + guard_bytes=job['guard_bytes']) def strong_batch_success(entries, frag_ids, status): strong_handled.update(entries) @@ -6231,7 +6334,8 @@ def main(): key = _interior_fail_key( phys_addr, entry, data, load_addr, size, job['producer_ranges'], job['cross_call_allow'], - expected_abi, args.cps, toml) + expected_abi, args.cps, toml, + job['guard_bytes']) optional_reject = optional_static_fragment_rejection( entry, job, status) if optional_reject: @@ -6428,7 +6532,8 @@ def main(): key = _interior_fail_key( phys_addr, entry, data, load_addr, size, job['producer_ranges'], job['cross_call_allow'], - expected_abi, args.cps, toml) + expected_abi, args.cps, toml, + job['guard_bytes']) optional_reject = optional_static_fragment_rejection( entry, job, status) if optional_reject: @@ -6472,7 +6577,8 @@ def main(): return compile_fragment_batch( specs, data, load_addr, size, phys_addr, cache_dir, args, frag_env, toml, job['producer_ranges'], - job['cross_call_allow'], hosted_owners=specs) + job['cross_call_allow'], hosted_owners=specs, + guard_bytes=job['guard_bytes']) hosted_rejected = set() hosted_published = False @@ -6624,7 +6730,8 @@ def main(): key = _interior_fail_key( phys_addr, a, data, load_addr, size, job['producer_ranges'], job['cross_call_allow'], - expected_abi, args.cps, toml) + expected_abi, args.cps, toml, + job['guard_bytes']) memo_action = interior_fail_memo_action(a, job, args.force) if key in fail_memo and memo_action != 'retry': # Deterministically doomed from these exact bytes — skip the @@ -6641,7 +6748,8 @@ def main(): frag_ids, status = compile_interior_fragment( a, data, load_addr, size, phys_addr, cache_dir, args, frag_env, toml, job['producer_ranges'], - job['cross_call_allow']) + job['cross_call_allow'], + guard_bytes=job['guard_bytes']) if frag_ids: built += 1 record_fragment_success( @@ -6825,9 +6933,10 @@ def main(): source = static_entry_sources.get(entry) if source is None: continue - data, load_addr, size, phys_addr = source + data, load_addr, size, phys_addr, guard_bytes = source part = generate_interior_fragment_static( - entry, data, load_addr, size, phys_addr, args) + entry, data, load_addr, size, phys_addr, args, + guard_bytes=guard_bytes) if part is None: continue static_parts.append(part) diff --git a/tools/tests/test_overlay_guard_bytes_declaration.py b/tools/tests/test_overlay_guard_bytes_declaration.py new file mode 100644 index 00000000..f6399f29 --- /dev/null +++ b/tools/tests/test_overlay_guard_bytes_declaration.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""The overlay producer must DECLARE its trailing delay-slot guard words. + +overlay_capture.c appends one coherent guard instruction past the end of a +dirty-page run so a MIPS branch at the run's final word (...FFC) has its +architectural delay slot (...000). The recompiler must be TOLD how many +trailing bytes that is, because the guard word is a legal delay-slot source +and an illegal block leader -- the word after it does not exist in the image. +Inferring the count inside the recompiler from `size % 4096 == 4` would fit +every capture written by the current format and would silently shift +underneath any future change to capture granularity. + +This pins the whole producer chain (bead beads-eio.3.100): + + overlay_capture.c writes "guard_bytes" (writer states intent) + -> compile_overlays.capture_guard_bytes (prefer it; reconstruct + for pre-field captures) + -> compile_overlays.make_psxexe (tagged PS-EXE header) + -> ps1_exe_parser.h exe_tag (recompiler reads it) +""" + +import struct +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools")) + +import compile_overlays # noqa: E402 + + +class GuardByteDerivationTests(unittest.TestCase): + def test_explicit_field_wins(self): + self.assertEqual( + compile_overlays.capture_guard_bytes({"guard_bytes": 4}, 0x2004), 4) + # A page-multiple+4 capture that explicitly says it has NO guard word + # must be believed: the writer knows, the size does not. + self.assertEqual( + compile_overlays.capture_guard_bytes({"guard_bytes": 0}, 0x2004), 0) + + def test_legacy_capture_reconstructs_from_the_page_run_format(self): + # Current format: whole 4 KiB pages plus one guard word. + self.assertEqual(compile_overlays.capture_guard_bytes({}, 0x2004), 4) + self.assertEqual(compile_overlays.capture_guard_bytes({}, 0x1004), 4) + # Pre-2026-07-25 captures are page-exact and have no guard word. + self.assertEqual(compile_overlays.capture_guard_bytes({}, 0x2000), 0) + self.assertEqual(compile_overlays.capture_guard_bytes({}, 0x1000), 0) + + def test_unknown_residue_falls_back_to_zero(self): + # Not this tool's format. Zero is the fail-closed answer: the emitter's + # mandatory-delay-slot check still refuses a transfer it cannot finish. + self.assertEqual(compile_overlays.capture_guard_bytes({}, 0x2008), 0) + self.assertEqual(compile_overlays.capture_guard_bytes({}, 100), 0) + + def test_nonsense_declaration_is_rejected(self): + self.assertEqual( + compile_overlays.capture_guard_bytes({"guard_bytes": 7}, 0x2004), 0) + self.assertEqual( + compile_overlays.capture_guard_bytes( + {"guard_bytes": "four"}, 0x2004), 0) + # A declaration that would consume the whole image is not a trailer. + self.assertEqual( + compile_overlays.capture_guard_bytes({"guard_bytes": 4}, 4), 0) + + +class PsxExeTagTests(unittest.TestCase): + PAYLOAD = b"\x00" * 0x2000 + struct.pack("