diff --git a/docs/config_schema.md b/docs/config_schema.md index c7eab4c1..bd299edb 100644 --- a/docs/config_schema.md +++ b/docs/config_schema.md @@ -71,7 +71,7 @@ respective files. | `exe` | game | path to PS-X EXE file, relative to project root | | `load_address` | both | hex string, virtual address of first byte (`"0xBFC00000"` BIOS, `"0x80010000"` typical game) | | `entry_pc` | both | hex string, first PC to execute | -| `text_size` | both | hex string, size in bytes of code segment (used by audits to bound region walks) | +| `text_size` | both | hex string, size in bytes of the static region. For games this also bounds main-EXE analysis and establishes the overlay floor. A smaller-than-header bound must be verified non-code and 4 KiB aligned. | | `stack_base` | game | hex string, initial `$sp` value for the game | | `disc` | game (single-disc) | path to .cue, relative to project root | | `discs` | game (multi-disc) | array of .cue paths; `disc` is sugar for `discs = [disc]` | @@ -85,6 +85,7 @@ seeds = "seeds/ghidra_funcs.txt" # game (note: game se bios_thunks = "seeds/tomba_bios_thunks.txt" # game-only out_dir = "generated" # both strict = true # both — currently always true +discovery = "whole-image" # game-only: "whole-image" or "reachable" out_stem = "SCPH1001" # optional; overrides the auto-derived stem ``` @@ -93,6 +94,20 @@ Output filenames: `/_full.c` and from the `rom`/`exe` file basename with the trailing `.BIN` or `.EXE` stripped (`Path.stem` is NOT used because it mishandles `SCUS_942.36`). +Game `discovery` defaults to `"whole-image"`, preserving the existing sweep and +pointer-table heuristics. Opt-in `"reachable"` starts at the executable entry +and evidence-backed seed roots, then follows callable direct `jal` targets. It +does not sweep arbitrary bytes for prologues or return-shaped words. Unresolved +`jalr`/indirect targets and unseen callbacks fail closed to runtime +interpretation; add an evidence-backed seed (or a `dispatch_root` seed for a +proven nonstandard boundary) when they should be compiled. + +When `game.text_size` is smaller than the PS-X EXE header size, recompilation +uses it as a static-analysis bound. The value must be nonzero, instruction- and +4 KiB-aligned, retain `entry_pc`, and not extend past PS1 RAM. The original EXE +is still loaded from the user's disc. A generated config's canonical final-page +reservation may be slightly larger than the header; it does not widen analysis. + ## Runtime block Consumed by the cmake macro `psxrecomp_v4_add_runtime_target` (eventually) diff --git a/docs/internal/upstream/douglas-mm8-reachable-discovery.md b/docs/internal/upstream/douglas-mm8-reachable-discovery.md new file mode 100644 index 00000000..c59b7028 --- /dev/null +++ b/docs/internal/upstream/douglas-mm8-reachable-discovery.md @@ -0,0 +1,28 @@ +# Douglas MM8: reachable main-EXE discovery + +Evaluated and adapted on 2026-07-13. + +## Provenance + +The opt-in reachable main-executable discovery and `game.text_size` analysis +bound were rebuilt from douglasjv's +[Mega Man 8 PSXrecomp project](https://github.com/douglasjv/mm8), specifically +commit +[`8415f3a95458c41c7f48fbe36caaf0ee82730720`](https://github.com/douglasjv/mm8/commit/8415f3a95458c41c7f48fbe36caaf0ee82730720) +(`Add reachable main executable discovery`). The implementation was adapted to +current master rather than copied verbatim: bounds are validated before +mutation, canonical page-rounded existing configs remain compatible, the EXE +entry is an explicit root, and main-image codegen retains the newer exact +static-range dispatch checks from `b39387e`. + +Reachable discovery is fail-closed. It follows callable direct `jal` targets +from verified roots; an unresolved `jalr`, unseen callback, rejected seed, or +target outside the verified bound does not create a compiled entry. Such PCs +remain eligible for the runtime interpreter and overlay capture paths. + +## Deliberately excluded + +No MM8 addresses, identifiers, `bg2d` or other widescreen configuration, +game-specific seeds/configuration, executable patches, generated game code, or +framework pin updates were imported. All other MM8 repository and dependency +changes are outside this focused branch. diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index c146d7e8..880f3400 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -206,3 +206,16 @@ target_include_directories(l2_structural_test PRIVATE lib/fmt/include ) target_link_libraries(l2_structural_test PRIVATE fmt rabbitizer) + +# Reachable main-EXE discovery test: parser-bound validation plus exact-entry +# analysis behavior for direct, indirect, seeded, and out-of-bound targets. +add_executable(reachable_discovery_test + tests/reachable_discovery_test.cpp + src/function_analysis.cpp + src/ps1_exe_parser.cpp +) +target_include_directories(reachable_discovery_test PRIVATE + include + lib/fmt/include +) +target_link_libraries(reachable_discovery_test PRIVATE fmt) diff --git a/recompiler/include/function_analysis.h b/recompiler/include/function_analysis.h index 3f9ddb11..bdb87486 100644 --- a/recompiler/include/function_analysis.h +++ b/recompiler/include/function_analysis.h @@ -48,9 +48,10 @@ public: // Scan entire executable for function boundaries FunctionAnalysisResult analyze(); - // Analyze only explicit entry points and direct JAL targets reachable from - // them. This is for runtime-loaded overlays where scanning the whole blob - // treats data and basic-block targets as standalone functions. + // Analyze only explicit entry points and callable direct-JAL targets + // reachable from them. Used by runtime-loaded overlays and opt-in main-EXE + // reachable discovery. Unresolved jalr/indirect targets do not mint + // functions; evidence-backed entries must be supplied explicitly. FunctionAnalysisResult analyze_exact_entries(const std::vector& entries); // Add a forced entry point address that is treated as a function start diff --git a/recompiler/include/ps1_exe_parser.h b/recompiler/include/ps1_exe_parser.h index 75afaa23..ae941104 100644 --- a/recompiler/include/ps1_exe_parser.h +++ b/recompiler/include/ps1_exe_parser.h @@ -106,4 +106,13 @@ public: static bool validate_header(const PS1ExeHeader& header, std::string& error_msg); }; +// Restrict a parsed main executable to a title-verified static-analysis size. +// The original disc image is not modified; only this in-memory analysis view +// is shortened. Returns false without mutation when the configured bound is +// invalid. A canonical page-rounded size above the header size is accepted as +// an existing runtime reservation and leaves the analysis view unchanged. +bool apply_static_analysis_bound(PS1Executable& exe, + uint32_t configured_size, + std::string& error_msg); + } // namespace PSXRecomp diff --git a/recompiler/src/config_loader.cpp b/recompiler/src/config_loader.cpp index cd2387b5..725dcea1 100644 --- a/recompiler/src/config_loader.cpp +++ b/recompiler/src/config_loader.cpp @@ -702,6 +702,16 @@ GameConfig load_game_config(const fs::path& config_path_in) { ? toml::find(recomp, "strict") : true; + const std::string discovery = + recomp.contains("discovery") + ? toml::find(recomp, "discovery") + : std::string{"whole-image"}; + if (discovery != "whole-image" && discovery != "reachable") { + throw std::runtime_error(fmt::format( + "{}: [recompiler].discovery must be 'whole-image' or 'reachable'", + config_path.string())); + } + std::string out_stem; if (recomp.contains("out_stem")) { out_stem = toml::find(recomp, "out_stem"); @@ -939,6 +949,7 @@ GameConfig load_game_config(const fs::path& config_path_in) { /*bios_thunks_path*/ bios_thunks_path, /*out_dir*/ out_dir, /*strict*/ strict, + /*discovery*/ discovery, /*out_stem*/ out_stem, /*runtime*/ parse_runtime_block(cfg, root), /*ws_sprite_tag_funcs*/ ws_sprite_tag_funcs, diff --git a/recompiler/src/config_loader.h b/recompiler/src/config_loader.h index 9de5fe53..f0632875 100644 --- a/recompiler/src/config_loader.h +++ b/recompiler/src/config_loader.h @@ -386,6 +386,7 @@ struct GameConfig { std::filesystem::path bios_thunks_path; // optional; empty if not set std::filesystem::path out_dir; bool strict; + std::string discovery; // "whole-image" (default) or "reachable" std::string out_stem; // derived if not explicit // [runtime] block (optional) diff --git a/recompiler/src/function_analysis.cpp b/recompiler/src/function_analysis.cpp index 6efa827c..33d7584f 100644 --- a/recompiler/src/function_analysis.cpp +++ b/recompiler/src/function_analysis.cpp @@ -549,7 +549,7 @@ FunctionAnalysisResult FunctionAnalyzer::analyze_exact_entries(const std::vector result.call_discovered_count = 0; result.state_continuation_count = 0; - fmt::print("\n=== Exact Overlay Function Analysis ===\n\n"); + fmt::print("\n=== Exact-Entry Function Analysis ===\n\n"); auto in_exe = [&](uint32_t addr) { return addr >= exe_.header.load_address && addr < exe_.end_address() && (addr & 3u) == 0; diff --git a/recompiler/src/main_psx.cpp b/recompiler/src/main_psx.cpp index c7f5b811..d529242b 100644 --- a/recompiler/src/main_psx.cpp +++ b/recompiler/src/main_psx.cpp @@ -119,6 +119,7 @@ int main(int argc, char** argv) { const char* extra_funcs_path = nullptr; bool inspect_mode = false; bool overlay_mode = false; + bool reachable_discovery = false; std::set ws_tag_funcs; // [widescreen] sprite_tag_funcs std::set ws_cull_bias, ws_cull_range, ws_cull_a1; // [widescreen.cull] std::set ws_cull_screen_x; // [widescreen.cull] screen_x_sites @@ -138,10 +139,13 @@ int main(int argc, char** argv) { ws_bg2d_cap_site = 0, ws_bg2d_init_func = 0; // [widescreen.bg2d] std::filesystem::path out_dir = "generated"; + uint32_t configured_text_size = 0; if (!config_path.empty()) { const auto cfg = PSXRecompV4::load_game_config(config_path); exe_path = cfg.exe_path; + configured_text_size = cfg.text_size; + reachable_discovery = cfg.discovery == "reachable"; extra_funcs_storage = cfg.seeds_path.string(); extra_funcs_path = extra_funcs_storage.c_str(); out_dir = cfg.out_dir; @@ -253,6 +257,24 @@ int main(int argc, char** argv) { return 1; } + // [game].text_size is the title-owned static-analysis bound. It may trim a + // verified data/padding tail, but it may never widen the parsed payload or + // exclude the executable entry. Runtime disc loading remains unchanged. + if (!config_path.empty()) { + const uint32_t parsed_size = exe->header.file_size; + std::string bound_error; + if (!PSXRecomp::apply_static_analysis_bound( + *exe, configured_text_size, bound_error)) { + fmt::print(stderr, "Invalid static analysis bound: {}\n", bound_error); + return 1; + } + if (exe->header.file_size < parsed_size) { + fmt::print("Static analysis bound: 0x{:X} bytes from game.text_size " + "(PS-X EXE header: 0x{:X})\n", + exe->header.file_size, parsed_size); + } + } + fmt::print("✓ Successfully parsed PS1-EXE!\n\n"); // Print header information @@ -427,8 +449,8 @@ int main(int argc, char** argv) { PSXRecomp::FunctionAnalysisResult analysis_result; - if (overlay_mode) { - // ── Overlay exact mode: partition seeds into walk roots and interior + if (overlay_mode || reachable_discovery) { + // ── Exact-entry mode: partition seeds into walk roots and interior // alias candidates. `interior`-marked seeds (classifier-proven // dispatch targets without a callable boundary) are NEVER walk roots — // a root inside a host function hard-caps (truncates) it, the @@ -453,7 +475,11 @@ int main(int argc, char** argv) { std::set roots; std::set interior; for (uint32_t a : exact_entries) { - if (trusted_root_seeds.count(a)) { + if (reachable_discovery && a == exe->header.initial_pc) { + // The PS-X EXE header is direct execution evidence for the + // main entry even when it uses a nonstandard prologue. + roots.insert(a); + } else if (trusted_root_seeds.count(a)) { /* Classifier-proven dispatch root (install-slot class): the * static code tail-dispatches into this PC, so it is a real * execution root despite having no callable boundary. */ @@ -546,7 +572,7 @@ int main(int argc, char** argv) { "(range-ownership completeness)\n", jal_alias_added); materialize_alias_groups(analysis_result, alias_entries); - fmt::print("Overlay alias entries emitted: {}\n\n", alias_entries.size()); + fmt::print("Exact-entry alias entries emitted: {}\n\n", alias_entries.size()); } else { // ── Iterative discovery: seed classification + static data-table scan ── // diff --git a/recompiler/src/ps1_exe_parser.cpp b/recompiler/src/ps1_exe_parser.cpp index 65b23144..322c550d 100644 --- a/recompiler/src/ps1_exe_parser.cpp +++ b/recompiler/src/ps1_exe_parser.cpp @@ -5,6 +5,76 @@ namespace PSXRecomp { +bool apply_static_analysis_bound(PS1Executable& exe, + uint32_t configured_size, + std::string& error_msg) { + const uint32_t image_size = exe.header.file_size; + const uint64_t configured_end = + static_cast(exe.header.load_address) + configured_size; + + if (configured_size == 0) { + error_msg = "game.text_size must be nonzero"; + return false; + } + if ((configured_size & 3u) != 0) { + error_msg = fmt::format( + "game.text_size 0x{:X} is not instruction-aligned", configured_size); + return false; + } + if (configured_end > 0x80200000ull) { + error_msg = fmt::format( + "game.text_size 0x{:X} extends past PS1 RAM", configured_size); + return false; + } + + const uint32_t effective_size = + configured_size < image_size ? configured_size : image_size; + const uint64_t effective_end = + static_cast(exe.header.load_address) + effective_size; + if (exe.header.initial_pc < exe.header.load_address || + static_cast(exe.header.initial_pc) >= effective_end) { + error_msg = fmt::format( + "game.text_size 0x{:X} excludes entry_pc 0x{:08X}", + configured_size, exe.header.initial_pc); + return false; + } + + if (configured_size > image_size) { + // Existing config generation reserves the remainder of the final 4 KiB + // page for the runtime overlay floor. That value is not evidence that + // bytes beyond the EXE payload exist, so it must not widen analysis. + const uint64_t rounded = + (static_cast(image_size) + 0xFFFull) & ~0xFFFull; + if (configured_size != rounded) { + error_msg = fmt::format( + "game.text_size 0x{:X} exceeds PS-X EXE size 0x{:X} " + "and is not its canonical page-rounded reservation", + configured_size, image_size); + return false; + } + return true; + } + + if (configured_size == image_size) return true; + + if ((configured_size & 0xFFFu) != 0) { + error_msg = fmt::format( + "truncated game.text_size 0x{:X} must be 4 KiB page-aligned", + configured_size); + return false; + } + if (exe.code_data.size() != image_size) { + error_msg = fmt::format( + "cannot apply analysis bound: EXE payload is {} bytes, header is {}", + exe.code_data.size(), image_size); + return false; + } + + exe.header.file_size = configured_size; + exe.code_data.resize(configured_size); + return true; +} + // Validate PS1-EXE header bool PS1ExeParser::validate_header(const PS1ExeHeader& header, std::string& error_msg) { // Check magic identifier diff --git a/recompiler/tests/reachable_discovery_test.cpp b/recompiler/tests/reachable_discovery_test.cpp new file mode 100644 index 00000000..64429fd2 --- /dev/null +++ b/recompiler/tests/reachable_discovery_test.cpp @@ -0,0 +1,139 @@ +#include "function_analysis.h" +#include "ps1_exe_parser.h" + +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr uint32_t kLoad = 0x80010000u; +int failures = 0; + +#define CHECK(cond, message) do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL: %s\n", (message)); \ + failures++; \ + } \ +} while (0) + +void put32(std::vector& data, size_t offset, uint32_t value) { + data[offset + 0] = static_cast(value); + data[offset + 1] = static_cast(value >> 8); + data[offset + 2] = static_cast(value >> 16); + data[offset + 3] = static_cast(value >> 24); +} + +uint32_t jal(uint32_t target) { + return 0x0C000000u | ((target >> 2) & 0x03FFFFFFu); +} + +std::vector make_exe_buffer(uint32_t image_size, + uint32_t entry = kLoad) { + std::vector data(2048u + image_size, 0); + std::memcpy(data.data(), "PS-X EXE", 8); + put32(data, 0x10, entry); + put32(data, 0x18, kLoad); + put32(data, 0x1C, image_size); + return data; +} + +PSXRecomp::PS1Executable parse(std::vector data) { + std::string error; + auto exe = PSXRecomp::PS1ExeParser::parse_buffer(data, error); + CHECK(exe.has_value(), error.c_str()); + return exe.value(); +} + +std::set starts(const PSXRecomp::FunctionAnalysisResult& result) { + std::set out; + for (const auto& function : result.functions) out.insert(function.start_addr); + return out; +} + +} // namespace + +int main() { + auto image = make_exe_buffer(0x2000); + const size_t text = 2048; + + // Root: direct call to a callable target, unresolved jalr, direct call + // beyond the eventual bound, then return. + put32(image, text + 0x00, jal(kLoad + 0x100)); + put32(image, text + 0x04, 0x00000000u); + put32(image, text + 0x08, 0x0320F809u); // jalr $ra,$t9 (unresolved) + put32(image, text + 0x0C, 0x00000000u); + put32(image, text + 0x10, jal(kLoad + 0x1100)); + put32(image, text + 0x14, 0x00000000u); + put32(image, text + 0x18, 0x03E00008u); + put32(image, text + 0x1C, 0x00000000u); + + // Callable direct target. + put32(image, text + 0x100, 0x27BDFFF0u); + put32(image, text + 0x104, 0x03E00008u); + put32(image, text + 0x108, 0x27BD0010u); + + // Valid-looking callback not referenced by a direct call. + put32(image, text + 0x200, 0x27BDFFF0u); + put32(image, text + 0x204, 0x03E00008u); + put32(image, text + 0x208, 0x27BD0010u); + + // Callable target outside the verified static bound. + put32(image, text + 0x1100, 0x27BDFFF0u); + put32(image, text + 0x1104, 0x03E00008u); + put32(image, text + 0x1108, 0x27BD0010u); + + auto exe = parse(image); + std::string error; + CHECK(PSXRecomp::apply_static_analysis_bound(exe, 0x1000, error), + "valid page-aligned analysis bound is accepted"); + CHECK(exe.code_size() == 0x1000 && exe.code_data.size() == 0x1000, + "accepted bound truncates header and analysis payload together"); + + PSXRecomp::FunctionAnalyzer analyzer(exe); + const auto reachable = starts(analyzer.analyze_exact_entries({kLoad})); + CHECK(reachable == std::set({kLoad, kLoad + 0x100}), + "reachable analysis follows direct callable JAL only"); + CHECK(!reachable.count(kLoad + 0x200), + "unseen indirect callback fails closed"); + CHECK(!reachable.count(kLoad + 0x1100), + "direct target beyond verified bound fails closed"); + + PSXRecomp::FunctionAnalyzer seeded_analyzer(exe); + const auto seeded = starts( + seeded_analyzer.analyze_exact_entries({kLoad, kLoad + 0x200})); + CHECK(seeded.count(kLoad + 0x200), + "evidence-backed explicit callback seed is compiled"); + + auto bad = parse(image); + const auto original_size = bad.code_data.size(); + CHECK(!PSXRecomp::apply_static_analysis_bound(bad, 0x1800, error), + "non-page-aligned truncation is rejected"); + CHECK(bad.code_size() == 0x2000 && bad.code_data.size() == original_size, + "invalid bound leaves parsed image unchanged"); + CHECK(!PSXRecomp::apply_static_analysis_bound(bad, 0x3000, error), + "oversized noncanonical bound is rejected"); + CHECK(!PSXRecomp::apply_static_analysis_bound(bad, 0, error), + "zero bound is rejected"); + + auto excludes_entry = parse(image); + excludes_entry.header.initial_pc = kLoad + 0x1000; + CHECK(!PSXRecomp::apply_static_analysis_bound(excludes_entry, 0x1000, error), + "bound excluding entry point is rejected"); + + auto rounded = parse(make_exe_buffer(0x1800)); + CHECK(PSXRecomp::apply_static_analysis_bound(rounded, 0x2000, error), + "canonical generated final-page reservation remains compatible"); + CHECK(rounded.code_size() == 0x1800, + "page reservation does not widen static analysis payload"); + + if (failures) { + std::fprintf(stderr, "FAILED (%d)\n", failures); + return 1; + } + std::puts("ALL PASS"); + return 0; +} diff --git a/recompiler/tests/test_reachable_discovery_codegen.py b/recompiler/tests/test_reachable_discovery_codegen.py new file mode 100644 index 00000000..e446a7cc --- /dev/null +++ b/recompiler/tests/test_reachable_discovery_codegen.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""End-to-end config and codegen checks for reachable main-EXE discovery. + +Usage: python test_reachable_discovery_codegen.py \ + [--recompiler ] +""" +import argparse +import os +import struct +import subprocess +import sys +import tempfile + +LOAD = 0x80010000 + + +def jal(target): + return 0x0C000000 | ((target >> 2) & 0x03FFFFFF) + + +def put32(data, offset, value): + struct.pack_into("