From 5c876bd385dd14429d950615e45de4688d54a979 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Tue, 25 Aug 2026 00:41:00 -0700 Subject: [PATCH] recompiler: add standalone static analyzer --- docs/FUNCTION_DISCOVERY.md | 263 +++++++ recompiler/CMakeLists.txt | 80 +- recompiler/include/analysis_db.h | 218 ++++++ recompiler/include/widescreen_scan.h | 88 +++ recompiler/src/analysis_db.cpp | 1050 ++++++++++++++++++++++++++ recompiler/src/analysis_export.cpp | 564 ++++++++++++++ recompiler/src/analysis_export.h | 65 ++ recompiler/src/bios_call_names.cpp | 117 +++ recompiler/src/main_analyze.cpp | 344 +++++++++ recompiler/src/widescreen_scan.cpp | 521 +++++++++++++ 10 files changed, 3298 insertions(+), 12 deletions(-) create mode 100644 docs/FUNCTION_DISCOVERY.md create mode 100644 recompiler/include/analysis_db.h create mode 100644 recompiler/include/widescreen_scan.h create mode 100644 recompiler/src/analysis_db.cpp create mode 100644 recompiler/src/analysis_export.cpp create mode 100644 recompiler/src/analysis_export.h create mode 100644 recompiler/src/bios_call_names.cpp create mode 100644 recompiler/src/main_analyze.cpp create mode 100644 recompiler/src/widescreen_scan.cpp diff --git a/docs/FUNCTION_DISCOVERY.md b/docs/FUNCTION_DISCOVERY.md new file mode 100644 index 00000000..f54f42a1 --- /dev/null +++ b/docs/FUNCTION_DISCOVERY.md @@ -0,0 +1,263 @@ +# Function discovery — `psxrecomp-analyze` + +A developer-facing static analysis tool for PS-X EXE images. It exists because +the recompiler already derives everything a modder or decompiler wants — +function boundaries, the call graph, jump tables, argument usage — and then +throws it away after emitting C. This turns that knowledge into an artifact. + +It is **not** part of the recompilation path and does not link `runtime/`. + +``` +recompiler/include/analysis_db.h schema + the two rules +recompiler/src/analysis_db.cpp the analysis passes +recompiler/src/analysis_export.cpp JSON / symbols.toml / Ghidra / TSV writers +recompiler/src/bios_call_names.cpp A0/B0/C0 kernel call names +recompiler/src/main_analyze.cpp the CLI +``` + +--- + +## The two rules + +**1. Static only.** No trace import, no overlay capture set, no executed-PC +feedback, no runtime translation unit in the link. Every function, edge, and +jump table reported was proven from the executable image alone. What static +analysis cannot reach is printed as a *gap*: + +``` +Static coverage gap (NOT filled from any runtime source): + 132 indirect transfer(s) have no proven target set. + 255504 bytes (40.1%) of the image belong to no code function. +``` + +The guarantee is enforced structurally: `recompiler/CMakeLists.txt` lists the +tool's sources explicitly and none of them come from `runtime/`. + +**2. Tolerant, not fabricating.** CLAUDE.md §0 says an unsupported opcode is a +hard failure — correct for the recompiler, wrong for a tool someone points at a +weird region. The analyzer degrades instead: a region it cannot fully decode is +marked `partial` with the offending PCs listed. That is not a stub. It emits no +code and asserts no semantics it did not prove. Reporting *incomplete* is +allowed here; reporting *fabricated* is not. + +--- + +## Usage + +```bash +psxrecomp-analyze [options] + +# typical: analyse, write the JSON bundle, keep a diffable TSV +psxrecomp-analyze disc/SLUS_005.62 --symbols symbols.toml \ + --out analysis --emit-tsv analysis/functions.tsv +``` + +Key options: `--exact` (reachability-only partition), `--seeds `, +`--emit-symbols` (merge into `symbols.toml`), `--emit-ghidra`, +`--emit-symbol-addrs`, `--disasm --at `, `--diff `. + +### Artifacts + +| File | Contents | +|---|---| +| `analysis.json` | image metadata, stats, one record per function | +| `edges.json` | call edges: direct, jump-table, address-taken, resolved indirect | +| `indirect.json` | every indirect transfer, resolved or not, with context | +| `refs.json` | statically resolved memory references (largest; `--no-refs` skips) | +| `functions.tsv` | one line per function — grep/awk friendly, and the `--diff` input | +| `symbol_addrs.txt` | decomp-toolchain symbol map | +| `psxrecomp_import.py` | Ghidra script: creates functions, applies names, bookmarks unresolved sites | + +The JSON files are the **stable public interface** for future front ends or +project-level CLI wrappers. Consumers should read those artifacts, never the +analyzer's in-process types. + +--- + +## What it recovers + +**Function boundaries** come from `FunctionAnalyzer` (the same code the emitters +use), then get scored: + +| Confidence | Meaning | +|---|---| +| `verified` | prologue + epilogue + `jr $ra`, fully decoded, called from reachable code | +| `high` | `jr $ra` exit, reachable from the entry point | +| `medium` | prologue-shaped, or address-taken with no proven call path | +| `low` | boundary inferred; unreachable, no prologue, or partial decode | +| `data` | classified as data masquerading as code | + +Every row also carries `confidence_reason`, so a claim can be argued with. + +**Jump tables.** `resolve_exact_bounded_jump_table()` in `function_analysis.cpp` +is tried first and is authoritative when it fires, but it accepts exactly one +instruction schedule because a wrong table there emits wrong *code*. That is the +wrong trade for a discovery tool: on Masters of Teras Kasi it resolved **0 of +210** indirect sites, because the common schedule puts the `sll` in the branch's +delay slot and materializes the table base between the `sll` and the `addu`. + +`recover_jump_table()` walks the actual backward slice instead of matching a +fixed window, and labels how it terminated: + +* `jump_table:proven` — the strict resolver accepted it. +* `jump_table:guarded` — an `sltiu` bound on the index register was found, so + the case count is proven. +* `jump_table:scanned` — no bound found; the table was walked until a word + stopped looking like a code address. **A hypothesis**, and labelled as one. + +**Indirect calls** through a statically-known pointer — materialized inline, or +loaded from a fixed address holding a known function start — are resolved as +`call_ptr:immediate` / `call_ptr:static_slot`. Pointers from struct fields or +computed addresses stay unresolved, correctly. + +**Kernel dispatch thunks.** PSY-Q titles call the BIOS through three-instruction +stubs (`addiu $t2,$zero,0xA0; jr $t2; addiu $t1,$zero,`). The generic rules +score these `low` — no prologue, no `jr $ra` — yet they are among the most-called +functions in any image. They are detected by exact pattern and named from the +A0/B0/C0 tables (`bios_call_names.cpp`, sourced from psx-spx), so +`func_800749DC` with 41 callers reads as `bios_A0_3F_printf` instead. + +**Signatures** are inferred by a read-before-write scan: arguments in `$a0–$a3`, +return in `$v0/$v1`, callee-saved set from the prologue's stores, leaf/non-leaf, +GTE/MMIO/syscall use, frame size. This is a **heuristic** — a loop that reads a +register before the backward edge writes it over-reports an argument — so each +record carries `sig.confident`, false whenever the scan hit such a construct. + +### Measured on three titles + +| Title | Functions | Tables recovered | Indirect left | Coverage | +|---|---|---|---|---| +| Masters of Teras Kasi (`SLUS_005.62`) | 1331 | 46 (804 targets) | 132 | 59.9% | +| Twisted Metal 4 (`SCUS_945.60`) | 1099 | 11 (236 targets) | 185 | 76.1% | +| Klonoa (`SLUS_005.85`) | 230 | 2 (126 targets) | 74 | 71.5% | + +Klonoa's boot EXE is 45 KB because the game streams in as overlays — a good +illustration of the limit below. + +--- + +## Where static-only costs coverage + +Two classes remain, and the tool says so rather than papering over them: + +* **`jalr` through a runtime pointer** — a vtable slot or callback filled in at + init. No static answer exists in general. +* **Overlay bodies** — for an overlay-heavy title, most of the code is not in + the boot EXE at all. Closing this statically means parsing the disc + filesystem, locating overlay blobs as files, and recovering load bases by + analysing the loader's own `CdRead`/`memcpy` calls. Tractable, not done here. + +`tools/overlay_xref.py` mines the runtime's capture set for exactly this, and it +works — but it is dynamic evidence. The rule this tool follows is not "runtime +never touched"; it is **you always know which claims are static**. If capture +import is ever added here, it must arrive as a separately-labelled source, never +silently merged. + +--- + +## Naming loop + +`symbols.toml` (see [SYMBOLS.md](SYMBOLS.md)) is the persistence layer. Both +writers are **append/edit-only and never regenerate the file**: it holds more +than functions — Ape Escape has 11 `[[object]]` entries and a top-level `game` +key, Crash Team Racing has 33 `[[site]]` entries, all hand-researched — and an +earlier version that rebuilt the file from parsed `[[func]]` entries silently +deleted every one of them. + +```bash +psxrecomp-analyze --symbols symbols.toml --emit-symbols \ + --min-confidence verified +``` + +New functions land as `status = "guessed"` with a note recording why the +analyzer trusted them. A name a human typed always outranks anything inferred, +and an entry this run did not rediscover is carried through rather than dropped. + +## Relationship to the emitters + +`psxrecomp-game` answers *"what C do I emit for this?"* and must be conservative, +because a wrong answer is a broken build. `psxrecomp-analyze` answers *"what is +this?"* and can afford a labelled hypothesis, because a wrong answer is a wrong +report that a human will notice. They share the boundary detection and the +strict table resolver; they differ in what they are allowed to guess, and the +labels exist so a consumer can tell which is which. + +--- + +## Widescreen site scanning (`--scan-widescreen`) + +Widescreen bring-up on a new title means finding a handful of exact instruction +addresses for `[widescreen.cull]`. Tomba's `game.toml` carries four bias/range +pairs and three `a1` nops, each located by hand from disassembly or by mining +the runtime's capture set with `tools/overlay_xref.py`. Every one of them has a +crisp syntactic signature, so for a main-EXE title that search is static. + +```bash +psxrecomp-analyze --scan-widescreen --emit-ws-sites ws.toml +``` + +### What it finds + +**The per-game screen extents.** These are not fixed — Tomba tests `0x140/0x141` +on a 320 display, Ape Escape `0x181` on 368, Wipeout 3 `0x200/0x240` on 512. +Frequency alone picks the wrong values (Wipeout 3's real immediates appear in +only three funnels, drowned by ordinary bounds checks in 83 GTE functions), so +the scan scores **co-occurrence**: the screen-extent signature is a width +compare and a height compare in the *same* function. Pairs are weighted toward +GTE code and toward the console's actual display modes, then completed with the +inclusive-bound sibling when the image contains a compare against it. That last +step is load-bearing — Tomba's pairing keys on `sltiu_imm - 2*bias == W`, and +`449 - 128` is `0x141`, so without it three of its four pairs vanish. + +When neither chosen set lands on a real console mode, the report and the emitted +TOML say so rather than presenting a guess with a confident face. + +**`auto_screen_x` coverage.** Functions carrying the screen-extent signature are +listed using `ws_cull_detect.h` — the same detector the recompiler and the +runtime interpreter use, so the verdict cannot drift. Knowing that 22 of Tomba's +functions are already handled without any per-address list is the most useful +thing to learn before hand-listing anything. + +**bias/range pairs**, the masked-u16 X window: + +``` +80022E78 addiu $v0, $v0, 64 <- bias_sites (+halfwidth) +80022E7C andi $v0, $v0, 0xFFFF +80022E80 sltiu $v0, $v0, 449 <- range_sites (W + 2*halfwidth) +80022E84 beq $v0, $zero, reject +``` + +The arithmetic tie between the two immediates is what makes this safe to +propose. Without it, any neighbouring `addiu`/`sltiu` would qualify. + +**`a1` margin sites** — a repurposable `nop` before a caller-supplied margin +joins an X term. These classifiers are *callees* of the render funnel, not +funnels themselves (Tomba's uses no GTE op and has no static caller at all), so +the signature is local: the function takes `$a1`, folds it in with `addu`, and +has a nop just before that is not already a branch delay slot. + +**`screen_x` sites** — width compares in functions `auto_screen_x` cannot reach, +which are exactly the addresses that still need listing by hand. + +### Measured against hand-found ground truth + +Tomba's `game.toml` sites were found by hand over the course of the widescreen +work. Scanning `SCUS_942.36` with everything auto-discovered: + +| key | recall | candidates proposed | +|---|---|---| +| `bias_sites` | **4/4** | 4 (no false positives) | +| `range_sites` | **4/4** | 4 (no false positives) | +| `a1_sites` | **3/3** | 17 | + +Independently, the scan recovers Ape Escape's documented `0x181` and both of +Wipeout 3's configured `0x200`/`0x240`. + +### Safety + +Every proposal already satisfies the instruction form `code_generator.cpp` +requires — `addi`/`addiu` for bias, `sltiu` for range and screen_x, a bare `nop` +for a1 — because the emitter hard-errors on a mismatch in main-EXE mode. A +suggestion that would break the build is not a suggestion. What the scanner +cannot tell you is whether widening a given site *helps*; that is still +playtesting. diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index 4f48b950..1796ea88 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -4,7 +4,22 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) include(CTest) -include("${CMAKE_CURRENT_SOURCE_DIR}/../runtime/chd_dependency.cmake") +# libchdr is needed by exactly one target here: the user-facing `psxrecomp` +# media CLI. It arrives via FetchContent + ExternalProject, which is by far the +# heaviest and most fragile thing in this configure — it needs the network on a +# cold cache, and ExternalProject cannot handle a source path containing a colon +# (a repo directory like "Marvel vs. Capcom: Clash of Super Heroes Recomp" +# truncates to "…/Marvel vs. Capcom.rule" and configure dies). +# +# Turning it off lets a caller that only wants an analysis or emitter binary +# configure this project without any of that. Nothing but the media CLI is +# affected; the emitters, psxrecomp-analyze, and every test are chd-free. +option(PSXRECOMP_ENABLE_CHD + "Build the psxrecomp media CLI (requires libchdr via FetchContent)" ON) + +if(PSXRECOMP_ENABLE_CHD) + include("${CMAKE_CURRENT_SOURCE_DIR}/../runtime/chd_dependency.cmake") +endif() add_subdirectory(lib/fmt) @@ -112,23 +127,60 @@ target_include_directories(psxrecomp-toml PRIVATE target_link_libraries(psxrecomp-toml PRIVATE fmt) # psxrecomp — user-facing media-to-source project generator. -add_executable(psxrecomp - src/main_cli.cpp - src/cli_boot_path.cpp +# The only libchdr consumer in this project; see PSXRECOMP_ENABLE_CHD above. +if(PSXRECOMP_ENABLE_CHD) + add_executable(psxrecomp + src/main_cli.cpp + src/cli_boot_path.cpp + src/ps1_exe_parser.cpp + ../runtime/src/cue_sheet.cpp + ../runtime/src/iso_reader.cpp + ) + target_include_directories(psxrecomp PRIVATE + include + ../runtime/include + lib/fmt/include + ) + target_link_libraries(psxrecomp PRIVATE fmt chdr-static) + if(MINGW) + target_link_options(psxrecomp PRIVATE -Wl,--stack,16777216) + elseif(MSVC) + target_link_options(psxrecomp PRIVATE /STACK:16777216) + endif() +endif() + +# psxrecomp-analyze — developer-facing static function discovery. +# +# NOT part of the recompilation path. It turns the boundary/CFG/jump-table +# knowledge the emitters already derive into a queryable artifact set for +# modding and decomp work (see docs/FUNCTION_DISCOVERY.md). +# +# The source list is the guarantee: no runtime/ translation unit appears here +# and none may be added. The tool's whole claim is that everything it reports +# was proven from the executable image alone, with no trace, overlay-capture, +# or executed-PC feedback. Linking the runtime would quietly make that false. +add_executable(psxrecomp-analyze src/ps1_exe_parser.cpp - ../runtime/src/cue_sheet.cpp - ../runtime/src/iso_reader.cpp + src/mips_decoder.cpp + src/function_analysis.cpp + src/analysis_db.cpp + src/bios_call_names.cpp + src/analysis_export.cpp + src/widescreen_scan.cpp + src/main_analyze.cpp ) -target_include_directories(psxrecomp PRIVATE +target_include_directories(psxrecomp-analyze PRIVATE include - ../runtime/include + src + ../runtime/include # ws_cull_detect.h — the shared auto_screen_x detector lib/fmt/include + lib/toml11 ) -target_link_libraries(psxrecomp PRIVATE fmt chdr-static) +target_link_libraries(psxrecomp-analyze PRIVATE fmt) if(MINGW) - target_link_options(psxrecomp PRIVATE -Wl,--stack,16777216) + target_link_options(psxrecomp-analyze PRIVATE -Wl,--stack,16777216) elseif(MSVC) - target_link_options(psxrecomp PRIVATE /STACK:16777216) + target_link_options(psxrecomp-analyze PRIVATE /STACK:16777216) endif() # ---- baked codegen hash (stale-binary guard) -------------------------------- @@ -212,7 +264,11 @@ endif() # libgcc/libstdc++ harvested from MSYS2. Keep opt-in for local debug builds. option(PSXRECOMP_STATIC_CLI "Statically link Windows runtime into CLI tools" OFF) if(PSXRECOMP_STATIC_CLI AND WIN32 AND (MINGW OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")) - foreach(_cli_target psxrecomp psxrecomp-game psxrecomp-toml psxrecomp-bios) + set(_cli_targets psxrecomp-game psxrecomp-toml psxrecomp-bios psxrecomp-analyze) + if(PSXRECOMP_ENABLE_CHD) + list(APPEND _cli_targets psxrecomp) + endif() + foreach(_cli_target IN LISTS _cli_targets) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") target_link_options(${_cli_target} PRIVATE -static -static-libgcc -static-libstdc++) diff --git a/recompiler/include/analysis_db.h b/recompiler/include/analysis_db.h new file mode 100644 index 00000000..5fc3ba41 --- /dev/null +++ b/recompiler/include/analysis_db.h @@ -0,0 +1,218 @@ +#pragma once +// analysis_db.h +// ---------------------------------------------------------------------------- +// Developer-facing static analysis database for a PS-X EXE. +// +// This is NOT part of the recompilation path. It exists so that the function +// knowledge the recompiler already derives — boundaries, call graph, indirect +// sites, jump tables, argument/return usage — becomes a queryable artifact for +// modders and decompilers instead of a throwaway intermediate. +// +// Two hard rules distinguish it from the emitters: +// +// 1. STATIC ONLY. Nothing here links, reads, or consults the runtime. There +// is no trace import, no overlay capture set, no executed-PC feedback. +// Every claim in the database is derived from the executable image alone. +// Coverage that requires dynamic evidence is reported as *missing*, never +// silently filled in. +// +// 2. TOLERANT, NOT FABRICATING. Unlike the recompiler (CLAUDE.md §0: an +// unsupported opcode is a hard failure), the analyzer degrades. A region +// it cannot fully decode is reported `partial` with the offending PCs +// listed. That is not a stub: it emits no code and asserts no semantics +// it did not prove. Reporting *incomplete* is permitted here; reporting +// *fabricated* is not. + +#include +#include +#include +#include +#include +#include + +#include "ps1_exe_parser.h" + +namespace PSXRecomp::Analysis { + +// How much the analyzer is willing to claim about a function boundary. +// Ordered strongest-first; the JSON/report renders the lowercase name. +enum class Confidence { + Verified, // prologue + epilogue + jr $ra exit, fully decoded, reached + // from the entry point by direct calls only + High, // fully decoded, jr $ra exit, reached from the entry point + Medium, // prologue-shaped or reached only via a recovered jump table + Low, // boundary inferred; unreached, no prologue, or partial decode + DataRegion, // classified as data masquerading as code +}; + +const char* confidence_name(Confidence c); + +// MIPS o32 register indices used in the bitmasks below. +constexpr uint8_t REG_ZERO = 0, REG_AT = 1, REG_V0 = 2, REG_V1 = 3; +constexpr uint8_t REG_A0 = 4, REG_A3 = 7, REG_S0 = 16, REG_S7 = 23; +constexpr uint8_t REG_GP = 28, REG_SP = 29, REG_FP = 30, REG_RA = 31; + +// Inferred calling-convention usage. Every field here is a HEURISTIC derived +// from a linear read-before-write scan over the function's own range; loops +// that read a register before the backward edge writes it will over-report an +// argument. Treat as a starting hypothesis for a decomp, not as ground truth — +// `sig_confident` is false whenever the scan hit a construct that weakens it. +struct Signature { + uint8_t arg_mask = 0; // bit i set => $a read before written + int arg_count = 0; // highest contiguous $a register used, + 1 + bool returns_v0 = false; // $v0 written on some path to a jr $ra + bool returns_v1 = false; + uint32_t saved_mask = 0; // registers stored to the stack frame + bool is_leaf = true; // no JAL/JALR anywhere in range + bool uses_gte = false; // any COP2 op + bool uses_syscall = false; + bool uses_break = false; + bool touches_mmio = false; // a statically resolved 0x1F80'xxxx access + bool reads_gp = false; + int32_t stack_frame = 0; + bool sig_confident = true; + std::string prototype; // rendered C prototype guess +}; + +struct FunctionRecord { + uint32_t addr = 0; + uint32_t end = 0; // exclusive + uint32_t size = 0; + uint32_t instruction_count = 0; + + std::string name; // symbols.toml name, else func_XXXXXXXX + bool user_named = false; + std::string status; // symbols.toml status (guessed|confirmed|hot) + std::string note; // symbols.toml note + + Confidence confidence = Confidence::Low; + std::string confidence_reason; + + bool has_prologue = false; + bool has_epilogue = false; + bool ends_jr_ra = false; + bool is_data = false; + bool partial = false; // contained words that would not decode + bool reachable = false; // from the EXE entry point + bool address_taken = false; // a statically resolved pointer names it + uint32_t alias_of = 0; // interior entry into another function + std::string bios_call; // "A0:3F printf" for a kernel dispatch thunk + + Signature sig; + std::vector block_leaders; + std::vector undecoded_pcs; + + // Filled by the graph pass. + uint32_t in_degree = 0; // distinct callers + uint32_t out_degree = 0; // distinct callees + uint32_t unresolved_indirect = 0; +}; + +struct CallEdge { + enum class Kind { Direct, JumpTable, AddressTaken, Indirect }; + uint32_t from_func = 0; + uint32_t site_pc = 0; + uint32_t to_addr = 0; + Kind kind = Kind::Direct; +}; + +const char* edge_kind_name(CallEdge::Kind k); + +// A statically resolved memory reference. Only pairs the analyzer could prove +// (lui/addiu, lui+load/store, $gp-relative) appear here — nothing inferred. +struct DataRef { + uint32_t from_func = 0; + uint32_t site_pc = 0; + uint32_t target = 0; + bool is_write = false; + uint8_t width = 0; // 1/2/4, 0 for address-materialization + std::string kind; // "ram" | "mmio" | "code" | "gp" | "addr" +}; + +struct IndirectSite { + uint32_t from_func = 0; + uint32_t pc = 0; + uint8_t reg = 0; + std::string kind; // "jr" | "jalr" + std::string classification; // "jump_table" | "bios_thunk" | "return_reg" + // | "unresolved" + std::vector targets; + uint32_t table_base = 0; + uint32_t table_count = 0; + std::vector context; // ~16 disassembled instructions before pc +}; + +struct Stats { + uint32_t total_functions = 0; + uint32_t total_instructions = 0; + uint32_t bytes_covered = 0; + uint32_t bytes_image = 0; + uint32_t confidence_counts[5] = {0, 0, 0, 0, 0}; + uint32_t reachable_functions = 0; + uint32_t orphan_functions = 0; + uint32_t named_functions = 0; + uint32_t direct_edges = 0; + uint32_t jump_tables_resolved = 0; + uint32_t jump_table_targets = 0; + uint32_t indirect_unresolved = 0; + uint32_t partial_functions = 0; + uint32_t undecoded_words = 0; + std::map opcode_histogram; +}; + +struct AnalysisDb { + std::string image_name; + uint32_t load_address = 0; + uint32_t entry_point = 0; + uint32_t initial_gp = 0; + uint32_t image_size = 0; + + std::vector functions; // sorted by addr + std::vector edges; + std::vector data_refs; + std::vector indirect; + Stats stats; + + const FunctionRecord* find(uint32_t addr) const; + // Function whose [addr,end) contains pc, or nullptr. + const FunctionRecord* containing(uint32_t pc) const; +}; + +struct Options { + std::filesystem::path symbols_toml; // optional, for name round-trip + std::vector extra_entries; // seed addresses to force + bool resolve_jump_tables = true; + bool collect_data_refs = true; + bool exact_entries = false; // reachability-only partition +}; + +// A parsed symbols.toml entry. Kept separate from FunctionRecord so names +// survive a re-analysis that no longer discovers the address. +struct SymbolEntry { + uint32_t pc = 0; + std::string name; + std::string status = "guessed"; + std::string note; + bool emit = false; +}; + +// Name for a PSX kernel A0/B0/C0 call, or "" if not carried. See +// bios_call_names.cpp for the table and its source. +const char* bios_call_name(uint32_t table, uint32_t index); + +bool load_symbols(const std::filesystem::path& path, + std::vector& out, + std::string& error); + +AnalysisDb build_analysis_db(const PS1Executable& exe, + const std::string& image_name, + const Options& opts, + std::string& error); + +// Disassemble [lo, hi) from the image. Used for the context windows and for +// the `disasm` export. +std::vector disassemble_range(const PS1Executable& exe, + uint32_t lo, uint32_t hi, + const AnalysisDb* db); + +} // namespace PSXRecomp::Analysis diff --git a/recompiler/include/widescreen_scan.h b/recompiler/include/widescreen_scan.h new file mode 100644 index 00000000..2ff69fbb --- /dev/null +++ b/recompiler/include/widescreen_scan.h @@ -0,0 +1,88 @@ +#pragma once +// widescreen_scan.h +// ---------------------------------------------------------------------------- +// Static discovery of [widescreen.cull] sites. +// +// Widescreen bring-up on a new title currently means finding a handful of exact +// instruction addresses by hand — Tomba's game.toml carries four bias/range +// pairs and three a1 nops, each located by reading disassembly or mining the +// runtime's overlay capture set with tools/overlay_xref.py. Every one of those +// sites has a crisp syntactic signature, so for a main-EXE title the search is +// a static one and needs no capture set at all. +// +// The site kinds and their REQUIRED instruction forms are dictated by +// code_generator.cpp, which hard-errors on a mismatch in main-EXE mode. This +// scanner therefore proposes only sites whose instruction already satisfies the +// emitter's check: a suggestion that would break the build is not a suggestion. + +#include +#include +#include +#include + +#include "analysis_db.h" +#include "ps1_exe_parser.h" + +namespace PSXRecomp::Analysis { + +struct WsCandidate { + // Matches the game.toml key it belongs under: bias_sites, range_sites, + // a1_sites, screen_x_sites, depth_sites. + std::string kind; + uint32_t pc = 0; + uint32_t func = 0; + uint32_t partner_pc = 0; // paired site (bias <-> range), else 0 + int confidence = 0; // 0..100 + std::string evidence; + std::string instr; // disassembled text at pc +}; + +struct WsScanOptions { + // Empty means "discover them": a title's screen extents are per-game + // (Tomba tests 0x140/0x141 on a 320 display, Ape Escape 0x181 on 368, + // Wipeout 3 0x200/0x240). Guessing wrong makes every later pass useless, + // so the scan derives candidates from the image before it uses them. + std::vector w_imms; + std::vector h_imms; + int max_candidates = 512; +}; + +struct WsScanResult { + std::vector w_imms; // used (configured or discovered) + std::vector h_imms; + bool imms_discovered = false; + // Whether the chosen sets land on real console display modes. A discovery + // that matches neither is a guess worth saying out loud rather than + // presenting with the same face as one that hit 320x240. + bool w_canonical = false; + bool h_canonical = false; + + // Functions carrying the screen-extent signature (a W compare and an H + // compare in the same function). `auto_screen_x = true` handles these + // without any per-address list, which is the single most useful thing to + // know before hand-listing anything. + std::vector extent_funcs; + + std::vector candidates; + std::map w_hist; // width-range slt immediate -> count + std::map h_hist; // height-range slt immediate -> count + uint32_t gte_funcs = 0; +}; + +WsScanResult scan_widescreen(const PS1Executable& exe, + const AnalysisDb& db, + const WsScanOptions& opts); + +// Ready-to-paste [widescreen] / [widescreen.cull] TOML. +bool write_ws_sites_toml(const WsScanResult& scan, + const std::filesystem::path& path, + int min_confidence, + std::string& error); + +bool write_ws_sites_json(const WsScanResult& scan, + const std::filesystem::path& path, + std::string& error); + +void print_ws_report(const WsScanResult& scan, const AnalysisDb& db, int top_n); + +} // namespace PSXRecomp::Analysis diff --git a/recompiler/src/analysis_db.cpp b/recompiler/src/analysis_db.cpp new file mode 100644 index 00000000..49243774 --- /dev/null +++ b/recompiler/src/analysis_db.cpp @@ -0,0 +1,1050 @@ +// analysis_db.cpp — see analysis_db.h for the two rules this file obeys. + +#include "analysis_db.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "function_analysis.h" +#include "mips_decoder.h" + +// toml11 is header-only; used only to read an existing symbols.toml so that +// hand-applied names survive re-analysis. +#include "toml.hpp" + +namespace PSXRecomp::Analysis { + +namespace { + +constexpr uint32_t kMmioLo = 0x1F801000; +constexpr uint32_t kMmioHi = 0x1F803000; + +inline uint32_t norm(uint32_t a) { return a & 0x1FFFFFFF; } + +// --------------------------------------------------------------------------- +// Register effect model +// +// Deliberately explicit rather than derived from DecodedInstruction's coarse +// is_load/is_alu flags: the signature pass needs to know *which* registers an +// instruction reads, and a wrong answer there silently corrupts every inferred +// prototype. Anything this table does not recognize contributes no reads and +// no writes, which is the conservative direction for read-before-write (it can +// only under-report arguments, never invent them). +// --------------------------------------------------------------------------- +struct RegEffect { + uint32_t reads = 0; + uint32_t writes = 0; +}; + +inline void add(uint32_t& mask, uint8_t r) { + if (r != 0) mask |= (1u << r); +} + +RegEffect reg_effect(const DecodedInstruction& d) { + RegEffect e; + switch (d.opcode) { + case 0x00: // SPECIAL + switch (d.funct) { + case 0x00: case 0x02: case 0x03: // SLL/SRL/SRA + add(e.reads, d.rt); add(e.writes, d.rd); break; + case 0x04: case 0x06: case 0x07: // SLLV/SRLV/SRAV + add(e.reads, d.rt); add(e.reads, d.rs); add(e.writes, d.rd); break; + case 0x08: // JR + add(e.reads, d.rs); break; + case 0x09: // JALR + add(e.reads, d.rs); add(e.writes, d.rd); break; + case 0x0C: case 0x0D: // SYSCALL/BREAK + break; + case 0x10: case 0x12: // MFHI/MFLO + add(e.writes, d.rd); break; + case 0x11: case 0x13: // MTHI/MTLO + add(e.reads, d.rs); break; + case 0x18: case 0x19: case 0x1A: case 0x1B: // MULT/MULTU/DIV/DIVU + add(e.reads, d.rs); add(e.reads, d.rt); break; + case 0x20: case 0x21: case 0x22: case 0x23: // ADD/ADDU/SUB/SUBU + case 0x24: case 0x25: case 0x26: case 0x27: // AND/OR/XOR/NOR + case 0x2A: case 0x2B: // SLT/SLTU + add(e.reads, d.rs); add(e.reads, d.rt); add(e.writes, d.rd); break; + default: break; + } + break; + case 0x01: // REGIMM + add(e.reads, d.rs); + if (d.rt == 0x10 || d.rt == 0x11) add(e.writes, REG_RA); // BLTZAL/BGEZAL + break; + case 0x02: break; // J + case 0x03: add(e.writes, REG_RA); break; // JAL + case 0x04: case 0x05: // BEQ/BNE + add(e.reads, d.rs); add(e.reads, d.rt); break; + case 0x06: case 0x07: // BLEZ/BGTZ + add(e.reads, d.rs); break; + case 0x08: case 0x09: case 0x0A: case 0x0B: // ADDI/ADDIU/SLTI/SLTIU + case 0x0C: case 0x0D: case 0x0E: // ANDI/ORI/XORI + add(e.reads, d.rs); add(e.writes, d.rt); break; + case 0x0F: add(e.writes, d.rt); break; // LUI + case 0x10: // COP0 + if (d.rs == 0x00) add(e.writes, d.rt); // MFC0 + else if (d.rs == 0x04) add(e.reads, d.rt); // MTC0 + break; + case 0x12: // COP2 / GTE + if (d.rs == 0x00 || d.rs == 0x02) add(e.writes, d.rt); // MFC2/CFC2 + else if (d.rs == 0x04 || d.rs == 0x06) add(e.reads, d.rt); // MTC2/CTC2 + break; + case 0x20: case 0x21: case 0x23: case 0x24: case 0x25: // LB/LH/LW/LBU/LHU + add(e.reads, d.rs); add(e.writes, d.rt); break; + case 0x22: case 0x26: // LWL/LWR: partial merge + add(e.reads, d.rs); add(e.reads, d.rt); add(e.writes, d.rt); break; + case 0x28: case 0x29: case 0x2A: case 0x2B: case 0x2E: // SB/SH/SWL/SW/SWR + add(e.reads, d.rs); add(e.reads, d.rt); break; + case 0x32: case 0x3A: // LWC2/SWC2 + add(e.reads, d.rs); break; + default: break; + } + return e; +} + +int mem_width(uint8_t opcode) { + switch (opcode) { + case 0x20: case 0x24: case 0x28: return 1; + case 0x21: case 0x25: case 0x29: return 2; + case 0x22: case 0x23: case 0x26: case 0x2A: case 0x2B: case 0x2E: + case 0x32: case 0x3A: return 4; + default: return 0; + } +} + +bool is_store_op(uint8_t opcode) { + return opcode == 0x28 || opcode == 0x29 || opcode == 0x2A || + opcode == 0x2B || opcode == 0x2E || opcode == 0x3A; +} + +bool is_load_op(uint8_t opcode) { + return opcode == 0x20 || opcode == 0x21 || opcode == 0x22 || + opcode == 0x23 || opcode == 0x24 || opcode == 0x25 || + opcode == 0x26 || opcode == 0x32; +} + +std::string classify_target(uint32_t target, uint32_t lo, uint32_t hi) { + uint32_t k = norm(target); + if (k >= norm(kMmioLo) && k < norm(kMmioHi)) return "mmio"; + if (target >= lo && target < hi) return "image"; + if (k < 0x00800000) return "ram"; + return "other"; +} + +// --------------------------------------------------------------------------- +// Disassembly formatting +// --------------------------------------------------------------------------- +std::string format_instr(const DecodedInstruction& d, + const AnalysisDb* db) { + auto rn = [](uint8_t r) { return MipsDecoder::register_name(r); }; + auto sym = [&](uint32_t a) -> std::string { + if (db) { + const FunctionRecord* f = db->find(a); + if (f) return fmt::format("0x{:08X} <{}>", a, f->name); + } + return fmt::format("0x{:08X}", a); + }; + + if (d.is_nop) return "nop"; + const char* m = d.mnemonic ? d.mnemonic : "???"; + std::string mn = m; + std::transform(mn.begin(), mn.end(), mn.begin(), + [](unsigned char c) { return std::tolower(c); }); + + switch (d.opcode) { + case 0x02: case 0x03: // J / JAL + return fmt::format("{:<8}{}", mn, sym(d.jump_target)); + case 0x04: case 0x05: // BEQ/BNE + return fmt::format("{:<8}{}, {}, 0x{:08X}", mn, rn(d.rs), rn(d.rt), + d.branch_target); + case 0x06: case 0x07: // BLEZ/BGTZ + return fmt::format("{:<8}{}, 0x{:08X}", mn, rn(d.rs), d.branch_target); + case 0x01: // REGIMM + return fmt::format("{:<8}{}, 0x{:08X}", mn, rn(d.rs), d.branch_target); + case 0x0F: // LUI + return fmt::format("{:<8}{}, 0x{:04X}", mn, rn(d.rt), d.uimm16); + case 0x08: case 0x09: case 0x0A: case 0x0B: + return fmt::format("{:<8}{}, {}, {}", mn, rn(d.rt), rn(d.rs), + d.sign_ext_imm); + case 0x0C: case 0x0D: case 0x0E: + return fmt::format("{:<8}{}, {}, 0x{:04X}", mn, rn(d.rt), rn(d.rs), + d.uimm16); + default: break; + } + + if (d.is_load || d.is_store) + return fmt::format("{:<8}{}, {}({})", mn, rn(d.rt), d.sign_ext_imm, + rn(d.rs)); + + if (d.opcode == 0x00) { + switch (d.funct) { + case 0x08: return fmt::format("{:<8}{}", mn, rn(d.rs)); + case 0x09: return fmt::format("{:<8}{}, {}", mn, rn(d.rd), rn(d.rs)); + case 0x00: case 0x02: case 0x03: + return fmt::format("{:<8}{}, {}, {}", mn, rn(d.rd), rn(d.rt), + (int)d.shamt); + case 0x0C: case 0x0D: return mn; + case 0x10: case 0x12: return fmt::format("{:<8}{}", mn, rn(d.rd)); + case 0x11: case 0x13: return fmt::format("{:<8}{}", mn, rn(d.rs)); + case 0x18: case 0x19: case 0x1A: case 0x1B: + return fmt::format("{:<8}{}, {}", mn, rn(d.rs), rn(d.rt)); + default: + return fmt::format("{:<8}{}, {}, {}", mn, rn(d.rd), rn(d.rs), + rn(d.rt)); + } + } + if (d.format == InstrFormat::UNKNOWN) + return fmt::format(".word 0x{:08X}", d.raw); + return fmt::format("{:<8}{}, {}, {}", mn, rn(d.rd), rn(d.rs), rn(d.rt)); +} + +// --------------------------------------------------------------------------- +// General jump-table recovery +// +// function_analysis.cpp's resolve_exact_bounded_jump_table() is tried first and +// is authoritative when it fires. It accepts exactly one instruction schedule +// (`sltiu; beq; nop; sll; addu; lw; jr`) because a wrong table there emits +// wrong CODE. This recoverer exists because that is the wrong trade for a +// discovery tool: on a real title the far more common schedule puts the `sll` +// in the branch's delay slot and materializes the table base between the `sll` +// and the `addu`, which the strict form rejects outright — leaving every switch +// in the image reported as an unresolved indirect jump. +// +// So this walks the actual backward slice instead of matching a fixed window, +// and labels how it terminated: `guarded` (an sltiu bound on the index register +// was found — the count is proven) or `scanned` (no bound found; the table was +// walked until a word stopped looking like a code address). A `scanned` table +// is a hypothesis and is reported as such; it never silently reads as proven. +struct RecoveredTable { + uint32_t base = 0; + uint32_t count = 0; + std::vector targets; + const char* method = ""; +}; + +// Nearest preceding instruction that writes `reg`, searching back at most +// `window` instructions and never before `lo`. Returns false if none. +bool find_def(const PS1Executable& exe, uint32_t lo, uint32_t from_pc, + uint8_t reg, uint32_t window, DecodedInstruction& out) { + uint32_t limit = (from_pc > lo + window * 4) ? from_pc - window * 4 : lo; + for (uint32_t pc = from_pc; pc >= limit; pc -= 4) { + auto w = exe.read_word(pc); + if (!w.has_value()) break; + DecodedInstruction d = MipsDecoder::decode(*w, pc); + if (reg_effect(d).writes & (1u << reg)) { out = d; return true; } + if (pc < 4 || pc == limit) break; + } + return false; +} + +// Constant value of `reg` at `at_pc`, if it comes from LUI or LUI+ADDIU. +bool const_value(const PS1Executable& exe, uint32_t lo, uint32_t at_pc, + uint8_t reg, uint32_t& out) { + DecodedInstruction d; + if (!find_def(exe, lo, at_pc, reg, 48, d)) return false; + if (d.opcode == 0x0F) { // LUI + out = static_cast(d.uimm16) << 16; + return true; + } + if (d.opcode == 0x09) { // ADDIU rt, rs, imm + uint32_t hi = 0; + if (d.address < lo + 4) return false; + if (!const_value(exe, lo, d.address - 4, d.rs, hi)) return false; + out = hi + static_cast(d.sign_ext_imm); + return true; + } + return false; +} + +bool recover_jump_table(const PS1Executable& exe, + uint32_t f_start, uint32_t f_end, + uint32_t jr_pc, uint8_t jr_rs, + const std::unordered_map& fn_index, + RecoveredTable& out) { + const uint32_t img_lo = exe.load_address(); + const uint32_t img_hi = exe.end_address(); + + // jr $rD <- lw $rD, off($rB) + DecodedInstruction lw; + if (jr_pc < f_start + 4) return false; + if (!find_def(exe, f_start, jr_pc - 4, jr_rs, 8, lw)) return false; + if (lw.opcode != 0x23) return false; // must be LW + const int32_t lw_off = lw.sign_ext_imm; + const uint8_t base_reg = lw.rs; + + // $rB <- addu $rB, , + DecodedInstruction addu; + if (lw.address < f_start + 4) return false; + if (!find_def(exe, f_start, lw.address - 4, base_reg, 8, addu)) return false; + if (!(addu.opcode == 0x00 && (addu.funct == 0x21 || addu.funct == 0x20))) + return false; // ADDU / ADD + + // One operand is `sll rX, rIdx, 2`; the other is a materialized constant. + uint8_t index_reg = 0; + uint32_t table_base = 0; + bool have_base = false; + const uint8_t ops[2] = {addu.rs, addu.rt}; + for (int k = 0; k < 2; ++k) { + DecodedInstruction d; + if (addu.address < f_start + 4) break; + if (!find_def(exe, f_start, addu.address - 4, ops[k], 16, d)) continue; + if (d.opcode == 0x00 && d.funct == 0x00 && d.shamt == 2) { + index_reg = d.rt; // SLL rd, rt, 2 + } + } + for (int k = 0; k < 2; ++k) { + uint32_t v = 0; + if (addu.address < f_start + 4) break; + if (const_value(exe, f_start, addu.address - 4, ops[k], v)) { + table_base = v; + have_base = true; + } + } + if (!index_reg || !have_base) return false; + + table_base += static_cast(lw_off); + if ((table_base & 3u) || table_base < img_lo || table_base + 3 >= img_hi) + return false; + + // Bound: nearest `sltiu/slti rX, rIdx, N` before the jr. + uint32_t count = 0; + const char* method = "scanned"; + { + uint32_t limit = (jr_pc > f_start + 64 * 4) ? jr_pc - 64 * 4 : f_start; + for (uint32_t pc = jr_pc; pc >= limit; pc -= 4) { + auto w = exe.read_word(pc); + if (!w.has_value()) break; + DecodedInstruction d = MipsDecoder::decode(*w, pc); + if ((d.opcode == 0x0B || d.opcode == 0x0A) && d.rs == index_reg && + d.uimm16 > 0 && d.uimm16 <= 4096) { + count = d.uimm16; + method = "guarded"; + break; + } + if (pc < 4 || pc == limit) break; + } + } + + auto plausible_target = [&](uint32_t t) { + if (t & 3u) return false; + if (t < img_lo || t + 3 >= img_hi) return false; + auto w = exe.read_word(t); + if (!w.has_value()) return false; + // A case label is either inside this function or a known function + // start; anything else is a coincidental word, not a target. + if (!(t >= f_start && t < f_end) && !fn_index.count(t)) return false; + return FunctionAnalyzer::is_valid_mips_word(*w); + }; + + if (count == 0) { + for (uint32_t k = 0; k < 256; ++k) { + auto w = exe.read_word(table_base + k * 4); + if (!w.has_value() || !plausible_target(*w)) break; + count = k + 1; + } + if (count < 2) return false; // a lone match is noise, not a table + } + + for (uint32_t k = 0; k < count; ++k) { + auto w = exe.read_word(table_base + k * 4); + if (!w.has_value()) return false; + if (!plausible_target(*w)) { + // A guarded count that does not validate means the slice was + // misread. Report nothing rather than a half-right table. + if (method[0] == 'g') return false; + break; + } + out.targets.push_back(*w); + } + if (out.targets.size() < 2) return false; + + out.base = table_base; + out.count = static_cast(out.targets.size()); + out.method = method; + return true; +} + +// Resolve `jalr $rD` / `jr $rD` where $rD holds a statically-known function +// address: either materialized inline (lui/addiu) or loaded from a fixed +// address whose stored word is a known function start. This is the callback / +// dispatch-slot pattern, and it is the second largest indirect class after +// switches. Anything whose pointer comes from a struct field or a computed +// address stays unresolved — correctly, since static analysis cannot know it. +bool recover_indirect_call(const PS1Executable& exe, uint32_t f_start, + uint32_t call_pc, uint8_t reg, + const std::unordered_map& fn_index, + uint32_t& target, const char*& how) { + if (call_pc < f_start + 4) return false; + DecodedInstruction def; + if (!find_def(exe, f_start, call_pc - 4, reg, 16, def)) return false; + + // Materialized directly: lui + addiu naming a function. + if (def.opcode == 0x09 || def.opcode == 0x0F) { + uint32_t v = 0; + if (const_value(exe, f_start, def.address, reg, v) && fn_index.count(v)) { + target = v; + how = "immediate"; + return true; + } + return false; + } + // Loaded from a statically-known address. + if (def.opcode == 0x23) { // LW rt, off(rB) + uint32_t base = 0; + if (def.address < f_start + 4) return false; + if (!const_value(exe, f_start, def.address - 4, def.rs, base)) + return false; + uint32_t slot = base + static_cast(def.sign_ext_imm); + auto w = exe.read_word(slot); + if (!w.has_value() || !fn_index.count(*w)) return false; + target = *w; + how = "static_slot"; + return true; + } + return false; +} + +std::string render_prototype(const FunctionRecord& f) { + std::string ret = (f.sig.returns_v0 || f.sig.returns_v1) ? "u32" : "void"; + std::string args; + for (int i = 0; i < f.sig.arg_count; ++i) { + if (i) args += ", "; + args += fmt::format("u32 a{}", i); + } + if (args.empty()) args = "void"; + return fmt::format("{} {}({})", ret, f.name, args); +} + +} // namespace + +const char* confidence_name(Confidence c) { + switch (c) { + case Confidence::Verified: return "verified"; + case Confidence::High: return "high"; + case Confidence::Medium: return "medium"; + case Confidence::Low: return "low"; + case Confidence::DataRegion: return "data"; + } + return "low"; +} + +const char* edge_kind_name(CallEdge::Kind k) { + switch (k) { + case CallEdge::Kind::Direct: return "direct"; + case CallEdge::Kind::JumpTable: return "jump_table"; + case CallEdge::Kind::AddressTaken: return "address_taken"; + case CallEdge::Kind::Indirect: return "indirect"; + } + return "direct"; +} + +const FunctionRecord* AnalysisDb::find(uint32_t addr) const { + auto it = std::lower_bound(functions.begin(), functions.end(), addr, + [](const FunctionRecord& f, uint32_t v) { return f.addr < v; }); + if (it != functions.end() && it->addr == addr) return &*it; + return nullptr; +} + +const FunctionRecord* AnalysisDb::containing(uint32_t pc) const { + auto it = std::upper_bound(functions.begin(), functions.end(), pc, + [](uint32_t v, const FunctionRecord& f) { return v < f.addr; }); + if (it == functions.begin()) return nullptr; + --it; + return (pc >= it->addr && pc < it->end) ? &*it : nullptr; +} + +// --------------------------------------------------------------------------- +// symbols.toml +// --------------------------------------------------------------------------- +bool load_symbols(const std::filesystem::path& path, + std::vector& out, + std::string& error) { + out.clear(); + if (path.empty()) return true; + std::error_code ec; + if (!std::filesystem::exists(path, ec)) return true; // absent is not an error + try { + const auto root = toml::parse(path.string()); + if (!root.contains("func")) return true; + const auto& arr = toml::find(root, "func"); + if (!arr.is_array()) return true; + for (const auto& e : arr.as_array()) { + SymbolEntry s; + if (!e.contains("pc")) continue; + s.pc = static_cast(toml::find(e, "pc")); + s.name = e.contains("name") ? toml::find(e, "name") : ""; + if (s.name.empty()) continue; + if (e.contains("status")) s.status = toml::find(e, "status"); + if (e.contains("note")) s.note = toml::find(e, "note"); + if (e.contains("emit")) s.emit = toml::find(e, "emit"); + out.push_back(s); + } + } catch (const std::exception& ex) { + error = fmt::format("symbols.toml parse failed: {}", ex.what()); + return false; + } + return true; +} + +// --------------------------------------------------------------------------- +// Main build +// --------------------------------------------------------------------------- +AnalysisDb build_analysis_db(const PS1Executable& exe, + const std::string& image_name, + const Options& opts, + std::string& error) { + AnalysisDb db; + db.image_name = image_name; + db.load_address = exe.load_address(); + db.entry_point = exe.entry_point(); + db.initial_gp = exe.header.initial_gp; + db.image_size = exe.code_size(); + + const uint32_t lo = exe.load_address(); + const uint32_t hi = exe.end_address(); + + auto read_w = [&](uint32_t a) -> uint32_t { + auto w = exe.read_word(a); + return w.has_value() ? *w : 0u; + }; + auto in_image = [&](uint32_t a) { return a >= lo && a + 3 < hi; }; + + // ---- pass 1: boundaries ------------------------------------------------ + FunctionAnalyzer analyzer(exe); + analyzer.add_forced_entry(exe.entry_point()); + for (uint32_t a : opts.extra_entries) analyzer.add_forced_entry(a); + + FunctionAnalysisResult ar; + if (opts.exact_entries) { + std::vector entries{exe.entry_point()}; + entries.insert(entries.end(), opts.extra_entries.begin(), + opts.extra_entries.end()); + ar = analyzer.analyze_exact_entries(entries); + } else { + ar = analyzer.analyze(); + } + + db.functions.reserve(ar.functions.size()); + for (const auto& f : ar.functions) { + FunctionRecord r; + r.addr = f.start_addr; + r.end = f.end_addr; + r.size = (f.end_addr > f.start_addr) ? (f.end_addr - f.start_addr) : 0; + r.instruction_count = r.size / 4; + r.name = fmt::format("func_{:08X}", f.start_addr); + r.has_prologue = f.has_prologue; + r.has_epilogue = f.has_epilogue; + r.is_data = f.is_data_section; + r.alias_of = f.alias_walk_lo; + r.sig.stack_frame = f.stack_frame_size; + db.functions.push_back(std::move(r)); + } + std::sort(db.functions.begin(), db.functions.end(), + [](const FunctionRecord& a, const FunctionRecord& b) { + return a.addr < b.addr; + }); + db.functions.erase( + std::unique(db.functions.begin(), db.functions.end(), + [](const FunctionRecord& a, const FunctionRecord& b) { + return a.addr == b.addr; + }), + db.functions.end()); + + std::unordered_map index; + for (size_t i = 0; i < db.functions.size(); ++i) + index[db.functions[i].addr] = i; + + auto containing_idx = [&](uint32_t pc) -> long { + auto it = std::upper_bound(db.functions.begin(), db.functions.end(), pc, + [](uint32_t v, const FunctionRecord& f) { return v < f.addr; }); + if (it == db.functions.begin()) return -1; + --it; + if (pc >= it->addr && pc < it->end) + return static_cast(it - db.functions.begin()); + return -1; + }; + + // ---- pass 2: per-function decode, CFG, edges, refs, signature ---------- + std::unordered_set callee_set; // for in_degree + std::vector> callers(db.functions.size()); + std::vector> callees(db.functions.size()); + + for (size_t fi = 0; fi < db.functions.size(); ++fi) { + FunctionRecord& f = db.functions[fi]; + if (f.is_data || f.size == 0) continue; + + // Block leaders: entry, every intra-range branch/jump target, and the + // instruction after each branch's delay slot. + std::set leaders{f.addr}; + std::vector code; + code.reserve(f.instruction_count); + for (uint32_t pc = f.addr; pc < f.end; pc += 4) { + if (!in_image(pc)) break; + DecodedInstruction d = MipsDecoder::decode(read_w(pc), pc); + if (d.format == InstrFormat::UNKNOWN) { + f.partial = true; + if (f.undecoded_pcs.size() < 64) f.undecoded_pcs.push_back(pc); + db.stats.undecoded_words++; + } + code.push_back(d); + } + for (const auto& d : code) { + if (d.is_branch) { + if (d.branch_target >= f.addr && d.branch_target < f.end) + leaders.insert(d.branch_target); + if (d.address + 8 < f.end) leaders.insert(d.address + 8); + } else if (d.opcode == 0x02) { // J + if (d.jump_target >= f.addr && d.jump_target < f.end) + leaders.insert(d.jump_target); + if (d.address + 8 < f.end) leaders.insert(d.address + 8); + } else if (d.is_jump) { + if (d.address + 8 < f.end) leaders.insert(d.address + 8); + } + } + f.block_leaders.assign(leaders.begin(), leaders.end()); + + // Terminal shape. + for (auto it = code.rbegin(); it != code.rend(); ++it) { + if (it->is_nop) continue; + if (it->is_jr_ra) { f.ends_jr_ra = true; } + break; + } + if (!f.ends_jr_ra) { + for (const auto& d : code) if (d.is_jr_ra) { f.ends_jr_ra = true; break; } + } + + // Signature scan + edge/ref collection in one walk. + uint32_t defined = 0; // registers written so far + uint32_t arg_mask = 0; + bool saw_backward_branch = false; + uint32_t hi_val[32] = {0}; + bool hi_ok[32] = {false}; + + for (size_t i = 0; i < code.size(); ++i) { + const DecodedInstruction& d = code[i]; + if (leaders.count(d.address)) { + std::memset(hi_ok, 0, sizeof(hi_ok)); + } + + db.stats.total_instructions++; + if (d.mnemonic) db.stats.opcode_histogram[d.mnemonic]++; + + RegEffect e = reg_effect(d); + for (uint8_t r = REG_A0; r <= REG_A3; ++r) { + if ((e.reads & (1u << r)) && !(defined & (1u << r))) + arg_mask |= (1u << (r - REG_A0)); + } + if (e.reads & (1u << REG_GP)) f.sig.reads_gp = true; + if (e.writes & (1u << REG_V0)) f.sig.returns_v0 = true; + if (e.writes & (1u << REG_V1)) f.sig.returns_v1 = true; + defined |= e.writes; + + if (d.is_branch && d.branch_target <= d.address) + saw_backward_branch = true; + if (d.opcode == 0x12) f.sig.uses_gte = true; + if (d.is_syscall) f.sig.uses_syscall = true; + if (d.is_break) f.sig.uses_break = true; + + // Callee-saved: `sw sN/ra/fp, off($sp)` inside the frame. + if (d.opcode == 0x2B && d.rs == REG_SP) { + if ((d.rt >= REG_S0 && d.rt <= REG_S7) || d.rt == REG_FP || + d.rt == REG_RA) + f.sig.saved_mask |= (1u << d.rt); + } + + // --- constant tracking for data refs / address-taken ------------ + if (d.opcode == 0x0F) { // LUI + hi_val[d.rt] = static_cast(d.uimm16) << 16; + hi_ok[d.rt] = true; + } else { + uint32_t resolved = 0; + bool have = false; + bool materialize = false; + + if (d.opcode == 0x09 && hi_ok[d.rs]) { // ADDIU rt, rs(hi), imm + resolved = hi_val[d.rs] + static_cast(d.sign_ext_imm); + have = true; + materialize = true; + } else if ((is_load_op(d.opcode) || is_store_op(d.opcode))) { + if (hi_ok[d.rs]) { + resolved = hi_val[d.rs] + + static_cast(d.sign_ext_imm); + have = true; + } else if (d.rs == REG_GP && db.initial_gp) { + resolved = db.initial_gp + + static_cast(d.sign_ext_imm); + have = true; + } + } + + if (have && opts.collect_data_refs) { + DataRef ref; + ref.from_func = f.addr; + ref.site_pc = d.address; + ref.target = resolved; + ref.is_write = is_store_op(d.opcode); + ref.width = static_cast(mem_width(d.opcode)); + ref.kind = materialize ? "addr" + : classify_target(resolved, lo, hi); + if (ref.kind == "mmio") f.sig.touches_mmio = true; + db.data_refs.push_back(std::move(ref)); + } + if (materialize) { + auto ti = index.find(resolved); + if (ti != index.end()) { + db.functions[ti->second].address_taken = true; + db.edges.push_back({f.addr, d.address, resolved, + CallEdge::Kind::AddressTaken}); + callers[ti->second].insert(f.addr); + callees[fi].insert(resolved); + } + } + + // Invalidate any hi constant this instruction clobbers. + for (uint8_t r = 1; r < 32; ++r) + if ((e.writes & (1u << r)) && !(d.opcode == 0x09 && r == d.rt && hi_ok[d.rs])) + hi_ok[r] = false; + if (d.opcode == 0x09 && hi_ok[d.rs] && d.rt != d.rs) + hi_ok[d.rt] = false; // the sum is an address, not a hi part + } + + // --- call edges -------------------------------------------------- + if (d.opcode == 0x03) { // JAL + f.sig.is_leaf = false; + db.edges.push_back({f.addr, d.address, d.jump_target, + CallEdge::Kind::Direct}); + db.stats.direct_edges++; + callees[fi].insert(d.jump_target); + auto ti = index.find(d.jump_target); + if (ti != index.end()) callers[ti->second].insert(f.addr); + } else if (d.opcode == 0x00 && d.funct == 0x09) { // JALR + f.sig.is_leaf = false; + } + } + + f.sig.arg_mask = static_cast(arg_mask); + f.sig.arg_count = 0; + for (int i = 0; i < 4; ++i) { + if (arg_mask & (1u << i)) f.sig.arg_count = i + 1; + } + f.sig.sig_confident = !saw_backward_branch && !f.partial; + } + + // ---- pass 2.5: PSY-Q kernel dispatch thunks --------------------------- + // Shape: an immediate load of 0xA0/0xB0/0xC0 into some register, a `jr` on + // that register, and an immediate load of the call index into $t1. This is + // an exact pattern, not a heuristic, so a match is Verified — which matters + // because the generic rules score these stubs `low` (no prologue, no + // jr $ra) despite them being among the most-called code in any image. + for (auto& f : db.functions) { + if (f.is_data || f.size < 12 || f.size > 32) continue; + uint32_t table = 0, idx = 0, tbl_reg = 32; + bool have_jr = false; + for (uint32_t pc = f.addr; pc < f.end && pc < f.addr + 16; pc += 4) { + if (!in_image(pc)) break; + DecodedInstruction d = MipsDecoder::decode(read_w(pc), pc); + const bool imm_load = (d.opcode == 0x09 || d.opcode == 0x0D) && + d.rs == REG_ZERO; // ADDIU/ORI rt, $zero, k + if (imm_load) { + if (d.uimm16 == 0xA0 || d.uimm16 == 0xB0 || d.uimm16 == 0xC0) { + table = d.uimm16; + tbl_reg = d.rt; + } else if (d.rt == 9) { // $t1 carries the index + idx = d.uimm16; + } + } else if (d.opcode == 0x00 && d.funct == 0x08 && d.rs == tbl_reg) { + have_jr = true; + } + } + if (!table || !have_jr) continue; + + const char* nm = bios_call_name(table, idx); + f.bios_call = fmt::format("{:02X}:{:02X}{}{}", table, idx, + *nm ? " " : "", nm); + f.name = *nm ? fmt::format("bios_{:02X}_{:02X}_{}", table, idx, nm) + : fmt::format("bios_{:02X}_{:02X}", table, idx); + f.confidence = Confidence::Verified; + f.confidence_reason = + fmt::format("PSY-Q kernel dispatch thunk to {:02X}({:02X}h)", + table, idx); + f.sig.prototype = fmt::format("/* kernel {:02X}({:02X}h) */ {}", + table, idx, f.name); + } + + // ---- pass 3: indirect sites + static jump-table recovery --------------- + for (size_t fi = 0; fi < db.functions.size(); ++fi) { + FunctionRecord& f = db.functions[fi]; + if (f.is_data || f.size == 0) continue; + for (uint32_t pc = f.addr; pc < f.end; pc += 4) { + if (!in_image(pc)) break; + DecodedInstruction d = MipsDecoder::decode(read_w(pc), pc); + const bool is_jr = (d.opcode == 0x00 && d.funct == 0x08); + const bool is_jalr = (d.opcode == 0x00 && d.funct == 0x09); + if (!is_jr && !is_jalr) continue; + if (is_jr && d.rs == REG_RA) continue; // ordinary return + + IndirectSite site; + site.from_func = f.addr; + site.pc = pc; + site.reg = d.rs; + site.kind = is_jr ? "jr" : "jalr"; + site.classification = "unresolved"; + + // The `jr` inside a recognized kernel thunk IS the dispatch — its + // target is the A0/B0/C0 vector, which pass 2.5 already identified + // exactly. Leaving it in the unresolved bucket would inflate the + // coverage gap with the one class of indirect transfer the tool + // understands best. + if (!f.bios_call.empty()) { + site.classification = fmt::format("bios_dispatch:{}", f.bios_call); + db.indirect.push_back(std::move(site)); + continue; + } + + if (is_jr && opts.resolve_jump_tables) { + // The emitter's proven form first; the general slice walk only + // if it declines. Provenance is kept in `classification` so a + // consumer can tell a proven table from an inferred one. + ExactJumpTable table; + if (resolve_exact_bounded_jump_table(exe, f.addr, f.end, pc, + d.rs, table) && + table.table_base && table.table_count) { + site.classification = "jump_table:proven"; + site.table_base = table.table_base; + site.table_count = table.table_count; + for (const auto& t : table.targets) + site.targets.push_back(t.second); + } else { + RecoveredTable rt; + if (recover_jump_table(exe, f.addr, f.end, pc, d.rs, index, + rt)) { + site.classification = + fmt::format("jump_table:{}", rt.method); + site.table_base = rt.base; + site.table_count = rt.count; + site.targets = rt.targets; + } + } + + if (!site.targets.empty()) { + for (uint32_t t : site.targets) { + if (t >= f.addr && t < f.end) { + // Intra-function switch case: control flow, not a + // call. It becomes a block leader instead of an edge. + f.block_leaders.push_back(t); + continue; + } + db.edges.push_back({f.addr, pc, t, + CallEdge::Kind::JumpTable}); + callees[fi].insert(t); + auto ti = index.find(t); + if (ti != index.end()) callers[ti->second].insert(f.addr); + } + std::sort(f.block_leaders.begin(), f.block_leaders.end()); + f.block_leaders.erase( + std::unique(f.block_leaders.begin(), f.block_leaders.end()), + f.block_leaders.end()); + db.stats.jump_tables_resolved++; + db.stats.jump_table_targets += + static_cast(site.targets.size()); + } + } + + if (site.classification == "unresolved") { + uint32_t t = 0; + const char* how = ""; + if (recover_indirect_call(exe, f.addr, pc, d.rs, index, t, how)) { + site.classification = fmt::format("call_ptr:{}", how); + site.targets.push_back(t); + db.edges.push_back({f.addr, pc, t, + CallEdge::Kind::Indirect}); + callees[fi].insert(t); + auto ti = index.find(t); + if (ti != index.end()) callers[ti->second].insert(f.addr); + } + } + + if (site.classification == "unresolved") { + f.unresolved_indirect++; + db.stats.indirect_unresolved++; + uint32_t ctx_lo = (pc >= f.addr + 64) ? pc - 64 : f.addr; + site.context = disassemble_range(exe, ctx_lo, pc + 8, nullptr); + } + db.indirect.push_back(std::move(site)); + } + } + + // ---- pass 4: pointer tables in data ----------------------------------- + // A run of >=2 consecutive 4-aligned words each naming a distinct known + // function start is treated as a dispatch table. A single isolated word is + // NOT: at this density a lone match is as likely to be a coincidental + // integer, and a false function pointer is worse than a missing one. + { + uint32_t run_start = 0; + std::vector run; + auto flush = [&]() { + if (run.size() >= 2) { + for (size_t k = 0; k < run.size(); ++k) { + auto ti = index.find(run[k]); + if (ti == index.end()) continue; + db.functions[ti->second].address_taken = true; + db.edges.push_back({0, + run_start + static_cast(k * 4), + run[k], CallEdge::Kind::AddressTaken}); + } + } + run.clear(); + }; + for (uint32_t a = lo; a + 3 < hi; a += 4) { + uint32_t w = read_w(a); + if (index.count(w)) { + if (run.empty()) run_start = a; + run.push_back(w); + } else { + flush(); + } + } + flush(); + } + + // ---- pass 5: reachability from the EXE entry point --------------------- + { + std::unordered_map> succ; + // Every edge kind here is statically proven — an Indirect edge only + // exists when the pointer's value was resolved — so all of them carry + // reachability. Nothing is reached "because the runtime saw it". + for (const auto& e : db.edges) + succ[e.from_func].push_back(e.to_addr); + long ei = containing_idx(db.entry_point); + std::queue q; + std::unordered_set seen; + if (ei >= 0) { + q.push(db.functions[ei].addr); + seen.insert(db.functions[ei].addr); + } + // Pointer-table edges have from_func == 0 (no owning function); seed + // their targets so a table-only-reachable handler is not called an + // orphan. + for (const auto& e : db.edges) { + if (e.from_func == 0 && seen.insert(e.to_addr).second) + q.push(e.to_addr); + } + while (!q.empty()) { + uint32_t cur = q.front(); q.pop(); + auto it = succ.find(cur); + if (it == succ.end()) continue; + for (uint32_t nx : it->second) + if (index.count(nx) && seen.insert(nx).second) q.push(nx); + } + for (auto& f : db.functions) f.reachable = seen.count(f.addr) > 0; + } + + // ---- pass 6: degrees --------------------------------------------------- + for (size_t i = 0; i < db.functions.size(); ++i) { + db.functions[i].in_degree = + static_cast(callers[i].size()); + db.functions[i].out_degree = + static_cast(callees[i].size()); + } + + // ---- pass 7: symbols --------------------------------------------------- + if (!opts.symbols_toml.empty()) { + std::vector syms; + std::string serr; + if (!load_symbols(opts.symbols_toml, syms, serr)) { + error = serr; + return db; + } + for (const auto& s : syms) { + auto it = index.find(s.pc); + if (it == index.end()) continue; + FunctionRecord& f = db.functions[it->second]; + f.name = s.name; + f.user_named = true; + f.status = s.status; + f.note = s.note; + } + } + + // ---- pass 8: confidence + prototypes ---------------------------------- + for (auto& f : db.functions) { + if (!f.bios_call.empty()) continue; // pass 2.5 already decided + if (f.is_data) { + f.confidence = Confidence::DataRegion; + f.confidence_reason = "classified as data"; + } else if (f.partial) { + f.confidence = Confidence::Low; + f.confidence_reason = fmt::format( + "{} word(s) did not decode", f.undecoded_pcs.size()); + } else if (f.has_prologue && f.has_epilogue && f.ends_jr_ra && + f.reachable && f.in_degree > 0) { + f.confidence = Confidence::Verified; + f.confidence_reason = + "prologue + epilogue + jr $ra, called from reachable code"; + } else if (f.ends_jr_ra && f.reachable) { + f.confidence = Confidence::High; + f.confidence_reason = "jr $ra exit, reachable from entry"; + } else if (f.has_prologue || f.address_taken) { + f.confidence = Confidence::Medium; + f.confidence_reason = f.address_taken + ? "address taken but no call-graph path from entry" + : "prologue-shaped, no proven call path"; + } else { + f.confidence = Confidence::Low; + f.confidence_reason = f.reachable + ? "reachable but no prologue and no jr $ra exit" + : "no prologue, no jr $ra exit, unreachable from entry"; + } + f.sig.prototype = render_prototype(f); + } + + // ---- stats ------------------------------------------------------------- + db.stats.total_functions = static_cast(db.functions.size()); + db.stats.bytes_image = hi - lo; + for (const auto& f : db.functions) { + db.stats.confidence_counts[static_cast(f.confidence)]++; + if (f.reachable) db.stats.reachable_functions++; + else db.stats.orphan_functions++; + if (f.user_named) db.stats.named_functions++; + if (f.partial) db.stats.partial_functions++; + if (!f.is_data) db.stats.bytes_covered += f.size; + } + + std::sort(db.edges.begin(), db.edges.end(), + [](const CallEdge& a, const CallEdge& b) { + if (a.from_func != b.from_func) return a.from_func < b.from_func; + return a.site_pc < b.site_pc; + }); + std::sort(db.data_refs.begin(), db.data_refs.end(), + [](const DataRef& a, const DataRef& b) { + return a.site_pc < b.site_pc; + }); + std::sort(db.indirect.begin(), db.indirect.end(), + [](const IndirectSite& a, const IndirectSite& b) { + return a.pc < b.pc; + }); + return db; +} + +std::vector disassemble_range(const PS1Executable& exe, + uint32_t lo, uint32_t hi, + const AnalysisDb* db) { + std::vector out; + for (uint32_t pc = lo; pc < hi; pc += 4) { + auto w = exe.read_word(pc); + if (!w.has_value()) break; + DecodedInstruction d = MipsDecoder::decode(*w, pc); + out.push_back(fmt::format("{:08X} {:08X} {}", pc, *w, + format_instr(d, db))); + } + return out; +} + +} // namespace PSXRecomp::Analysis diff --git a/recompiler/src/analysis_export.cpp b/recompiler/src/analysis_export.cpp new file mode 100644 index 00000000..4ca979a1 --- /dev/null +++ b/recompiler/src/analysis_export.cpp @@ -0,0 +1,564 @@ +// analysis_export.cpp — artifact writers for the static analysis database. + +#include "analysis_export.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" + +namespace PSXRecomp::Analysis { + +namespace { + +std::string json_escape(const std::string& s) { + std::string out; + out.reserve(s.size() + 8); + for (char c : s) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: + if (static_cast(c) < 0x20) + out += fmt::format("\\u{:04x}", static_cast(c)); + else + out += c; + } + } + return out; +} + +std::string q(const std::string& s) { return "\"" + json_escape(s) + "\""; } + +bool open_out(const std::filesystem::path& p, std::ofstream& f, + std::string& error) { + std::error_code ec; + if (p.has_parent_path()) + std::filesystem::create_directories(p.parent_path(), ec); + f.open(p, std::ios::binary | std::ios::trunc); + if (!f) { + error = fmt::format("cannot write {}", p.string()); + return false; + } + return true; +} + +std::string saved_regs_str(uint32_t mask) { + static const char* names[32] = { + "zero","at","v0","v1","a0","a1","a2","a3", + "t0","t1","t2","t3","t4","t5","t6","t7", + "s0","s1","s2","s3","s4","s5","s6","s7", + "t8","t9","k0","k1","gp","sp","fp","ra"}; + std::string s; + for (int i = 0; i < 32; ++i) { + if (mask & (1u << i)) { + if (!s.empty()) s += "|"; + s += names[i]; + } + } + return s; +} + +std::string tag_string(const FunctionRecord& f) { + std::string t; + auto put = [&](const char* s) { if (!t.empty()) t += "|"; t += s; }; + if (!f.bios_call.empty()) put("bios"); + if (f.sig.is_leaf) put("leaf"); + if (f.sig.uses_gte) put("gte"); + if (f.sig.uses_syscall) put("syscall"); + if (f.sig.uses_break) put("break"); + if (f.sig.touches_mmio) put("mmio"); + if (f.sig.reads_gp) put("gp"); + if (f.address_taken) put("addr_taken"); + if (!f.reachable) put("orphan"); + if (f.partial) put("partial"); + if (f.unresolved_indirect) put("indirect"); + if (f.alias_of) put("alias"); + return t; +} + +} // namespace + +// --------------------------------------------------------------------------- +bool write_json_bundle(const AnalysisDb& db, const std::filesystem::path& dir, + bool include_refs, std::string& error) { + std::error_code ec; + std::filesystem::create_directories(dir, ec); + + { + std::ofstream f; + if (!open_out(dir / "analysis.json", f, error)) return false; + f << "{\n"; + f << " \"schema\": \"psxrecomp static analysis v1\",\n"; + f << " \"image\": " << q(db.image_name) << ",\n"; + f << fmt::format(" \"load_address\": {},\n", db.load_address); + f << fmt::format(" \"entry_point\": {},\n", db.entry_point); + f << fmt::format(" \"initial_gp\": {},\n", db.initial_gp); + f << fmt::format(" \"image_size\": {},\n", db.image_size); + f << " \"static_only\": true,\n"; + f << " \"stats\": {\n"; + const auto& s = db.stats; + f << fmt::format(" \"total_functions\": {},\n", s.total_functions); + f << fmt::format(" \"total_instructions\": {},\n", s.total_instructions); + f << fmt::format(" \"bytes_covered\": {},\n", s.bytes_covered); + f << fmt::format(" \"bytes_image\": {},\n", s.bytes_image); + f << fmt::format(" \"verified\": {},\n", s.confidence_counts[0]); + f << fmt::format(" \"high\": {},\n", s.confidence_counts[1]); + f << fmt::format(" \"medium\": {},\n", s.confidence_counts[2]); + f << fmt::format(" \"low\": {},\n", s.confidence_counts[3]); + f << fmt::format(" \"data\": {},\n", s.confidence_counts[4]); + f << fmt::format(" \"reachable\": {},\n", s.reachable_functions); + f << fmt::format(" \"orphans\": {},\n", s.orphan_functions); + f << fmt::format(" \"named\": {},\n", s.named_functions); + f << fmt::format(" \"direct_edges\": {},\n", s.direct_edges); + f << fmt::format(" \"jump_tables_resolved\": {},\n", s.jump_tables_resolved); + f << fmt::format(" \"jump_table_targets\": {},\n", s.jump_table_targets); + f << fmt::format(" \"indirect_unresolved\": {},\n", s.indirect_unresolved); + f << fmt::format(" \"partial_functions\": {},\n", s.partial_functions); + f << fmt::format(" \"undecoded_words\": {}\n", s.undecoded_words); + f << " },\n"; + f << " \"functions\": [\n"; + for (size_t i = 0; i < db.functions.size(); ++i) { + const auto& fn = db.functions[i]; + f << " {"; + f << fmt::format("\"addr\": {}, ", fn.addr); + f << fmt::format("\"end\": {}, ", fn.end); + f << fmt::format("\"size\": {}, ", fn.size); + f << fmt::format("\"instructions\": {}, ", fn.instruction_count); + f << "\"name\": " << q(fn.name) << ", "; + f << fmt::format("\"user_named\": {}, ", fn.user_named ? "true" : "false"); + f << "\"status\": " << q(fn.status) << ", "; + f << "\"note\": " << q(fn.note) << ", "; + f << "\"confidence\": " << q(confidence_name(fn.confidence)) << ", "; + f << "\"confidence_reason\": " << q(fn.confidence_reason) << ", "; + f << fmt::format("\"reachable\": {}, ", fn.reachable ? "true" : "false"); + f << fmt::format("\"address_taken\": {}, ", fn.address_taken ? "true" : "false"); + f << fmt::format("\"partial\": {}, ", fn.partial ? "true" : "false"); + f << fmt::format("\"is_data\": {}, ", fn.is_data ? "true" : "false"); + f << fmt::format("\"has_prologue\": {}, ", fn.has_prologue ? "true" : "false"); + f << fmt::format("\"has_epilogue\": {}, ", fn.has_epilogue ? "true" : "false"); + f << fmt::format("\"ends_jr_ra\": {}, ", fn.ends_jr_ra ? "true" : "false"); + f << fmt::format("\"alias_of\": {}, ", fn.alias_of); + f << "\"bios_call\": " << q(fn.bios_call) << ", "; + f << fmt::format("\"in_degree\": {}, ", fn.in_degree); + f << fmt::format("\"out_degree\": {}, ", fn.out_degree); + f << fmt::format("\"unresolved_indirect\": {}, ", fn.unresolved_indirect); + f << fmt::format("\"blocks\": {}, ", fn.block_leaders.size()); + f << "\"sig\": {"; + f << fmt::format("\"args\": {}, ", fn.sig.arg_count); + f << fmt::format("\"arg_mask\": {}, ", fn.sig.arg_mask); + f << fmt::format("\"returns\": {}, ", + (fn.sig.returns_v0 || fn.sig.returns_v1) ? "true" : "false"); + f << fmt::format("\"leaf\": {}, ", fn.sig.is_leaf ? "true" : "false"); + f << fmt::format("\"gte\": {}, ", fn.sig.uses_gte ? "true" : "false"); + f << fmt::format("\"syscall\": {}, ", fn.sig.uses_syscall ? "true" : "false"); + f << fmt::format("\"mmio\": {}, ", fn.sig.touches_mmio ? "true" : "false"); + f << fmt::format("\"stack_frame\": {}, ", fn.sig.stack_frame); + f << "\"saved\": " << q(saved_regs_str(fn.sig.saved_mask)) << ", "; + f << fmt::format("\"confident\": {}, ", fn.sig.sig_confident ? "true" : "false"); + f << "\"prototype\": " << q(fn.sig.prototype); + f << "}"; + f << "}" << (i + 1 < db.functions.size() ? "," : "") << "\n"; + } + f << " ]\n}\n"; + } + + { + std::ofstream f; + if (!open_out(dir / "edges.json", f, error)) return false; + f << "{\n \"schema\": \"psxrecomp call edges v1\",\n \"edges\": [\n"; + for (size_t i = 0; i < db.edges.size(); ++i) { + const auto& e = db.edges[i]; + f << fmt::format(" {{\"from\": {}, \"pc\": {}, \"to\": {}, \"kind\": \"{}\"}}{}\n", + e.from_func, e.site_pc, e.to_addr, + edge_kind_name(e.kind), + i + 1 < db.edges.size() ? "," : ""); + } + f << " ]\n}\n"; + } + + { + std::ofstream f; + if (!open_out(dir / "indirect.json", f, error)) return false; + f << "{\n \"schema\": \"psxrecomp indirect sites v1\",\n \"sites\": [\n"; + for (size_t i = 0; i < db.indirect.size(); ++i) { + const auto& s = db.indirect[i]; + f << " {"; + f << fmt::format("\"func\": {}, \"pc\": {}, \"reg\": {}, ", + s.from_func, s.pc, s.reg); + f << "\"kind\": " << q(s.kind) << ", "; + f << "\"classification\": " << q(s.classification) << ", "; + f << fmt::format("\"table_base\": {}, \"table_count\": {}, ", + s.table_base, s.table_count); + f << "\"targets\": ["; + for (size_t k = 0; k < s.targets.size(); ++k) + f << (k ? "," : "") << s.targets[k]; + f << "], \"context\": ["; + for (size_t k = 0; k < s.context.size(); ++k) + f << (k ? ", " : "") << q(s.context[k]); + f << "]}"; + f << (i + 1 < db.indirect.size() ? "," : "") << "\n"; + } + f << " ]\n}\n"; + } + + if (include_refs) { + std::ofstream f; + if (!open_out(dir / "refs.json", f, error)) return false; + f << "{\n \"schema\": \"psxrecomp data refs v1\",\n \"refs\": [\n"; + for (size_t i = 0; i < db.data_refs.size(); ++i) { + const auto& r = db.data_refs[i]; + f << fmt::format(" {{\"func\": {}, \"pc\": {}, \"target\": {}, " + "\"write\": {}, \"width\": {}, \"kind\": \"{}\"}}{}\n", + r.from_func, r.site_pc, r.target, + r.is_write ? "true" : "false", r.width, r.kind, + i + 1 < db.data_refs.size() ? "," : ""); + } + f << " ]\n}\n"; + } + + return true; +} + +// --------------------------------------------------------------------------- +bool write_symbols_toml(const AnalysisDb& db, const std::filesystem::path& path, + Confidence min_conf, bool include_unnamed, + std::string& error) { + // APPEND-ONLY. An earlier version regenerated the file from the [[func]] + // entries it had parsed, which silently deleted everything else in it — + // and symbols.toml carries more than functions: Ape Escape has 11 + // [[object]] entries and a top-level `game` key, Crash Team Racing has 33 + // [[site]] entries, all hand-researched. Existing bytes are now never + // rewritten; newly discovered functions are appended and nothing else is + // touched. + std::vector existing; + if (!load_symbols(path, existing, error)) return false; + + std::set have; + for (const auto& s : existing) have.insert(s.pc); + + std::string original; + { + std::ifstream in(path, std::ios::binary); + if (in) + original.assign(std::istreambuf_iterator(in), + std::istreambuf_iterator()); + } + + std::string added_text; + uint32_t added = 0; + if (include_unnamed) { + for (const auto& f : db.functions) { + if (f.is_data) continue; + if (static_cast(f.confidence) > static_cast(min_conf)) continue; + if (have.count(f.addr)) continue; + const std::string note = + f.bios_call.empty() + ? fmt::format("{} — {} arg(s), {} caller(s), {}", + confidence_name(f.confidence), f.sig.arg_count, + f.in_degree, f.sig.is_leaf ? "leaf" : "non-leaf") + : fmt::format("kernel dispatch thunk {} — {} caller(s)", + f.bios_call, f.in_degree); + added_text += fmt::format( + "\n[[func]]\npc = 0x{:08X}\nname = \"{}\"\nemit = false\n" + "status = \"guessed\"\nnote = \"{}\"\n", + f.addr, f.name, note); + added++; + } + } + + if (added == 0) { + fmt::print(" symbols.toml: {} entries, none new\n", existing.size()); + return true; + } + + std::ofstream f; + if (!open_out(path, f, error)) return false; + if (original.empty()) { + f << "# " << db.image_name << " — progressive symbol map\n"; + f << "# Discover → label here → manipulate via PSX_FN_* (sync_symbols.py).\n"; + f << "# See psxrecomp/docs/SYMBOLS.md and docs/FUNCTION_DISCOVERY.md.\n"; + } else { + f << original; + if (original.back() != '\n') f << "\n"; + } + f << "\n# --- appended by `psxrecomp-analyze --emit-symbols` (" + << confidence_name(min_conf) << " or better) ---\n"; + f << added_text; + fmt::print(" symbols.toml: {} existing entries kept, {} appended\n", + existing.size(), added); + return true; +} + +// --------------------------------------------------------------------------- +bool write_symbol_addrs(const AnalysisDb& db, const std::filesystem::path& path, + std::string& error) { + std::ofstream f; + if (!open_out(path, f, error)) return false; + f << "// " << db.image_name << " — generated by psxrecomp-analyze\n"; + f << "// Static analysis only; no runtime evidence was consulted.\n"; + for (const auto& fn : db.functions) { + if (fn.is_data) continue; + f << fmt::format("{} = 0x{:08X}; // type:func size:0x{:X} conf:{}{}\n", + fn.name, fn.addr, fn.size, + confidence_name(fn.confidence), + fn.user_named ? " named" : ""); + } + return true; +} + +// --------------------------------------------------------------------------- +bool write_ghidra_script(const AnalysisDb& db, const std::filesystem::path& path, + std::string& error) { + std::ofstream f; + if (!open_out(path, f, error)) return false; + f << "# psxrecomp_import.py — generated by psxrecomp-analyze\n"; + f << "# Ghidra script: creates functions, applies names, and bookmarks the\n"; + f << "# indirect sites static analysis could not resolve.\n"; + f << "# Run against a project where " << db.image_name + << " is loaded at its EXE load address.\n"; + f << "# @category PSXRecomp\n\n"; + f << "from ghidra.program.model.symbol import SourceType\n\n"; + f << "fm = currentProgram.getFunctionManager()\n"; + f << "st = currentProgram.getSymbolTable()\n"; + f << "af = currentProgram.getAddressFactory().getDefaultAddressSpace()\n\n"; + f << "def addr(v):\n return af.getAddress(v)\n\n"; + f << "FUNCS = [\n"; + for (const auto& fn : db.functions) { + if (fn.is_data) continue; + f << fmt::format(" (0x{:08X}, 0x{:X}, \"{}\", \"{}\"),\n", + fn.addr, fn.size, json_escape(fn.name), + confidence_name(fn.confidence)); + } + f << "]\n\n"; + f << "UNRESOLVED = [\n"; + for (const auto& s : db.indirect) { + if (s.classification != "unresolved") continue; + f << fmt::format(" (0x{:08X}, \"{} ${}\"),\n", s.pc, s.kind, s.reg); + } + f << "]\n\n"; + f << "created = renamed = 0\n"; + f << "for a, size, name, conf in FUNCS:\n"; + f << " ea = addr(a)\n"; + f << " fn = fm.getFunctionAt(ea)\n"; + f << " if fn is None:\n"; + f << " fn = createFunction(ea, name)\n"; + f << " if fn is not None:\n"; + f << " created += 1\n"; + f << " if fn is not None and not name.startswith(\"func_\"):\n"; + f << " fn.setName(name, SourceType.IMPORTED)\n"; + f << " renamed += 1\n"; + f << " if fn is not None:\n"; + f << " setPlateComment(ea, \"psxrecomp: confidence=%s\" % conf)\n\n"; + f << "for a, what in UNRESOLVED:\n"; + f << " createBookmark(addr(a), \"psxrecomp\", \"unresolved %s\" % what)\n\n"; + f << "print(\"psxrecomp: %d functions created, %d named, %d unresolved \"\n"; + f << " \"indirect sites bookmarked\" % (created, renamed, len(UNRESOLVED)))\n"; + return true; +} + +// --------------------------------------------------------------------------- +bool write_tsv(const AnalysisDb& db, const std::filesystem::path& path, + std::string& error) { + std::ofstream f; + if (!open_out(path, f, error)) return false; + f << "addr\tsize\tname\tconfidence\targs\tret\tcallers\tcallees\t" + "frame\tsaved\ttags\n"; + for (const auto& fn : db.functions) { + f << fmt::format("0x{:08X}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n", + fn.addr, fn.size, fn.name, + confidence_name(fn.confidence), + fn.sig.arg_count, + (fn.sig.returns_v0 || fn.sig.returns_v1) ? "u32" : "void", + fn.in_degree, fn.out_degree, fn.sig.stack_frame, + saved_regs_str(fn.sig.saved_mask), + tag_string(fn)); + } + return true; +} + +// --------------------------------------------------------------------------- +bool write_disasm(const AnalysisDb& db, const PS1Executable& exe, uint32_t addr, + const std::filesystem::path& path, std::string& error) { + std::ofstream f; + if (!open_out(path, f, error)) return false; + + auto emit_one = [&](const FunctionRecord& fn) { + f << "\n/* ------------------------------------------------------------ */\n"; + f << fmt::format("/* {} [0x{:08X}..0x{:08X}) {} bytes */\n", + fn.name, fn.addr, fn.end, fn.size); + f << fmt::format("/* {} confidence={} ({}) */\n", fn.sig.prototype, + confidence_name(fn.confidence), fn.confidence_reason); + f << fmt::format("/* callers={} callees={} frame={} saved={} */\n", + fn.in_degree, fn.out_degree, fn.sig.stack_frame, + saved_regs_str(fn.sig.saved_mask)); + if (!fn.sig.sig_confident) + f << "/* NOTE: signature scan is approximate here (loop or partial " + "decode) */\n"; + std::set leaders(fn.block_leaders.begin(), + fn.block_leaders.end()); + for (const auto& line : disassemble_range(exe, fn.addr, fn.end, &db)) { + uint32_t pc = 0; + std::istringstream(line.substr(0, 8)) >> std::hex >> pc; + if (leaders.count(pc) && pc != fn.addr) + f << fmt::format("\n.L_{:08X}:\n", pc); + f << " " << line << "\n"; + } + }; + + if (addr) { + const FunctionRecord* fn = db.find(addr); + if (!fn) { + fn = db.containing(addr); + if (!fn) { + error = fmt::format("no function at or containing 0x{:08X}", addr); + return false; + } + } + emit_one(*fn); + } else { + for (const auto& fn : db.functions) + if (!fn.is_data) emit_one(fn); + } + return true; +} + +// --------------------------------------------------------------------------- +void print_report(const AnalysisDb& db, int top_n) { + const auto& s = db.stats; + fmt::print("\n=== {} ===\n", db.image_name); + fmt::print(" load 0x{:08X} entry 0x{:08X} gp 0x{:08X} size 0x{:X}\n", + db.load_address, db.entry_point, db.initial_gp, db.image_size); + fmt::print("\nFunctions: {} ({} instructions, {:.1f}% of image covered)\n", + s.total_functions, s.total_instructions, + s.bytes_image ? 100.0 * s.bytes_covered / s.bytes_image : 0.0); + fmt::print(" verified {:6} high {:6} medium {:6} low {:6} data {:6}\n", + s.confidence_counts[0], s.confidence_counts[1], + s.confidence_counts[2], s.confidence_counts[3], + s.confidence_counts[4]); + fmt::print(" reachable from entry: {} orphans: {} named: {}\n", + s.reachable_functions, s.orphan_functions, s.named_functions); + + fmt::print("\nControl flow:\n"); + fmt::print(" direct calls {}\n", s.direct_edges); + fmt::print(" jump tables recovered {} ({} targets)\n", + s.jump_tables_resolved, s.jump_table_targets); + fmt::print(" indirect sites unresolved {}\n", s.indirect_unresolved); + if (s.partial_functions) + fmt::print(" partial decodes {} functions, {} words\n", + s.partial_functions, s.undecoded_words); + + // Coverage honesty: say plainly what static analysis did not reach. + fmt::print("\nStatic coverage gap (NOT filled from any runtime source):\n"); + fmt::print(" {} indirect transfer(s) have no proven target set.\n", + s.indirect_unresolved); + uint32_t uncovered = (s.bytes_image > s.bytes_covered) + ? s.bytes_image - s.bytes_covered : 0; + fmt::print(" {} bytes ({:.1f}%) of the image belong to no code function.\n", + uncovered, s.bytes_image ? 100.0 * uncovered / s.bytes_image : 0.0); + + if (top_n > 0) { + std::vector by_callers; + for (const auto& f : db.functions) + if (!f.is_data) by_callers.push_back(&f); + std::sort(by_callers.begin(), by_callers.end(), + [](const FunctionRecord* a, const FunctionRecord* b) { + if (a->in_degree != b->in_degree) + return a->in_degree > b->in_degree; + return a->size > b->size; + }); + fmt::print("\nMost-called functions (best first targets for naming):\n"); + int n = 0; + for (const auto* f : by_callers) { + if (n++ >= top_n) break; + fmt::print(" {:>4} callers 0x{:08X} {:<28} {:<9} {}\n", + f->in_degree, f->addr, f->name, + confidence_name(f->confidence), f->sig.prototype); + } + + std::vector unres; + for (const auto& s2 : db.indirect) + if (s2.classification == "unresolved") unres.push_back(&s2); + if (!unres.empty()) { + fmt::print("\nUnresolved indirect sites (the discovery worklist):\n"); + n = 0; + for (const auto* u : unres) { + if (n++ >= top_n) break; + const FunctionRecord* owner = db.find(u->from_func); + fmt::print(" 0x{:08X} {} ${:<2} in {}\n", u->pc, u->kind, + u->reg, owner ? owner->name : "?"); + } + if (static_cast(unres.size()) > top_n) + fmt::print(" … {} more\n", unres.size() - top_n); + } + } + fmt::print("\n"); +} + +// --------------------------------------------------------------------------- +bool print_diff(const AnalysisDb& db, const std::filesystem::path& previous_tsv, + std::string& error) { + std::ifstream in(previous_tsv); + if (!in) { + error = fmt::format("cannot read {}", previous_tsv.string()); + return false; + } + struct Prev { uint32_t size; std::string name, conf; }; + std::map prev; + std::string line; + std::getline(in, line); // header + while (std::getline(in, line)) { + std::istringstream ls(line); + std::string addr_s, size_s, name, conf; + if (!std::getline(ls, addr_s, '\t')) continue; + if (!std::getline(ls, size_s, '\t')) continue; + if (!std::getline(ls, name, '\t')) continue; + if (!std::getline(ls, conf, '\t')) continue; + uint32_t a = 0, sz = 0; + try { + a = static_cast(std::stoul(addr_s, nullptr, 16)); + sz = static_cast(std::stoul(size_s)); + } catch (...) { continue; } + prev[a] = {sz, name, conf}; + } + + int added = 0, removed = 0, resized = 0, reconf = 0; + fmt::print("\n=== diff vs {} ===\n", previous_tsv.string()); + for (const auto& f : db.functions) { + auto it = prev.find(f.addr); + if (it == prev.end()) { + fmt::print(" + 0x{:08X} {:<28} {} ({} bytes)\n", f.addr, f.name, + confidence_name(f.confidence), f.size); + added++; + } else { + if (it->second.size != f.size) { + fmt::print(" ~ 0x{:08X} {:<28} size {} -> {}\n", f.addr, + f.name, it->second.size, f.size); + resized++; + } + if (it->second.conf != confidence_name(f.confidence)) { + fmt::print(" ~ 0x{:08X} {:<28} {} -> {}\n", f.addr, f.name, + it->second.conf, confidence_name(f.confidence)); + reconf++; + } + } + } + for (const auto& [a, p] : prev) { + if (!db.find(a)) { + fmt::print(" - 0x{:08X} {:<28} {} (no longer discovered)\n", a, + p.name, p.conf); + removed++; + } + } + fmt::print("\n {} added, {} removed, {} resized, {} confidence changes\n\n", + added, removed, resized, reconf); + return true; +} + +} // namespace PSXRecomp::Analysis diff --git a/recompiler/src/analysis_export.h b/recompiler/src/analysis_export.h new file mode 100644 index 00000000..c943ada9 --- /dev/null +++ b/recompiler/src/analysis_export.h @@ -0,0 +1,65 @@ +#pragma once +// analysis_export.h — writers that turn an AnalysisDb into artifacts. +// +// The JSON files are the stable public interface: RetComM Studio and any other +// front end consume those, never the analyzer's in-process types. Everything +// else here exists to hand the data to tools people already use rather than to +// make them adopt a new one. + +#include +#include + +#include "analysis_db.h" + +namespace PSXRecomp::Analysis { + +// analysis.json / edges.json / refs.json / indirect.json in `dir`. +bool write_json_bundle(const AnalysisDb& db, + const std::filesystem::path& dir, + bool include_refs, + std::string& error); + +// Merge discovered functions into a symbols.toml, preserving every existing +// entry's name/status/note verbatim. New entries land as status="guessed". +// `min_conf` gates which newly discovered functions are worth writing. +bool write_symbols_toml(const AnalysisDb& db, + const std::filesystem::path& path, + Confidence min_conf, + bool include_unnamed, + std::string& error); + +// decomp-toolchain symbol map: "name = 0xADDR; // size:0xNN". +bool write_symbol_addrs(const AnalysisDb& db, + const std::filesystem::path& path, + std::string& error); + +// Ghidra headless/script importer: creates functions, applies names, and +// annotates unresolved indirect sites as bookmarks. +bool write_ghidra_script(const AnalysisDb& db, + const std::filesystem::path& path, + std::string& error); + +// Line-oriented listing for grep/awk/spreadsheet workflows. Also the input +// `--diff` consumes: a stable one-line-per-function format diffs cleanly and +// needs no JSON parser on the read side. +bool write_tsv(const AnalysisDb& db, + const std::filesystem::path& path, + std::string& error); + +// Annotated disassembly for one function (or the whole image when addr == 0). +bool write_disasm(const AnalysisDb& db, + const PS1Executable& exe, + uint32_t addr, + const std::filesystem::path& path, + std::string& error); + +// Human-readable summary to stdout. +void print_report(const AnalysisDb& db, int top_n); + +// Difference between a previous run's functions.tsv and the current db: +// what appeared, what vanished, what changed boundary or confidence. +bool print_diff(const AnalysisDb& db, + const std::filesystem::path& previous_tsv, + std::string& error); + +} // namespace PSXRecomp::Analysis diff --git a/recompiler/src/bios_call_names.cpp b/recompiler/src/bios_call_names.cpp new file mode 100644 index 00000000..ef79888e --- /dev/null +++ b/recompiler/src/bios_call_names.cpp @@ -0,0 +1,117 @@ +// bios_call_names.cpp — names for the PSX kernel's A0/B0/C0 call tables. +// +// A PSY-Q-built title reaches the kernel through three-instruction dispatch +// thunks (`addiu $t2, $zero, 0xA0; jr $t2; addiu $t1, $zero, `). Static +// analysis sees those as tiny prologue-less stubs with no jr $ra, which is +// exactly the shape it trusts least — yet they are among the most-called +// functions in any image. Naming them turns the single worst-looking cluster in +// the report into the best-understood one. +// +// Table source: the A0/B0/C0 function lists in psx-spx (Nocash's PlayStation +// Specifications), "BIOS Function Summary". Entries left empty below are ones +// not carried here; those render as `bios_A0_XX` with no name claim rather than +// a guess. + +#include +#include + +namespace PSXRecomp::Analysis { + +namespace { + +const char* const kA0[] = { + /*00*/ "FileOpen", "FileSeek", "FileRead", "FileWrite", "FileClose", + /*05*/ "FileIoctl", "exit", "FileGetDeviceFlag", "FileGetc", "FilePutc", + /*0A*/ "todigit", "atof", "strtoul", "strtol", "abs", "labs", + /*10*/ "atoi", "atol", "atob", "setjmp", "longjmp", "strcat", "strncat", + /*17*/ "strcmp", "strncmp", "strcpy", "strncpy", "strlen", "index", + /*1D*/ "rindex", "strchr", "strrchr", + /*20*/ "strpbrk", "strspn", "strcspn", "strtok", "strstr", "toupper", + /*26*/ "tolower", "bcopy", "bzero", "bcmp", "memcpy", "memset", "memmove", + /*2D*/ "memcmp", "memchr", "rand", + /*30*/ "srand", "qsort", "strtod", "malloc", "free", "lsearch", "bsearch", + /*37*/ "calloc", "realloc", "InitHeap", "SystemErrorExit", "std_in_getchar", + /*3C*/ "std_out_putchar", "std_in_gets", "std_out_puts", "printf", + /*40*/ "SystemErrorUnresolvedException", "LoadExeHeader", "LoadExeFile", + /*43*/ "DoExecute", "FlushCache", "init_a0_b0_c0_vectors", "GPU_dw", + /*47*/ "gpu_send_dma", "SendGP1Command", "GPU_cw", "GPU_cwp", + /*4B*/ "send_gpu_linked_list", "gpu_abort_dma", "GetGPUStatus", "gpu_sync", + /*4F*/ "SystemError", + /*50*/ "SystemError", "LoadAndExecute", "SystemError", "SystemError", + /*54*/ "CdInit", "_bu_init", "CdRemove", "", "", "", "", + /*5B*/ "dev_tty_init", "dev_tty_open", "dev_tty_in_out", "dev_tty_ioctl", + /*5F*/ "dev_cd_open", + /*60*/ "dev_cd_read", "dev_cd_close", "dev_cd_firstfile", "dev_cd_nextfile", + /*64*/ "dev_cd_chdir", "dev_card_open", "dev_card_read", "dev_card_write", + /*68*/ "dev_card_close", "dev_card_firstfile", "dev_card_nextfile", + /*6B*/ "dev_card_erase", "dev_card_undelete", "dev_card_format", + /*6E*/ "dev_card_rename", "card_clear_error", + /*70*/ "_bu_init", "CdInit", "CdRemove", "", "", "", "", "", + /*78*/ "CdAsyncSeekL", "", "", "", "CdAsyncGetStatus", "", + /*7E*/ "CdAsyncReadSector", "", + /*80*/ "", "CdAsyncSetMode", "", "", "", "", "", "", + /*88*/ "", "", "", "", "", "", "", "", + /*90*/ "CdromIoIrqFunc1", "CdromDmaIrqFunc1", "CdromIoIrqFunc2", + /*93*/ "CdromDmaIrqFunc2", "CdromGetInt5errCode", "CdInitSubFunc", + /*96*/ "AddCDROMDevice", "AddMemCardDevice", "AddDuartTtyDevice", + /*99*/ "AddDummyTtyDevice", "SystemError", "SystemError", "SetConf", + /*9D*/ "GetConf", "SetCdromIrqAutoAbort", "SetMemSize", +}; + +const char* const kB0[] = { + /*00*/ "alloc_kernel_memory", "free_kernel_memory", "init_timer", + /*03*/ "get_timer", "enable_timer_irq", "disable_timer_irq", + /*06*/ "restart_timer", "DeliverEvent", "OpenEvent", "CloseEvent", + /*0A*/ "WaitEvent", "TestEvent", "EnableEvent", "DisableEvent", + /*0E*/ "OpenThread", "CloseThread", + /*10*/ "ChangeThread", "", "InitPad", "StartPad", "StopPad", + /*15*/ "OutdatedPadInitAndStart", "OutdatedPadGetButtons", + /*17*/ "ReturnFromException", "SetDefaultExitFromException", + /*19*/ "SetCustomExitFromException", "", "", "", "", "", "", + /*20*/ "UnDeliverEvent", "", "", "", "", "", "", "", + /*28*/ "", "", "", "", "", "", "", "", + /*30*/ "", "", + /*32*/ "FileOpen", "FileSeek", "FileRead", "FileWrite", "FileClose", + /*37*/ "FileIoctl", "exit", "FileGetDeviceFlag", "FileGetc", "FilePutc", + /*3C*/ "std_in_getchar", "std_out_putchar", "std_in_gets", "std_out_puts", + /*40*/ "chdir", "FormatDevice", "firstfile", "nextfile", "FileRename", + /*45*/ "FileDelete", "FileUndelete", "AddDevice", "RemoveDevice", + /*49*/ "PrintInstalledDevices", "InitCard", "StartCard", "StopCard", + /*4D*/ "_card_info_subfunc", "_card_write", "_card_read", + /*50*/ "_new_card", "Krom2RawAdd", "SystemError", "Krom2Offset", + /*54*/ "GetLastError", "GetLastFileError", "GetC0Table", "GetB0Table", + /*58*/ "get_bu_callback_port", "testdevice", "SystemError", + /*5B*/ "ChangeClearPad", "get_card_status", "wait_card_status", +}; + +const char* const kC0[] = { + /*00*/ "EnqueueTimerAndVblankIrqs", "EnqueueSyscallHandler", "SysEnqIntRP", + /*03*/ "SysDeqIntRP", "get_free_EvCB_slot", "get_free_TCB_slot", + /*06*/ "ExceptionHandler", "InstallExceptionHandlers", "SysInitMemory", + /*09*/ "SysInitKernelVariables", "ChangeClearRCnt", "SystemError", + /*0C*/ "InitDefInt", "SetIrqAutoAck", "", "", + /*10*/ "", "", "InstallDevices", "FlushStdInOutPut", "SystemError", + /*15*/ "tty_cdevinput", "tty_cdevscan", "tty_circgetc", "tty_circputc", + /*19*/ "ioabort", "set_card_find_mode", "KernelRedirect", "AdjustA0Table", + /*1D*/ "get_card_find_mode", +}; + +template +const char* lookup(const char* const (&tab)[N], uint32_t idx) { + if (idx >= N) return ""; + return tab[idx]; +} + +} // namespace + +// Returns "" when the index is out of range or intentionally uncarried. +const char* bios_call_name(uint32_t table, uint32_t index) { + switch (table) { + case 0xA0: return lookup(kA0, index); + case 0xB0: return lookup(kB0, index); + case 0xC0: return lookup(kC0, index); + default: return ""; + } +} + +} // namespace PSXRecomp::Analysis diff --git a/recompiler/src/main_analyze.cpp b/recompiler/src/main_analyze.cpp new file mode 100644 index 00000000..5e989fc0 --- /dev/null +++ b/recompiler/src/main_analyze.cpp @@ -0,0 +1,344 @@ +// main_analyze.cpp — `psxrecomp-analyze`, the developer-facing function +// discovery tool. +// +// This binary deliberately links NOTHING from runtime/. That is the guarantee +// the tool sells: every function, edge, and jump table it reports was proven +// from the executable image alone. Coverage that would need dynamic evidence +// is reported as a gap, never filled in silently. + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +#include "fmt/format.h" + +#include "analysis_db.h" +#include "analysis_export.h" +#include "widescreen_scan.h" +#include "ps1_exe_parser.h" + +namespace fs = std::filesystem; +using namespace PSXRecomp; +using namespace PSXRecomp::Analysis; + +namespace { + +void usage() { + fmt::print(R"(psxrecomp-analyze — static function discovery for PS-X EXE images + +USAGE + psxrecomp-analyze [options] + +INPUT + --symbols symbols.toml to read names from (and merge back into + with --emit-symbols). Existing names always win. + --entry <0xADDR> extra seed entry point; repeatable + --seeds file of one address per line (# comments allowed) + --exact reachability-only partition (no whole-image prologue + scan). Fewer, higher-confidence functions. + +OUTPUT + --out write analysis.json / edges.json / indirect.json + (+ refs.json unless --no-refs) + --no-refs skip refs.json (it is the largest artifact) + --emit-symbols merge discovered functions into --symbols path + --min-confidence gate for --emit-symbols: verified|high|medium|low + (default: high) + --emit-symbol-addrs decomp-style "name = 0xADDR;" map + --emit-ghidra Ghidra import script (.py) + --emit-tsv one line per function; also the --diff input + --disasm annotated listing for --at (or whole image) + --at <0xADDR> function to disassemble with --disasm + --diff compare against a previous run's TSV + +WIDESCREEN + --scan-widescreen find [widescreen.cull] site candidates statically + --ws-w-imms screen_w_imms override, e.g. 0x140,0x141 (else discovered) + --ws-h-imms screen_h_imms override, e.g. 0xF0,0xE0 + --emit-ws-sites write a ready-to-paste [widescreen.cull] TOML block + --ws-min-confidence gate for --emit-ws-sites (default 70) + --top rows in the stdout report's top lists (default 15) + --quiet suppress the stdout report + +Static analysis only. No runtime, trace, or overlay-capture input is consulted. +)"); +} + +bool parse_addr(const std::string& s, uint32_t& out) { + try { + size_t idx = 0; + unsigned long v = std::stoul(s, &idx, + (s.rfind("0x", 0) == 0 || s.rfind("0X", 0) == 0) + ? 16 : 0); + if (idx != s.size()) return false; + out = static_cast(v); + return true; + } catch (...) { + return false; + } +} + +bool load_seed_file(const fs::path& p, std::vector& out, + std::string& err) { + std::ifstream in(p); + if (!in) { err = fmt::format("cannot read {}", p.string()); return false; } + std::string line; + while (std::getline(in, line)) { + auto hash = line.find('#'); + if (hash != std::string::npos) line.resize(hash); + // Seed files in the wild carry a trailing label; take the first token. + std::istringstream ls(line); + std::string tok; + if (!(ls >> tok)) continue; + uint32_t a = 0; + if (parse_addr(tok, a)) out.push_back(a); + } + return true; +} + +// FunctionAnalyzer narrates its boundary scan straight to stdout. That is +// useful in the log pane of a GUI, but --quiet has to mean quiet, so redirect +// the descriptor around the call rather than editing the shared analyzer. +class StdoutMute { +public: + explicit StdoutMute(bool active) : active_(active) { + if (!active_) return; + std::fflush(stdout); +#ifdef _WIN32 + saved_ = _dup(_fileno(stdout)); + FILE* null_fp = nullptr; + fopen_s(&null_fp, "NUL", "w"); + if (null_fp) { _dup2(_fileno(null_fp), _fileno(stdout)); fclose(null_fp); } +#else + saved_ = dup(fileno(stdout)); + FILE* null_fp = std::fopen("/dev/null", "w"); + if (null_fp) { dup2(fileno(null_fp), fileno(stdout)); std::fclose(null_fp); } +#endif + } + ~StdoutMute() { + if (!active_ || saved_ < 0) return; + std::fflush(stdout); +#ifdef _WIN32 + _dup2(saved_, _fileno(stdout)); + _close(saved_); +#else + dup2(saved_, fileno(stdout)); + close(saved_); +#endif + } + StdoutMute(const StdoutMute&) = delete; + StdoutMute& operator=(const StdoutMute&) = delete; + +private: + bool active_; + int saved_ = -1; +}; + +Confidence parse_confidence(const std::string& s) { + if (s == "verified") return Confidence::Verified; + if (s == "high") return Confidence::High; + if (s == "medium") return Confidence::Medium; + return Confidence::Low; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc < 2) { usage(); return 1; } + + fs::path exe_path; + Options opts; + fs::path out_dir, symbol_addrs, ghidra, tsv, disasm, diff; + bool emit_symbols = false, quiet = false, no_refs = false; + bool scan_ws = false; + fs::path ws_sites; + WsScanOptions ws_opts; + int ws_min_conf = 70; + Confidence min_conf = Confidence::High; + uint32_t disasm_at = 0; + int top_n = 15; + + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + auto need = [&](const char* what) -> std::string { + if (i + 1 >= argc) { + fmt::print(stderr, "error: {} requires a value\n", what); + std::exit(1); + } + return argv[++i]; + }; + if (a == "-h" || a == "--help") { usage(); return 0; } + else if (a == "--symbols") opts.symbols_toml = need("--symbols"); + else if (a == "--out") out_dir = need("--out"); + else if (a == "--no-refs") no_refs = true; + else if (a == "--exact") opts.exact_entries = true; + else if (a == "--emit-symbols") emit_symbols = true; + else if (a == "--min-confidence") min_conf = parse_confidence(need("--min-confidence")); + else if (a == "--emit-symbol-addrs") symbol_addrs = need("--emit-symbol-addrs"); + else if (a == "--emit-ghidra") ghidra = need("--emit-ghidra"); + else if (a == "--emit-tsv") tsv = need("--emit-tsv"); + else if (a == "--disasm") disasm = need("--disasm"); + else if (a == "--diff") diff = need("--diff"); + else if (a == "--quiet") quiet = true; + else if (a == "--scan-widescreen") scan_ws = true; + else if (a == "--emit-ws-sites") { ws_sites = need("--emit-ws-sites"); scan_ws = true; } + else if (a == "--ws-min-confidence") ws_min_conf = std::atoi(need("--ws-min-confidence").c_str()); + else if (a == "--ws-w-imms" || a == "--ws-h-imms") { + const bool width = (a == "--ws-w-imms"); + std::string list = need(a.c_str()); + auto& dst = width ? ws_opts.w_imms : ws_opts.h_imms; + std::string tok; + std::istringstream ls(list); + while (std::getline(ls, tok, ',')) { + uint32_t v = 0; + if (!tok.empty() && parse_addr(tok, v)) dst.push_back(v); + } + scan_ws = true; + } + else if (a == "--top") top_n = std::atoi(need("--top").c_str()); + else if (a == "--at") { + uint32_t v = 0; + std::string s = need("--at"); + if (!parse_addr(s, v)) { + fmt::print(stderr, "error: bad address '{}'\n", s); + return 1; + } + disasm_at = v; + } else if (a == "--entry") { + uint32_t v = 0; + std::string s = need("--entry"); + if (!parse_addr(s, v)) { + fmt::print(stderr, "error: bad address '{}'\n", s); + return 1; + } + opts.extra_entries.push_back(v); + } else if (a == "--seeds") { + std::string err; + if (!load_seed_file(need("--seeds"), opts.extra_entries, err)) { + fmt::print(stderr, "error: {}\n", err); + return 1; + } + } else if (!a.empty() && a[0] == '-') { + fmt::print(stderr, "error: unknown option '{}'\n", a); + return 1; + } else if (exe_path.empty()) { + exe_path = a; + } else { + fmt::print(stderr, "error: unexpected argument '{}'\n", a); + return 1; + } + } + + if (exe_path.empty()) { usage(); return 1; } + if (no_refs) opts.collect_data_refs = false; + + std::string err; + auto exe = PS1ExeParser::parse_file(exe_path, err); + if (!exe) { + fmt::print(stderr, "error: {}\n", err); + return 1; + } + + std::sort(opts.extra_entries.begin(), opts.extra_entries.end()); + opts.extra_entries.erase( + std::unique(opts.extra_entries.begin(), opts.extra_entries.end()), + opts.extra_entries.end()); + + if (!quiet) { + fmt::print("Analyzing {} ({} bytes at 0x{:08X})", + exe_path.filename().string(), exe->code_size(), + exe->load_address()); + if (!opts.extra_entries.empty()) + fmt::print(" with {} seed(s)", opts.extra_entries.size()); + fmt::print("\n"); + } + + AnalysisDb db; + { + StdoutMute mute(quiet); + db = build_analysis_db(*exe, exe_path.filename().string(), opts, err); + } + if (!err.empty()) { + fmt::print(stderr, "error: {}\n", err); + return 1; + } + + if (!quiet) print_report(db, top_n); + + if (!out_dir.empty()) { + if (!write_json_bundle(db, out_dir, !no_refs, err)) { + fmt::print(stderr, "error: {}\n", err); + return 1; + } + fmt::print(" wrote {}/analysis.json, edges.json, indirect.json{}\n", + out_dir.string(), no_refs ? "" : ", refs.json"); + } + if (emit_symbols) { + if (opts.symbols_toml.empty()) { + fmt::print(stderr, + "error: --emit-symbols requires --symbols \n"); + return 1; + } + if (!write_symbols_toml(db, opts.symbols_toml, min_conf, true, err)) { + fmt::print(stderr, "error: {}\n", err); + return 1; + } + } + if (!symbol_addrs.empty()) { + if (!write_symbol_addrs(db, symbol_addrs, err)) { + fmt::print(stderr, "error: {}\n", err); return 1; + } + fmt::print(" wrote {}\n", symbol_addrs.string()); + } + if (!ghidra.empty()) { + if (!write_ghidra_script(db, ghidra, err)) { + fmt::print(stderr, "error: {}\n", err); return 1; + } + fmt::print(" wrote {}\n", ghidra.string()); + } + if (!tsv.empty()) { + if (!write_tsv(db, tsv, err)) { + fmt::print(stderr, "error: {}\n", err); return 1; + } + fmt::print(" wrote {}\n", tsv.string()); + } + if (!disasm.empty()) { + if (!write_disasm(db, *exe, disasm_at, disasm, err)) { + fmt::print(stderr, "error: {}\n", err); return 1; + } + fmt::print(" wrote {}\n", disasm.string()); + } + if (scan_ws) { + WsScanResult ws = scan_widescreen(*exe, db, ws_opts); + if (!quiet) print_ws_report(ws, db, top_n); + if (!out_dir.empty()) { + if (!write_ws_sites_json(ws, out_dir / "widescreen_sites.json", err)) { + fmt::print(stderr, "error: {}\n", err); return 1; + } + fmt::print(" wrote {}/widescreen_sites.json\n", out_dir.string()); + } + if (!ws_sites.empty()) { + if (!write_ws_sites_toml(ws, ws_sites, ws_min_conf, err)) { + fmt::print(stderr, "error: {}\n", err); return 1; + } + fmt::print(" wrote {}\n", ws_sites.string()); + } + } + if (!diff.empty()) { + if (!print_diff(db, diff, err)) { + fmt::print(stderr, "error: {}\n", err); return 1; + } + } + return 0; +} diff --git a/recompiler/src/widescreen_scan.cpp b/recompiler/src/widescreen_scan.cpp new file mode 100644 index 00000000..fdc4a72a --- /dev/null +++ b/recompiler/src/widescreen_scan.cpp @@ -0,0 +1,521 @@ +// widescreen_scan.cpp — see widescreen_scan.h. + +#include "widescreen_scan.h" + +#include +#include +#include + +#include "fmt/format.h" +#include "mips_decoder.h" + +// The SAME detector the recompiler and the runtime interpreter use for +// auto_screen_x. Sharing it verbatim is the point: a function this scanner +// calls a screen-extent funnel is exactly one the emitter will widen, so the +// report's "already covered by auto_screen_x" claim cannot drift from reality. +#include "ws_cull_detect.h" + +namespace PSXRecomp::Analysis { + +namespace { + +// Plausible PSX display extents. The console's horizontal modes are 256/320/ +// 368/384/512/640 and the vertical 240/256/480; a trivial-reject immediate is +// one of those (occasionally +1 for an inclusive bound), so these windows are +// wide enough to catch any real title and narrow enough to keep the histogram +// meaningful. +constexpr uint32_t kWLo = 0x100, kWHi = 0x290; // 256 .. 656 +constexpr uint32_t kHLo = 0xC0, kHHi = 0x201; // 192 .. 513 + +bool plausible_w(uint32_t imm) { return imm >= kWLo && imm <= kWHi; } +bool plausible_h(uint32_t imm) { return imm >= kHLo && imm <= kHHi; } + +// The console's actual display modes. A trivial-reject immediate is normally +// one of these (or one past, for an inclusive bound), so hitting one is real +// evidence rather than coincidence — it is weighted, never required, because a +// title is free to reject against a custom viewport. +bool canonical_w(uint32_t v) { + static const uint32_t k[] = {256, 320, 368, 384, 512, 576, 640}; + for (uint32_t c : k) if (v == c || v == c + 1) return true; + return false; +} +bool canonical_h(uint32_t v) { + static const uint32_t k[] = {224, 240, 256, 480}; + for (uint32_t c : k) if (v == c || v == c + 1) return true; + return false; +} + +uint32_t rd_of(uint32_t w) { return (w >> 11) & 0x1F; } +uint32_t rt_of(uint32_t w) { return (w >> 16) & 0x1F; } +uint32_t rs_of(uint32_t w) { return (w >> 21) & 0x1F; } +uint32_t op_of(uint32_t w) { return w >> 26; } +uint32_t funct_of(uint32_t w) { return w & 0x3F; } +int32_t simm_of(uint32_t w) { return static_cast(w & 0xFFFF); } +uint32_t uimm_of(uint32_t w) { return w & 0xFFFF; } + +std::string dis1(const PS1Executable& exe, uint32_t pc, const AnalysisDb& db) { + auto lines = disassemble_range(exe, pc, pc + 4, &db); + if (lines.empty()) return {}; + // disassemble_range prefixes "PC WORD "; keep just the mnemonic text. + const std::string& l = lines.front(); + const size_t cut = l.find(" ", 18); + return cut == std::string::npos ? l : l.substr(20); +} + +struct FnWords { + uint32_t addr = 0; + uint32_t end = 0; + bool uses_gte = false; + bool takes_a1 = false; // $a1 read before written — a real parameter + std::vector words; +}; + +} // namespace + +WsScanResult scan_widescreen(const PS1Executable& exe, + const AnalysisDb& db, + const WsScanOptions& opts) { + WsScanResult out; + + // ---- collect each code function's instruction words once --------------- + std::vector fns; + fns.reserve(db.functions.size()); + for (const auto& f : db.functions) { + if (f.is_data || f.size == 0) continue; + FnWords fw; + fw.addr = f.addr; + fw.end = f.end; + fw.uses_gte = f.sig.uses_gte; + fw.takes_a1 = (f.sig.arg_mask & 0x2u) != 0; // bit 1 == $a1 + fw.words.reserve(f.instruction_count); + for (uint32_t pc = f.addr; pc < f.end; pc += 4) { + auto w = exe.read_word(pc); + if (!w.has_value()) break; + fw.words.push_back(*w); + } + if (fw.uses_gte) out.gte_funcs++; + fns.push_back(std::move(fw)); + } + + // ---- pass 1: discover the per-game W/H immediates ---------------------- + // A trivial-reject compare shows up many times across the render funnels, + // so the right immediates are the frequent ones — but only inside GTE code. + // Restricting the histogram that way is what keeps ordinary bounds checks + // (array sizes, string lengths) from drowning out the display extents. + for (const auto& fw : fns) { + for (uint32_t w : fw.words) { + const uint32_t op = op_of(w); + if (op != 0x0A && op != 0x0B) continue; + const uint32_t imm = uimm_of(w); + if (plausible_w(imm)) out.w_hist[imm]++; + if (plausible_h(imm)) out.h_hist[imm]++; + } + } + + out.w_imms = opts.w_imms; + out.h_imms = opts.h_imms; + if (out.w_imms.empty() || out.h_imms.empty()) { + out.imms_discovered = true; + // Frequency alone picks the wrong immediates: ordinary bounds checks in + // 83 GTE functions outvoted Wipeout 3's real 0x200/0x240/0x100, which + // appear in only three funnels. What actually identifies the pair is + // CO-OCCURRENCE — the screen-extent signature is a width compare and a + // height compare in the SAME function — so the search scores pairs, not + // singletons. That also resolves the genuine ambiguity where a value is + // plausible as both (0x100 is a 256-wide mode and Wipeout 3's height): + // it wins whichever side its partner puts it on. + // Weighted, not gated, on GTE use: Wipeout 3's three extent funnels + // carry no COP2 op of their own (they reject before projecting), so a + // hard `uses_gte` filter found nothing at all there. Weighting keeps the + // precision that filter was buying without discarding those titles. + std::map, uint32_t> pairs; + for (const auto& fw : fns) { + const uint32_t weight = fw.uses_gte ? 3u : 1u; + std::set imms; + for (uint32_t w : fw.words) { + const uint32_t op = op_of(w); + if (op == 0x0A || op == 0x0B) imms.insert(uimm_of(w)); + } + for (uint32_t a : imms) { + if (!plausible_w(a)) continue; + for (uint32_t b : imms) { + if (a == b || !plausible_h(b)) continue; + pairs[{a, b}] += weight; + } + } + } + auto score = [&](uint32_t a, uint32_t b, uint32_t n) { + return n * (canonical_w(a) ? 3u : 1u) * (canonical_h(b) ? 3u : 1u); + }; + uint32_t best_w = 0, best_h = 0, best = 0; + for (const auto& [ab, n] : pairs) { + const uint32_t sc = score(ab.first, ab.second, n); + if (sc > best) { best = sc; best_w = ab.first; best_h = ab.second; } + } + if (best) { + auto partners = [&](bool want_w) { + std::vector> v; + for (const auto& [ab, n] : pairs) { + const bool fixed = want_w ? (ab.second == best_h) : (ab.first == best_w); + if (!fixed) continue; + const uint32_t key = want_w ? ab.first : ab.second; + v.push_back({key, score(ab.first, ab.second, n)}); + } + std::sort(v.begin(), v.end(), [](auto& x, auto& y) { + if (x.second != y.second) return x.second > y.second; + return x.first < y.first; + }); + std::vector r; + for (const auto& [k, sc] : v) { + if (r.size() >= 3) break; + if (sc * 12 < v.front().second) break; // noise, not a second extent + r.push_back(k); + } + return r; + }; + if (out.w_imms.empty()) out.w_imms = partners(true); + if (out.h_imms.empty()) { + out.h_imms = partners(false); + const std::vector chosen_w = out.w_imms; + out.h_imms.erase( + std::remove_if(out.h_imms.begin(), out.h_imms.end(), + [&](uint32_t v) { + return std::find(chosen_w.begin(), chosen_w.end(), + v) != chosen_w.end(); + }), + out.h_imms.end()); + } + } + + // A title usually tests BOTH the exclusive and the inclusive bound + // (the framework default is {0x140, 0x141}), and the pair scoring only + // surfaces whichever is more common. Dropping the sibling is not + // cosmetic: Tomba's bias/range pairing keys on + // `sltiu_imm - 2*bias == W`, and 449 - 128 is 0x141, so without it + // three of its four pairs go undetected. The sibling is added only when + // the image actually contains a compare against it. + auto complete_bounds = [](std::vector& set, + const std::map& hist) { + std::vector add; + for (uint32_t v : set) + for (int d : {-1, 1}) { + const uint32_t sib = static_cast(static_cast(v) + d); + if (hist.count(sib) && + std::find(set.begin(), set.end(), sib) == set.end() && + std::find(add.begin(), add.end(), sib) == add.end()) + add.push_back(sib); + } + set.insert(set.end(), add.begin(), add.end()); + std::sort(set.begin(), set.end()); + }; + complete_bounds(out.w_imms, out.w_hist); + complete_bounds(out.h_imms, out.h_hist); + } + for (uint32_t v : out.w_imms) if (canonical_w(v)) out.w_canonical = true; + for (uint32_t v : out.h_imms) if (canonical_h(v)) out.h_canonical = true; + + if (out.w_imms.empty() || out.h_imms.empty()) return out; // nothing to key off + + // ---- pass 2: screen-extent funnels (auto_screen_x coverage) ------------ + for (const auto& fw : fns) { + if (psx_ws_cull_scan(fw.words.data(), static_cast(fw.words.size()), + out.w_imms.data(), static_cast(out.w_imms.size()), + out.h_imms.data(), static_cast(out.h_imms.size()))) + out.extent_funcs.push_back(fw.addr); + } + const std::set extent(out.extent_funcs.begin(), out.extent_funcs.end()); + + // ---- pass 3: bias/range pairs (masked-u16 X window) -------------------- + // The idiom, verbatim from Tomba 0x80022E78: + // addiu $v0, $v0, 64 <- bias_site (+halfwidth) + // andi $v0, $v0, 0xFFFF <- optional u16 mask + // sltiu $v0, $v0, 449 <- range_site (W + 2*halfwidth) + // beq $v0, $zero, reject + // The arithmetic tie between the two immediates is what makes this safe to + // propose: `sltiu_imm - 2*bias_imm` must land on a configured W immediate. + // Without that check any `addiu`+`sltiu` neighbours would qualify. + for (const auto& fw : fns) { + for (size_t i = 0; i + 2 < fw.words.size(); ++i) { + const uint32_t a = fw.words[i]; + if (op_of(a) != 0x08 && op_of(a) != 0x09) continue; // addi/addiu + const int32_t k = simm_of(a); + if (k <= 0 || k > 1024) continue; + const uint32_t dest = rt_of(a); + if (dest == 0) continue; + + for (size_t j = i + 1; j <= i + 3 && j < fw.words.size(); ++j) { + const uint32_t s = fw.words[j]; + if (op_of(s) == 0x0C && rt_of(s) == dest && rs_of(s) == dest) + continue; // andi mask + if (op_of(s) != 0x0B) break; // must be sltiu + if (rs_of(s) != dest) break; // on the biased value + const uint32_t win = uimm_of(s); + const int32_t base = static_cast(win) - 2 * k; + bool keyed = false; + for (uint32_t wi : out.w_imms) + if (base == static_cast(wi)) keyed = true; + if (!keyed) break; + + const uint32_t bias_pc = fw.addr + static_cast(i * 4); + const uint32_t range_pc = fw.addr + static_cast(j * 4); + const int conf = extent.count(fw.addr) ? 95 : 85; + const std::string why = fmt::format( + "masked-u16 X window: +{} bias, sltiu {} = W({}) + 2*{}", + k, win, base, k); + out.candidates.push_back({"bias_sites", bias_pc, fw.addr, range_pc, + conf, why, dis1(exe, bias_pc, db)}); + out.candidates.push_back({"range_sites", range_pc, fw.addr, bias_pc, + conf, why, dis1(exe, range_pc, db)}); + break; + } + } + } + + // ---- pass 4: a1 margin sites ------------------------------------------ + // Some classifiers take the horizontal margin as a PARAMETER and are called + // by the render funnel rather than being one; Tomba's live at 0x80022C08, + // which uses no GTE op and has no static caller at all (it is reached + // through a dispatch). Gating this pass on the containing function being a + // screen-extent funnel therefore excluded every real site. The signature + // that actually holds is local: the function takes $a1, folds it into a + // coordinate with `addu`, and has a repurposable nop just before — a nop is + // the only form code_generator.cpp will rewrite here, and never one already + // owned by a branch delay slot. + for (const auto& fw : fns) { + if (!fw.takes_a1) continue; + bool emitted = false; + for (size_t i = 0; i < fw.words.size() && !emitted; ++i) { + const uint32_t w = fw.words[i]; + if (op_of(w) != 0x00 || funct_of(w) != 0x21) continue; // addu + if (rs_of(w) != 5 && rt_of(w) != 5) continue; // uses $a1 + if (rd_of(w) == 5) continue; // not a1 = a1 + x + for (size_t back = 1; back <= 4 && back <= i; ++back) { + const size_t n = i - back; + if (fw.words[n] != 0u) continue; // nop only + if (n > 0) { + DecodedInstruction prev = MipsDecoder::decode( + fw.words[n - 1], fw.addr + static_cast((n - 1) * 4)); + if (prev.is_branch || prev.is_jump) break; // delay slot + } + const uint32_t pc = fw.addr + static_cast(n * 4); + const bool near_gte = fw.uses_gte || extent.count(fw.addr); + out.candidates.push_back( + {"a1_sites", pc, fw.addr, fw.addr + static_cast(i * 4), + near_gte ? 65 : 50, + fmt::format("takes $a1; repurposable nop {} word(s) before `addu` " + "folds it into a coordinate{}", back, + near_gte ? " (GTE/extent context)" : ""), + dis1(exe, pc, db)}); + emitted = true; // one site per classifier + break; + } + } + } + + // ---- pass 5: X compares outside any extent funnel ---------------------- + // These are the ones auto_screen_x will NOT reach, so they are exactly the + // addresses that still need hand-listing under screen_x_sites. + for (const auto& fw : fns) { + if (extent.count(fw.addr)) continue; + for (size_t i = 0; i < fw.words.size(); ++i) { + const uint32_t w = fw.words[i]; + if (op_of(w) != 0x0B) continue; // sltiu only + bool keyed = false; + for (uint32_t wi : out.w_imms) + if (uimm_of(w) == wi) keyed = true; + if (!keyed) continue; + const uint32_t pc = fw.addr + static_cast(i * 4); + bool already = false; + for (const auto& c : out.candidates) + if (c.pc == pc) already = true; + if (already) continue; + out.candidates.push_back( + {"screen_x_sites", pc, fw.addr, 0, fw.uses_gte ? 70 : 40, + fw.uses_gte + ? "W compare in a GTE function with no height compare — " + "outside auto_screen_x's reach" + : "W compare in a non-GTE function; verify before use", + dis1(exe, pc, db)}); + } + } + + std::sort(out.candidates.begin(), out.candidates.end(), + [](const WsCandidate& a, const WsCandidate& b) { + if (a.confidence != b.confidence) return a.confidence > b.confidence; + return a.pc < b.pc; + }); + if (static_cast(out.candidates.size()) > opts.max_candidates) + out.candidates.resize(opts.max_candidates); + std::sort(out.extent_funcs.begin(), out.extent_funcs.end()); + return out; +} + +} // namespace PSXRecomp::Analysis + +namespace PSXRecomp::Analysis { + +namespace { + +std::vector of_kind(const WsScanResult& s, const char* kind, + int min_conf) { + std::vector v; + for (const auto& c : s.candidates) + if (c.kind == kind && c.confidence >= min_conf) v.push_back(&c); + std::sort(v.begin(), v.end(), + [](const WsCandidate* a, const WsCandidate* b) { return a->pc < b->pc; }); + return v; +} + +std::string imm_list(const std::vector& v) { + std::string s; + for (size_t i = 0; i < v.size(); ++i) + s += fmt::format("{}\"0x{:X}\"", i ? ", " : "", v[i]); + return s; +} + +} // namespace + +bool write_ws_sites_toml(const WsScanResult& scan, + const std::filesystem::path& path, + int min_confidence, + std::string& error) { + std::error_code ec; + if (path.has_parent_path()) std::filesystem::create_directories(path.parent_path(), ec); + std::ofstream f(path, std::ios::binary | std::ios::trunc); + if (!f) { error = fmt::format("cannot write {}", path.string()); return false; } + + f << "# [widescreen] candidates from `psxrecomp-analyze --scan-widescreen`.\n"; + f << "# Static analysis only — no runtime, trace, or overlay capture was read.\n"; + f << "#\n"; + f << "# These are CANDIDATES. Every one satisfies the instruction form\n"; + f << "# code_generator.cpp requires, so none can break the build, but only\n"; + f << "# playtesting shows whether a site is the right one to widen.\n"; + f << "# Paste into game.toml and cut what does not help.\n\n"; + + f << "[widescreen.cull]\n"; + f << "auto_screen_x = true"; + if (!scan.extent_funcs.empty()) + f << fmt::format(" # {} function(s) carry the screen-extent signature", + scan.extent_funcs.size()); + f << "\n"; + if (scan.imms_discovered && !(scan.w_canonical && scan.h_canonical)) + f << "# WARNING: these do not match a console display mode — treat as a guess.\n"; + if (scan.imms_discovered) { + f << "# Discovered from the image: width/height compare immediates scored by\n"; + f << "# CO-OCCURRENCE in the same function (the screen-extent signature),\n"; + f << "# weighted toward GTE code and the console's real display modes.\n"; + f << "# Verify before trusting — extra entries only widen what auto_screen_x\n"; + f << "# considers, but a wrong one widens the wrong compare.\n"; + } + f << fmt::format("screen_w_imms = [{}]\n", imm_list(scan.w_imms)); + f << fmt::format("screen_h_imms = [{}]\n", imm_list(scan.h_imms)); + + auto emit = [&](const char* key, const char* note) { + auto v = of_kind(scan, key, min_confidence); + if (v.empty()) return; + f << "\n# " << note << "\n"; + for (const auto* c : v) + f << fmt::format("# 0x{:08X} {:<28} {}\n", c->pc, c->instr, c->evidence); + f << key << " = ["; + for (size_t i = 0; i < v.size(); ++i) + f << fmt::format("{}\"0x{:08X}\"", i ? ", " : "", v[i]->pc); + f << "]\n"; + }; + emit("bias_sites", "Horizontal-margin adds (addi/addiu)."); + emit("range_sites", "X window compares paired with the bias above (sltiu)."); + emit("screen_x_sites", + "W compares outside any screen-extent funnel — auto_screen_x cannot reach these."); + emit("a1_sites", "Repurposable nops before a caller-supplied margin joins an X term."); + return true; +} + +bool write_ws_sites_json(const WsScanResult& scan, + const std::filesystem::path& path, + std::string& error) { + std::error_code ec; + if (path.has_parent_path()) std::filesystem::create_directories(path.parent_path(), ec); + std::ofstream f(path, std::ios::binary | std::ios::trunc); + if (!f) { error = fmt::format("cannot write {}", path.string()); return false; } + + auto esc = [](const std::string& s) { + std::string o; + for (char c : s) { + if (c == '"' || c == '\\') o += '\\'; + o += c; + } + return o; + }; + + f << "{\n \"schema\": \"psxrecomp widescreen sites v1\",\n"; + f << " \"static_only\": true,\n"; + f << fmt::format(" \"imms_discovered\": {},\n", scan.imms_discovered ? "true" : "false"); + f << fmt::format(" \"w_canonical\": {},\n", scan.w_canonical ? "true" : "false"); + f << fmt::format(" \"h_canonical\": {},\n", scan.h_canonical ? "true" : "false"); + f << " \"w_imms\": ["; + for (size_t i = 0; i < scan.w_imms.size(); ++i) f << (i ? "," : "") << scan.w_imms[i]; + f << "],\n \"h_imms\": ["; + for (size_t i = 0; i < scan.h_imms.size(); ++i) f << (i ? "," : "") << scan.h_imms[i]; + f << "],\n \"extent_funcs\": ["; + for (size_t i = 0; i < scan.extent_funcs.size(); ++i) + f << (i ? "," : "") << scan.extent_funcs[i]; + f << "],\n \"candidates\": [\n"; + for (size_t i = 0; i < scan.candidates.size(); ++i) { + const auto& c = scan.candidates[i]; + f << fmt::format(" {{\"kind\": \"{}\", \"pc\": {}, \"func\": {}, " + "\"partner\": {}, \"confidence\": {}, \"instr\": \"{}\", " + "\"evidence\": \"{}\"}}{}\n", + c.kind, c.pc, c.func, c.partner_pc, c.confidence, + esc(c.instr), esc(c.evidence), + i + 1 < scan.candidates.size() ? "," : ""); + } + f << " ]\n}\n"; + return true; +} + +void print_ws_report(const WsScanResult& scan, const AnalysisDb& db, int top_n) { + fmt::print("\n=== widescreen site scan (static) ===\n"); + if (scan.w_imms.empty() || scan.h_imms.empty()) { + fmt::print(" No screen-extent immediates found. This title may not use a\n" + " GTE trivial-reject funnel, or its extents fall outside the\n" + " scanned range — pass --ws-w-imms / --ws-h-imms explicitly.\n\n"); + return; + } + fmt::print(" screen_w_imms = [{}] screen_h_imms = [{}] ({})\n", + [&] { std::string s; for (size_t i=0;i= top_n) { fmt::print(" … {} more\n", v.size() - top_n); break; } + const FunctionRecord* fn = db.find(c->func); + fmt::print(" 0x{:08X} {:>3}% {:<26} in {}\n", c->pc, c->confidence, + c->instr, fn ? fn->name : "?"); + fmt::print(" {}\n", c->evidence); + } + } + fmt::print("\n"); +} + +} // namespace PSXRecomp::Analysis