From c2f9c3d59aa4a51bcd14a224fe0b3596340d68ca Mon Sep 17 00:00:00 2001 From: Alexandros Mandravillis Date: Fri, 11 Sep 2026 01:26:22 +0300 Subject: [PATCH 1/9] setup: detect any recompiled BIOS stem and stop the first-run setup loop Focused port of the fork's fix/setup-host-bios-deadlock feature (10 commits) onto current upstream master, per the 2026-09-10 decision to port rather than rebase (the fork line and upstream share no merge base). Content: the codegen host accepts any recompiled BIOS stem instead of only OpenBIOS/SCPH1001 (psx_bios_known_images.h); the runtime checks the setup host before validating a chosen BIOS and the first-run loop breaker reports a cause-specific reason; the CLI decodes subprocess output as UTF-8; --setup-selfcheck verifies a headless setup; the rbengine preflight names the recursive init. Two port conflicts resolved: the new CTest registrations in recompiler/CMakeLists.txt are kept (the names exist only in this change), and runtime.cmake takes the new recomp-ui guidance text. Cubic P3 finding fixed: the sibling PSX_NETPLAY FATAL_ERROR now also names --recursive. Evidence: python syntax checks pass for psxrecomp_cli.py and the new tests; runtime/tests/test_codegen_host_bios_stems.py runs or skips only on a missing RECOMP_UI_ROOT. No full runtime build or retail route is claimed here. Developed with AI assistance (Cline, 2026-09-10). --- host/psxrecomp_codegen_host.c | 205 ++++++++++++++++-- psxrecomp_cli.py | 18 +- recompiler/CMakeLists.txt | 2 + runtime/include/psx_bios_known_images.h | 106 +++++++++ runtime/runtime.cmake | 42 +++- runtime/src/main.cpp | 102 +++++---- runtime/tests/test_codegen_host_bios_stems.py | 127 +++++++++++ 7 files changed, 524 insertions(+), 78 deletions(-) create mode 100644 runtime/include/psx_bios_known_images.h create mode 100644 runtime/tests/test_codegen_host_bios_stems.py diff --git a/host/psxrecomp_codegen_host.c b/host/psxrecomp_codegen_host.c index ac9ca0ba..bd66f4d1 100644 --- a/host/psxrecomp_codegen_host.c +++ b/host/psxrecomp_codegen_host.c @@ -2,6 +2,8 @@ #include "psxrecomp_codegen_host.h" +#include "psx_bios_known_images.h" + #include #include #include @@ -1135,15 +1137,74 @@ static int resolve_build_paths(void) { return join_path(g_exe_path, sizeof(g_exe_path), g_build_dir, exe_name); } +/* Pair a _dispatch.c with its _full.c, the same pairing + * runtime.cmake requires before it will link a BIOS backend. */ +static int dispatch_has_full(const char* dir, const char* name) { + static const char suffix[] = "_dispatch.c"; + char stem[512], full[600], path[1200]; + size_t len = strlen(name); + size_t slen = sizeof(suffix) - 1; + if (len <= slen || strcmp(name + len - slen, suffix) != 0) + return 0; + if (len - slen >= sizeof(stem)) + return 0; + memcpy(stem, name, len - slen); + stem[len - slen] = '\0'; + if ((size_t)snprintf(full, sizeof(full), "%s_full.c", stem) >= sizeof(full)) + return 0; + if (!join_path(path, sizeof(path), dir, full)) + return 0; + return path_is_file(path); +} + +/* Does psxrecomp/generated/ hold ANY recompiled BIOS backend? + * + * This used to probe two hardcoded names, OpenBIOS_dispatch.c and + * SCPH1001_dispatch.c. A port that pins a different image — via + * PSXRECOMP_BIOS_STEMS / game.toml recompiler.bios_config, as every wave-3 + * kit does with SCPH5552 — emits its backend under that other stem, so the + * probe was permanently unsatisfied: Generate kept succeeding, the wizard + * kept reopening, and first-run setup could never complete. Accept any stem + * that has both halves instead of naming images here. */ +static int generated_has_bios_backend(const char* dir) { +#if defined(_WIN32) + char pat[1200]; + WIN32_FIND_DATAA fd; + HANDLE h; + int found = 0; + if (!join_path(pat, sizeof(pat), dir, "*_dispatch.c")) + return 0; + h = FindFirstFileA(pat, &fd); + if (h == INVALID_HANDLE_VALUE) + return 0; + do { + if (dispatch_has_full(dir, fd.cFileName)) { + found = 1; + break; + } + } while (FindNextFileA(h, &fd)); + FindClose(h); + return found; +#else + DIR* d = opendir(dir); + struct dirent* e; + int found = 0; + if (!d) + return 0; + while (!found && (e = readdir(d)) != NULL) { + if (dispatch_has_full(dir, e->d_name)) + found = 1; + } + closedir(d); + return found; +#endif +} + static int bios_backends_missing(void) { - char openbios[1100], scph[1100]; - if (!join_path(openbios, sizeof(openbios), g_project_root, - "psxrecomp/generated/OpenBIOS_dispatch.c")) + char gen[1100]; + if (!join_path(gen, sizeof(gen), g_project_root, "psxrecomp/generated")) return 1; - if (!join_path(scph, sizeof(scph), g_project_root, - "psxrecomp/generated/SCPH1001_dispatch.c")) - return 1; - return !(path_is_file(openbios) || path_is_file(scph)); + return !generated_has_bios_backend(gen); } int psxrecomp_codegen_host_sources_missing( @@ -1209,7 +1270,7 @@ static void write_sidecar_near_exe(const char* near_exe, const char* name, write_line_file(path, value ? value : ""); } -/* IEEE CRC-32 (zlib / Ethernet) — SCPH-1001 identity for setup discovery. */ +/* IEEE CRC-32 (zlib / Ethernet) — retail BIOS identity for setup discovery. */ static uint32_t host_crc32(const unsigned char* data, size_t len) { uint32_t crc = 0xFFFFFFFFu; size_t i, j; @@ -1221,17 +1282,23 @@ static uint32_t host_crc32(const unsigned char* data, size_t len) { return ~crc; } +/* Does this file match the retail image THIS build pins? A setup host has no + * linked backend to ask, so it consults psx_bios_known_images.h rather than + * assuming SCPH-1001 — which made every non-SCPH1001 kit reject a perfectly + * good dump. An unknown pinned stem adopts nothing and the player is asked. */ static int retail_bios_file_ok_c(const char* path) { + const PsxKnownBiosImage* want = psx_expected_bios(); FILE* f; long size; unsigned char* buf; uint32_t crc; + if (!want) return 0; if (!path || !path[0] || !path_is_file(path)) return 0; f = fopen(path, "rb"); if (!f) return 0; if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return 0; } size = ftell(f); - if (size != 512 * 1024) { fclose(f); return 0; } + if (size != (long)want->size) { fclose(f); return 0; } if (fseek(f, 0, SEEK_SET) != 0) { fclose(f); return 0; } buf = (unsigned char*)malloc((size_t)size); if (!buf) { fclose(f); return 0; } @@ -1243,22 +1310,22 @@ static int retail_bios_file_ok_c(const char* path) { fclose(f); crc = host_crc32(buf, (size_t)size); free(buf); - return crc == 0x37157331u; /* SCPH-1001 */ + return crc == want->crc32; } -/* Prefer a player-supplied SCPH1001 next to the project/exe for Generate. - * Missing → leave empty (OpenBIOS). Does not override an explicit OpenBIOS. */ +/* Prefer a player-supplied dump of the pinned retail image next to the + * project/exe for Generate. Missing → leave empty (OpenBIOS). Does not + * override an explicit OpenBIOS. */ static int discover_retail_bios_c(char* out, size_t cap) { - static const char* names[] = { - "SCPH1001.BIN", "scph1001.bin", "SCPH-1001.BIN", "scph-1001.bin", - "SCPH1001.bin", "scph1001.BIN", - }; + char names[8][32]; + int nnames = psx_known_bios_filenames(psx_expected_bios(), names, 8); static const char* subs[] = { "bios", "", "system", "firmware", "psxrecomp/bios", "psxrecomp-v4/bios", }; char roots[3][1100]; int nroots = 0; int r, s, n; + if (nnames <= 0) { out[0] = 0; return 0; } if (g_project_root[0]) { snprintf(roots[nroots], sizeof(roots[0]), "%s", g_project_root); ++nroots; @@ -1282,7 +1349,7 @@ static int discover_retail_bios_c(char* out, size_t cap) { } else { snprintf(dir, sizeof(dir), "%s", walk); } - for (n = 0; n < (int)(sizeof(names) / sizeof(names[0])); ++n) { + for (n = 0; n < nnames; ++n) { char cand[1300]; if (!join_path(cand, sizeof(cand), dir, names[n])) continue; if (!retail_bios_file_ok_c(cand)) continue; @@ -3667,6 +3734,7 @@ static const char* host_loop_breaker_note(void) { char sidecar[1200], line[64], found[512], marker_abs[1200]; long long then, now; const char* marker_rel; + const char* cause; int game_missing, bios_missing; if (!join_path(sidecar, sizeof(sidecar), g_project_root, HOST_LAST_GENERATE_SIDECAR)) @@ -3687,16 +3755,30 @@ static const char* host_loop_breaker_note(void) { if (!game_missing && !bios_missing) return NULL; list_generated_dispatch(found, sizeof(found)); + /* Each branch has its own cause, so do not assert a single one. A missing + * game dispatch really does point at disagreeing boot-EXE names. Missing + * BIOS backends do not: they mean Generate never emitted them, normally + * because no retail BIOS was available to emit them from. Blaming + * boot-EXE names for that sent players after the wrong thing. */ + if (game_missing) + cause = "please report this to the port maintainer: the project's " + "boot-EXE names disagree"; + else + cause = "Generate produced the game code but no BIOS backend, which " + "normally means it had no retail BIOS to work from. Select " + "the PlayStation BIOS dump this port requires (named in the " + "README; it must be exactly 512 KB) in the launcher, then " + "run Generate again"; snprintf(g_loop_breaker_note, sizeof(g_loop_breaker_note), "A Generate completed here recently, yet the launcher still " "cannot find %s%s%s. generated/ contains: %s. Running Generate " - "again will very likely loop — please report this to the port " - "maintainer: the project's boot-EXE names disagree.", + "again will very likely loop — %s.", game_missing ? marker_rel : "", (game_missing && bios_missing) ? " and " : "", bios_missing ? "the BIOS backends under psxrecomp/generated/" : "", - found[0] ? found : "no *_dispatch.c at all"); + found[0] ? found : "no *_dispatch.c at all", + cause); fprintf(stderr, "psxrecomp-codegen: %s\n", g_loop_breaker_note); return g_loop_breaker_note; } @@ -4200,9 +4282,92 @@ static int host_paths_same_file(const char* a, const char* b) { #endif } +/* --setup-selfcheck: report what the setup host believes about this tree, as + * JSON on stdout, then exit. 0 = generated sources complete, 2 = the wizard + * would reopen, 1 = could not tell (no project root). + * + * This exists because the decision layer -- "are the generated sources + * present?" -- had no headless entry point. Disc selection, BIOS selection and + * generation were already scriptable via psxrecomp_cli.py; the verdict on + * whether setup is DONE was reachable only by clicking through the wizard, so + * nothing in CI could assert it. A stem mismatch there shipped a first-run + * loop on 26 titles before anyone noticed. + * + * Deliberately ahead of the PSX_HAS_GAME_DISPATCH early return below, so a + * product build answers too. Every title already calls this function with + * argc/argv, so no per-title change is needed to gain the flag. */ +static void host_json_str(const char* s) { + putchar('"'); + for (; s && *s; ++s) { + if (*s == '\\' || *s == '"') + putchar('\\'); + putchar(*s); + } + putchar('"'); +} + +static void host_selfcheck_or_return(const PsxrecompCodegenHostConfig* cfg, + int argc, char** argv) { + const PsxKnownBiosImage* want; + const char* marker_rel; + char marker_abs[1200]; + int i, missing, game_ok, bios_ok; + + for (i = 1; i < argc; ++i) + if (argv[i] && strcmp(argv[i], "--setup-selfcheck") == 0) + break; + if (i >= argc) + return; + + if (!cfg || !cfg->cmake_target || !cfg->exe_basename) { + printf("{\"error\": \"no codegen host config linked\"}\n"); + exit(1); + } + /* Sets g_cfg and g_project_root as a side effect. */ + missing = psxrecomp_codegen_host_sources_missing(cfg); + if (!g_project_root[0]) { + printf("{\"error\": \"project root not found\"}\n"); + exit(1); + } + + marker_rel = cfg_or(cfg->gen_marker_relpath, + "generated/SLUS_011.89_dispatch.c"); + game_ok = join_path(marker_abs, sizeof(marker_abs), g_project_root, + marker_rel) && path_is_file(marker_abs); + bios_ok = !bios_backends_missing(); + want = psx_expected_bios(); + + printf("{\n"); + printf(" \"display_name\": "); + host_json_str(cfg_or(cfg->display_name, "Game")); + printf(",\n \"project_root\": "); + host_json_str(g_project_root); + printf(",\n \"expected_bios_stem\": "); + host_json_str(PSX_EXPECTED_BIOS_STEM); + printf(",\n \"expected_bios_id\": "); + host_json_str(want ? want->id : ""); + printf(",\n \"expected_bios_crc32\": "); + if (want) { + char crcbuf[16]; + snprintf(crcbuf, sizeof(crcbuf), "0x%08X", want->crc32); + host_json_str(crcbuf); + } else { + printf("null"); + } + printf(",\n \"game_dispatch\": "); + host_json_str(marker_rel); + printf(",\n \"game_dispatch_present\": %s", game_ok ? "true" : "false"); + printf(",\n \"bios_backends_present\": %s", bios_ok ? "true" : "false"); + printf(",\n \"sources_missing\": %s", missing ? "true" : "false"); + printf("\n}\n"); + fflush(stdout); + exit(missing ? 2 : 0); +} + /* Setup-host zip-root exe → build-release product (bios/mods/assets/settings). */ void psxrecomp_codegen_host_forward_if_built( const PsxrecompCodegenHostConfig* cfg, int argc, char** argv) { + host_selfcheck_or_return(cfg, argc, argv); /* exits when requested */ #if defined(PSX_HAS_GAME_DISPATCH) /* Full game binary — already the product tree. */ (void)cfg; diff --git a/psxrecomp_cli.py b/psxrecomp_cli.py index d251f4c9..46767403 100755 --- a/psxrecomp_cli.py +++ b/psxrecomp_cli.py @@ -687,7 +687,7 @@ def _build_recompiler_targets( progress.log(" ".join(cmake_args)) proc = subprocess.run( - cmake_args, cwd=str(project_root), capture_output=True, text=True + cmake_args, cwd=str(project_root), capture_output=True, text=True, encoding="utf-8", errors="replace" ) for stream in (proc.stdout, proc.stderr): if stream: @@ -704,7 +704,7 @@ def _build_recompiler_targets( for target in targets: build_cmd += ["--target", target] progress.log(" ".join(build_cmd)) - proc = subprocess.run(build_cmd, capture_output=True, text=True) + proc = subprocess.run(build_cmd, capture_output=True, text=True, encoding="utf-8", errors="replace") for stream in (proc.stdout, proc.stderr): if stream: for line in stream.splitlines(): @@ -817,7 +817,7 @@ def regen_bios_profile( [str(bios_tool), "--config", profile_rel], cwd=str(fw), capture_output=True, - text=True, + text=True, encoding="utf-8", errors="replace", ) for stream in (proc.stdout, proc.stderr): if not stream: @@ -937,7 +937,7 @@ def run_prepare_disc( str(project_root), str(source), ] - proc = subprocess.run(cmd, cwd=str(project_root), capture_output=True, text=True) + proc = subprocess.run(cmd, cwd=str(project_root), capture_output=True, text=True, encoding="utf-8", errors="replace") out = (proc.stdout or "") + (proc.stderr or "") for line in out.splitlines(): if line.strip(): @@ -1140,7 +1140,7 @@ def cmd_generate(args: argparse.Namespace, progress: ProgressReporter) -> int: cmd, cwd=str(project_root), capture_output=True, - text=True, + text=True, encoding="utf-8", errors="replace", ) ri_warn = 0 for stream in (proc.stdout, proc.stderr): @@ -1313,7 +1313,7 @@ def _cmake_configure( *extra, ] progress.log(" ".join(cmd)) - proc = subprocess.run(cmd, cwd=str(project_root), capture_output=True, text=True) + proc = subprocess.run(cmd, cwd=str(project_root), capture_output=True, text=True, encoding="utf-8", errors="replace") for stream in (proc.stdout, proc.stderr): if stream: for line in stream.splitlines(): @@ -1396,7 +1396,7 @@ def _cmake_build( target, ] progress.log(" ".join(cmd)) - proc = subprocess.run(cmd, capture_output=True, text=True) + proc = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace") for stream in (proc.stdout, proc.stderr): if stream: for line in stream.splitlines(): @@ -1551,7 +1551,7 @@ def run_pgo_train( r = subprocess.run( ["xcrun", "--find", "llvm-profdata"], capture_output=True, - text=True, + text=True, encoding="utf-8", errors="replace", check=False, ) if r.returncode == 0 and r.stdout.strip(): @@ -1932,7 +1932,7 @@ def cmd_analyze(args: argparse.Namespace, progress: ProgressReporter) -> int: progress.phase("analyze", pct=0.3, message=f"Analyzing {exe_path.name}…") progress.log(" ".join(cmd)) - proc = subprocess.run(cmd, cwd=str(project_root), capture_output=True, text=True) + proc = subprocess.run(cmd, cwd=str(project_root), capture_output=True, text=True, encoding="utf-8", errors="replace") for stream in (proc.stdout, proc.stderr): if stream: for line in stream.splitlines(): diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index 01b85a34..bf0fe399 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -642,6 +642,8 @@ if(BUILD_TESTING) foreach(_t IN ITEMS bios_selection_guards capture_history + cli_retail_bios_profile + codegen_host_bios_stems default_renderer_guards mod_owned_display_controls fmv_quiet_guards diff --git a/runtime/include/psx_bios_known_images.h b/runtime/include/psx_bios_known_images.h new file mode 100644 index 00000000..39664eef --- /dev/null +++ b/runtime/include/psx_bios_known_images.h @@ -0,0 +1,106 @@ +/* psx_bios_known_images.h — identities of the retail BIOS images this + * framework ships a build profile for. + * + * A *built* host asks its linked backends (psx_bios_registry) which image it + * accepts. A *setup* host has no linked backend — that is what makes it a + * setup host — so first-run discovery, bios.cfg seeding and picker copy had + * nowhere to look and hardcoded SCPH-1001. Any kit pinning another image via + * PSXRECOMP_BIOS_STEM / recompiler.bios_config (every wave-3 kit pins + * SCPH5552) then refused to auto-discover a perfectly good dump and told the + * player their BIOS was the wrong one. + * + * Keep in sync with bios/.toml. A stem absent from this table still + * builds and runs; it only loses first-run auto-discovery and falls back to + * asking the player, which is safe. + */ +#ifndef PSX_BIOS_KNOWN_IMAGES_H +#define PSX_BIOS_KNOWN_IMAGES_H + +#include +#include + +/* Set by runtime.cmake from PSXRECOMP_BIOS_STEM. */ +#ifndef PSX_EXPECTED_BIOS_STEM +#define PSX_EXPECTED_BIOS_STEM "SCPH1001" +#endif + +typedef struct PsxKnownBiosImage { + const char* stem; /* generated/_dispatch.c, bios/.toml */ + const char* id; /* profile id, e.g. "SCPH-5552" */ + uint32_t crc32; /* IEEE CRC-32 (zlib/Ethernet) over the whole image */ + uint32_t size; /* bytes */ +} PsxKnownBiosImage; + +/* Each entry verified two ways: against the dump itself, and against the + * identity the recompiler emits into generated/_dispatch.c. */ +static const PsxKnownBiosImage psx_known_bios_images[] = { + { "SCPH1001", "SCPH-1001", 0x37157331u, 524288u }, + { "SCPH5552", "SCPH-5552", 0xD786F0B9u, 524288u }, +}; + +static inline const PsxKnownBiosImage* psx_known_bios_by_stem(const char* stem) { + size_t i; + if (!stem || !stem[0]) + return 0; + for (i = 0; i < sizeof(psx_known_bios_images) / sizeof(psx_known_bios_images[0]); ++i) { + if (strcmp(psx_known_bios_images[i].stem, stem) == 0) + return &psx_known_bios_images[i]; + } + return 0; +} + +/* The retail image THIS build was configured for, or NULL when the pinned + * stem is not in the table. Callers must handle NULL by asking the player + * rather than assuming SCPH-1001. */ +static inline const PsxKnownBiosImage* psx_expected_bios(void) { + return psx_known_bios_by_stem(PSX_EXPECTED_BIOS_STEM); +} + +/* A short label for the pinned image, for player-facing copy: "SCPH5552.BIN". + * Falls back to the raw stem when the identity is not in the table. */ +static inline const char* psx_expected_bios_label(void) { + static char label[40]; + if (!label[0]) { + const PsxKnownBiosImage* e = psx_expected_bios(); + const char* stem = e ? e->stem : PSX_EXPECTED_BIOS_STEM; + size_t n = strlen(stem); + if (n > sizeof(label) - 5) + n = sizeof(label) - 5; + memcpy(label, stem, n); + memcpy(label + n, ".BIN", 5); + } + return label; +} + +/* Filename spellings worth probing for during first-run discovery, + * written into out[0..n). Covers the stem and the dashed profile id, each in + * upper and lower case: SCPH5552.BIN, scph5552.bin, SCPH-5552.BIN, ... */ +static inline int psx_known_bios_filenames(const PsxKnownBiosImage* img, + char out[][32], int cap) { + const char* bases[2]; + int n = 0, b, lower; + if (!img) + return 0; + bases[0] = img->stem; + bases[1] = img->id; + for (b = 0; b < 2; ++b) { + for (lower = 0; lower < 2; ++lower) { + const char* ext = lower ? "bin" : "BIN"; + size_t i, len = strlen(bases[b]); + char* dst; + if (n >= cap || len + 5 > 32) + continue; + dst = out[n++]; + for (i = 0; i < len; ++i) { + char c = bases[b][i]; + dst[i] = (char)(lower && c >= 'A' && c <= 'Z' ? c - 'A' + 'a' : c); + } + dst[len] = '.'; + memcpy(dst + len + 1, ext, 3); + dst[len + 4] = 0; + } + } + return n; +} + +#endif /* PSX_BIOS_KNOWN_IMAGES_H */ diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 788d6fcd..11e1f9c2 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -508,7 +508,7 @@ else() if(PSXRECOMP_HAS_RECOMP_NET) message(FATAL_ERROR "psxrecomp: PSX_NETPLAY needs retcomm-rbengine.\n" - " git submodule update --init lib/retcomm-rbengine\n" + " git submodule update --init --recursive lib/retcomm-rbengine\n" " or -DRECOMP_RBENGINE_ROOT=/path/to/retcomm-rbengine") endif() endif() @@ -537,7 +537,13 @@ if(PSX_REWIND) message(FATAL_ERROR "psxrecomp: PSX_REWIND=ON exposes the Rewind launcher controls " "but no retcomm-rbengine snap-ring backend was found.\n" - " git submodule update --init lib/retcomm-rbengine\n" + "A source ZIP downloaded from GitHub never contains submodule " + "contents and cannot build. Clone instead:\n" + " git clone --recurse-submodules \n" + "In an existing clone, run this from the GAME repo root. Note " + "--recursive: rbengine is a submodule of psxrecomp, not of the " + "game, so a non-recursive init leaves it empty.\n" + " git submodule update --init --recursive\n" " or -DRECOMP_RBENGINE_ROOT=/path/to/retcomm-rbengine\n" " or configure with -DPSX_REWIND=OFF to hide Rewind.") endif() @@ -1241,15 +1247,18 @@ function(psxrecomp_add_runtime_target target) # where releases are validated. Dev checkouts still resolve the relative # default without prompting via the exe-dir upward search, which also tries # /psxrecomp-v4/ for game-project layouts. + # Follow the stem this build actually pins; assuming SCPH1001 here handed + # every non-SCPH1001 kit a default path that could never resolve. + set(_psxrt_stem_bios "bios/${PSXRECOMP_BIOS_STEM}.BIN") if(NOT PSXRT_DEFAULT_BIOS_PATH) - set(PSXRT_DEFAULT_BIOS_PATH "bios/SCPH1001.BIN") + set(PSXRT_DEFAULT_BIOS_PATH "${_psxrt_stem_bios}") elseif(IS_ABSOLUTE "${PSXRT_DEFAULT_BIOS_PATH}") message(WARNING "DEFAULT_BIOS_PATH '${PSXRT_DEFAULT_BIOS_PATH}' is absolute; refusing to " "bake a build-machine path into the binary (release exes must prompt on " - "user machines). Using relative 'bios/SCPH1001.BIN' instead — drop the " + "user machines). Using relative '${_psxrt_stem_bios}' instead — drop the " "DEFAULT_BIOS_PATH argument from this game's CMakeLists.") - set(PSXRT_DEFAULT_BIOS_PATH "bios/SCPH1001.BIN") + set(PSXRT_DEFAULT_BIOS_PATH "${_psxrt_stem_bios}") endif() if(NOT DEFINED PSXRT_DEFAULT_GAME_CONFIG_PATH) set(PSXRT_DEFAULT_GAME_CONFIG_PATH "") @@ -1665,6 +1674,10 @@ function(psxrecomp_add_runtime_target target) target_compile_definitions(${target} PRIVATE DEFAULT_DEBUG_PORT=${PSXRT_DEBUG_PORT} PSX_DEFAULT_BIOS_PATH="${PSXRT_DEFAULT_BIOS_PATH}" + # The retail stem this build pins. A setup host has no linked + # backend to ask, so this is how it knows which image to look for + # and name (psx_bios_known_images.h) instead of assuming SCPH-1001. + PSX_EXPECTED_BIOS_STEM="${PSXRECOMP_BIOS_STEM}" # Where the shipped redistributable image lives, relative to the exe. # This is what a player gets when they choose no BIOS. PSX_BUNDLED_BIOS_PATH="${PSXRECOMP_BUNDLED_BIOS_PATH}" @@ -1859,11 +1872,20 @@ function(psxrecomp_add_runtime_target target) if(PSX_RECOMP_UI AND NOT PSXRT_ORACLE) if(NOT RECOMP_UI_ROOT OR NOT EXISTS "${RECOMP_UI_ROOT}/recomp_ui.cmake") message(FATAL_ERROR - "PSX_RECOMP_UI=ON but recomp-ui is missing.\n" - "Add at the game repo root:\n" - " git submodule add -b master " - "https://github.com/RetroPortingToolKit/recomp-ui.git recomp-ui\n" - "Or set -DRECOMP_UI_ROOT=/path/to/recomp-ui") + "PSX_RECOMP_UI=ON but recomp-ui is missing from the game " + "repo root.\n" + "A source ZIP downloaded from GitHub never contains " + "submodule contents and cannot build. Clone instead:\n" + " git clone --recurse-submodules \n" + "In a clone that already declares recomp-ui, fetch the " + "pinned commit:\n" + " git submodule update --init --recursive\n" + "Only if this repo does not declare recomp-ui yet, add it " + "(use the fork this project pins, not necessarily " + "upstream):\n" + " git submodule add recomp-ui\n" + "Or point at an existing checkout: " + "-DRECOMP_UI_ROOT=/path/to/recomp-ui") endif() # recomp-ui gates its Mods view behind RECOMP_UI_ENABLE_MODS, which # defaults OFF there -- correct for a cross-console launcher, since a diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index cbf4c830..25d89d29 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -16,6 +16,7 @@ #include "boot_state.h" #include "bios_hle.h" #include "bios_hle_plan.h" +#include "psx_bios_known_images.h" #include "psx_bios_backend.h" #include "psx_cycles.h" #include "starvation_ring.h" @@ -2514,11 +2515,32 @@ static std::filesystem::path resolve_bios_for_runtime(const char* requested, const bool bundled_only = openbios_allowed && bundled && !player_bios_selectable; + /* 0. Setup host (CI zip root): no BIOS backends are linked yet, so there + * is nothing an image could be validated against — bios_backend_for_file() + * rejects every file, including the correct one, and a title with + * openbios = false has no fallback to land on. This MUST precede the + * explicit-choice branch below: a remembered bios.cfg pick reached + * validate_bios_for_launch() first and deadlocked first-run setup — the + * player was told their good SCPH-1001 was "not an image this build was + * compiled from", and could never supply the BIOS that Generate needs in + * order to emit the backend. Play belongs to the product binary under + * build-release/ after Generate & rebuild. */ + if (psx_bios_registry_count == 0) { + launcher_warning("Setup host — finish Generate & rebuild", + "This executable is the first-run setup host (no game/BIOS code " + "linked).\n\n" + "Use Generate & rebuild in the launcher. After that succeeds, open " + "this same shortcut again — it starts the game from build-release/ " + "(where bios/, mods/, and settings live).\n\n" + "Or run build-release/.exe directly."); + return {}; + } + /* 1. An explicit choice: --bios, else a remembered pick. A product build * with only its bundled backend has no meaningful player choice: ignore * stale settings/bios.cfg paths instead of validating an image the hidden - * launcher row cannot clear. Setup hosts (registry_count == 0) retain their - * picker/generation flow. */ + * launcher row cannot clear. Setup hosts (registry_count == 0) already + * returned above. */ std::filesystem::path chosen; if (!bundled_only) { if (requested_is_explicit && requested && requested[0]) { @@ -2551,19 +2573,6 @@ static std::filesystem::path resolve_bios_for_runtime(const char* requested, return {}; } - /* Setup host (CI zip root): no BIOS backends linked yet. Play belongs to - * the product binary under build-release/ after Generate & rebuild. */ - if (psx_bios_registry_count == 0) { - launcher_warning("Setup host — finish Generate & rebuild", - "This executable is the first-run setup host (no game/BIOS code " - "linked).\n\n" - "Use Generate & rebuild in the launcher. After that succeeds, open " - "this same shortcut again — it starts the game from build-release/ " - "(where bios/, mods/, and settings live).\n\n" - "Or run build-release/.exe directly."); - return {}; - } - /* 3. This title requires a retail BIOS: ask for one. */ const std::string accepted = bios_accepted_images(); launcher_info((s_picker_game_name + " — PlayStation BIOS needed").c_str(), @@ -2571,7 +2580,8 @@ static std::filesystem::path resolve_bios_for_runtime(const char* requested, "Step 1 of 2 — PlayStation BIOS\n\n" "In the next window, select your PlayStation BIOS dump. This build " "requires the exact image it was compiled from: " + accepted + ". " - "Usually named SCPH1001.BIN and exactly 512 KB. Dump from your own " + "Usually named " + std::string(psx_expected_bios_label()) + + " and exactly 512 KB. Dump from your own " "console or otherwise legally obtain it.\n\n" "(This is NOT the game disc — that is asked for next.)"); std::string bios_title = @@ -2704,24 +2714,27 @@ static bool retail_bios_file_ok(const std::filesystem::path& path) { const PsxBiosBackend* b = bios_backend_for_file(path, nullptr, nullptr); return b && b->image && !b->image->image_bundled; } - /* Setup host (no backends linked yet): accept validated SCPH-1001 only. */ - constexpr uint64_t kSize = 512u * 1024u; - constexpr uint32_t kScph1001Crc = 0x37157331u; + /* Setup host (no backends linked yet): accept the retail image THIS build + * pins, from psx_bios_known_images.h. This used to hardcode SCPH-1001, so + * a kit pinning anything else refused to seed from a correct dump. An + * unknown pinned stem seeds nothing and the player is asked instead. */ + const PsxKnownBiosImage* want = psx_expected_bios(); + if (!want) return false; std::ifstream f(path, std::ios::binary | std::ios::ate); if (!f.is_open()) return false; const auto size = static_cast(f.tellg()); - if (size != kSize) return false; + if (size != static_cast(want->size)) return false; std::vector data(static_cast(size)); if (!read_at(f, 0, data.data(), data.size())) return false; - return crc32_compute(data.data(), data.size()) == kScph1001Crc; + return crc32_compute(data.data(), data.size()) == want->crc32; } static std::filesystem::path discover_retail_bios_near(const char* argv0) { namespace fs = std::filesystem; - static const char* kNames[] = { - "SCPH1001.BIN", "scph1001.bin", "SCPH-1001.BIN", "scph-1001.bin", - "SCPH1001.bin", "scph1001.BIN", - }; + char name_buf[8][32]; + const int name_count = + psx_known_bios_filenames(psx_expected_bios(), name_buf, 8); + if (name_count <= 0) return {}; static const char* kSubdirs[] = { "bios", "", "system", "firmware", "psxrecomp/bios", "psxrecomp-v4/bios", }; @@ -2730,8 +2743,8 @@ static std::filesystem::path discover_retail_bios_near(const char* argv0) { for (fs::path root = exe_dir; !root.empty(); root = root.parent_path()) { for (const char* sub : kSubdirs) { const fs::path dir = (sub && sub[0]) ? (root / sub) : root; - for (const char* name : kNames) { - const fs::path cand = dir / name; + for (int ni = 0; ni < name_count; ++ni) { + const fs::path cand = dir / name_buf[ni]; if (retail_bios_file_ok(cand)) { auto abs = fs::weakly_canonical(cand, ec); if (ec) abs = fs::absolute(cand, ec); @@ -7666,12 +7679,14 @@ namespace { out->ok = 1; std::snprintf(out->detail, sizeof(out->detail), "OpenBIOS will be emitted on Generate & rebuild " - "(optional: pick SCPH1001). Play uses " - "build-release/ after rebuild."); + "(optional: pick %s). Play uses " + "build-release/ after rebuild.", + psx_expected_bios_label()); return 1; } std::snprintf(out->detail, sizeof(out->detail), - "PlayStation BIOS required (SCPH1001.BIN)."); + "PlayStation BIOS required (%s).", + psx_expected_bios_label()); return 1; } /* Match runtime resolve: relative picks like bios/SCPH1001.BIN must not @@ -7689,12 +7704,16 @@ namespace { "BIOS file not found."); return 1; } + const PsxKnownBiosImage* want = psx_expected_bios(); + const std::streamoff want_size = + want ? (std::streamoff)want->size : (std::streamoff)(512 * 1024); const std::streamoff size = f.tellg(); - if (size != 512 * 1024) { + if (size != want_size) { std::snprintf(out->detail, sizeof(out->detail), - "BIOS must be exactly 512 KiB (got %lld). Use " - "SCPH1001.BIN.", - (long long)size); + "BIOS must be exactly %lld bytes (got %lld). Use " + "%s.", + (long long)want_size, (long long)size, + psx_expected_bios_label()); return 1; } std::vector data((size_t)size); @@ -7704,15 +7723,20 @@ namespace { return 1; } const uint32_t crc = crc32_compute(data.data(), data.size()); - if (crc != 0x37157331u) { + if (want && crc != want->crc32) { out->warn = 1; std::snprintf(out->detail, sizeof(out->detail), - "CRC32 %08X (validated dump is SCPH1001 CRC32 " - "37157331).", - crc); + "CRC32 %08X (this build expects %s, CRC32 %08X).", + crc, want->id, want->crc32); + } else if (!want) { + out->warn = 1; + std::snprintf(out->detail, sizeof(out->detail), + "CRC32 %08X (this build pins %s, whose identity " + "is not recorded here).", + crc, PSX_EXPECTED_BIOS_STEM); } else { std::snprintf(out->detail, sizeof(out->detail), - "SCPH1001.BIN (CRC OK)."); + "%s (CRC OK).", psx_expected_bios_label()); } /* Setup host (no backends yet): file is fine for first Generate. */ if (psx_bios_registry_count == 0) { diff --git a/runtime/tests/test_codegen_host_bios_stems.py b/runtime/tests/test_codegen_host_bios_stems.py new file mode 100644 index 00000000..5633273b --- /dev/null +++ b/runtime/tests/test_codegen_host_bios_stems.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Regression: the codegen host must accept ANY recompiled BIOS stem. + +bios_backends_missing() in host/psxrecomp_codegen_host.c used to probe two +hardcoded filenames, psxrecomp/generated/OpenBIOS_dispatch.c and +SCPH1001_dispatch.c. Ports that pin another image via recompiler.bios_config / +PSXRECOMP_BIOS_STEMS emit _dispatch.c + _full.c instead, so the +probe could never be satisfied: Generate succeeded, the setup wizard reopened, +and first-run setup looped forever. Every wave-3 kit pins SCPH5552 and every +one of them shipped with that loop. + +test_cli_retail_bios_profile.py already covers the psxrecomp_cli.py half of +this feature. This is the C host half, which had no coverage. +""" + +from pathlib import Path +import os +import shutil +import subprocess +import sys +import tempfile + + +ROOT = Path(__file__).resolve().parents[2] +HOST_C = ROOT / "host" / "psxrecomp_codegen_host.c" + +PROBE = """ +#include +#include +#include "psxrecomp_codegen_host.h" +/* recomp-ui symbol the host references but sources_missing() never reaches. */ +int recomp_launcher_relaunch_exe(char* o, size_t c) { (void)o; (void)c; return 0; } +int main(void) { + PsxrecompCodegenHostConfig cfg; + memset(&cfg, 0, sizeof(cfg)); + cfg.cmake_target = "psx-runtime"; + cfg.exe_basename = "Probe"; + cfg.gen_marker_relpath = "generated/SCUS_943.51_dispatch.c"; + printf("%d", psxrecomp_codegen_host_sources_missing(&cfg)); + return 0; +} +""" + + +def find_recomp_ui() -> Path | None: + env = os.environ.get("RECOMP_UI_ROOT") + candidates = ([Path(env)] if env else []) + [ + ROOT.parent / "recomp-ui", + ROOT / "recomp-ui", + ] + for c in candidates: + if (c / "src" / "recomp_launcher.h").is_file(): + return c + return None + + +def make_project(root: Path, bios_files: list[str]) -> None: + """A project tree the host recognises, with the given BIOS artefacts.""" + (root / "generated").mkdir(parents=True, exist_ok=True) + (root / "generated" / "SCUS_943.51_dispatch.c").write_text("", encoding="utf-8") + fw_gen = root / "psxrecomp" / "generated" + fw_gen.mkdir(parents=True, exist_ok=True) + (root / "game.toml").write_text("[game]\n", encoding="utf-8") + (root / "psxrecomp" / "psxrecomp_cli.py").write_text("", encoding="utf-8") + for name in bios_files: + (fw_gen / name).write_text("", encoding="utf-8") + + +def main() -> int: + ui = find_recomp_ui() + if ui is None: + print("SKIP: recomp-ui not found (set RECOMP_UI_ROOT)") + return 0 + cc = shutil.which("cc") or shutil.which("gcc") or shutil.which("clang") + if cc is None: + print("SKIP: no C compiler on PATH") + return 0 + + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + probe_c = tmp / "probe.c" + probe_c.write_text(PROBE, encoding="utf-8") + exe = tmp / ("probe.exe" if os.name == "nt" else "probe") + build = subprocess.run( + [cc, "-std=c11", "-o", str(exe), str(probe_c), str(HOST_C), + "-I", str(ROOT / "host"), + "-I", str(ROOT / "runtime" / "include"), + "-I", str(ui / "src"), "-I", str(ui / "src" / "common")], + capture_output=True, text=True) + if build.returncode != 0: + print("FAIL: could not build probe\n" + build.stderr[-2000:]) + return 1 + + cases = [ + # (BIOS artefacts present, expected sources_missing, why) + (["SCPH5552_dispatch.c", "SCPH5552_full.c"], 0, + "a pinned non-SCPH1001 stem must satisfy the host"), + (["OpenBIOS_dispatch.c", "OpenBIOS_full.c"], 0, + "the bundled stem must still satisfy the host"), + (["SCPH5552_dispatch.c"], 1, + "a dispatch with no _full.c is not a linkable backend"), + ([], 1, + "no BIOS backend at all means setup is genuinely incomplete"), + ] + failures = 0 + for artefacts, expected, why in cases: + project = tmp / ("case%d" % len(artefacts + [why])) + shutil.rmtree(project, ignore_errors=True) + make_project(project, artefacts) + env = dict(os.environ, PSXRECOMP_PROJECT_ROOT=str(project)) + run = subprocess.run([str(exe)], capture_output=True, text=True, env=env) + got = run.stdout.strip() + if got != str(expected): + print("FAIL: %s\n artefacts=%s expected=%s got=%r" + % (why, artefacts or "(none)", expected, got)) + failures += 1 + else: + print("ok: %s" % why) + if failures: + return 1 + + print("PASS: codegen host accepts any paired BIOS stem") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 3cbe6c1cdebed66e0f2d7d7ef451cd93d71202c7 Mon Sep 17 00:00:00 2001 From: Kevin <9866117+kerokline@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:30:32 -0400 Subject: [PATCH 2/9] kernel-bless: publish install-slot ranges so patched BIOS bodies run native The BIOS exception handler ran interpreted for the life of every process, 1.22 billion instructions and 44.6% of all interpreted work in Breath of Fire III, even though it is compiled and even though the stub that unblessed it sat at exactly the address bios/SCPH1001.toml already declared as an install slot. Two mechanisms existed and did not compose: the emitter hooked the declared slot PC, while the runtime's bless verifier memcmp'd the whole body including that slot, so the body was MISMATCH forever and native code never reached the hook. Three parts, none of which pays off alone: 1. Publish the declared ranges to the runtime. The emitter emits a PsxKernelPatchRange table next to the kernel body table and routes it through the backend descriptor; the bless verifier compares a body in segments that skip them. CLEAN now means "every byte outside every declared range still matches ROM", which is the claim the native code actually depends on. 2. Generalise the slot model to what the patches are. The old model assumed a 4-word lui/addiu/jalr/nop stub written over ROM zeros and fired on `word != 0`. Measured against both images, only one of five observed shapes fits: patchers also plant a `jr` over real ROM instructions, rewrite an 11-word prologue and insert an `mfc0 Cause`, and NOP out a driver routine. A slot is now a range with a length and an explicit `resume` (jalr / fallthrough / none), detected by comparing every word against its ROM-baked value. 3. Hand back from the interpreter at the range end. Kernel page 0 is permanently dirty, because the handler saves registers there on every exception, so straight-line interpretation would run the whole function after the patch. The emitter registers the resume PC as a continuation key and dirty_ram_interp surfaces there. A PC inside a declared range must never be a native dispatch key: the compiled bytes there are not what executes. Seven pre-existing jal-return continuations violated that and wedged the boot at frame 0 in an infinite dispatch loop (OpenBIOS RAM 0x357C). The emitter now drops continuation keys inside a range so dispatch misses through to the interpreter, and refuses to emit if a range covers a function entry, which would silently remove a callable function instead. Both profiles declare their measured ranges, and SCPH1001.toml finally pins [program.image] sha256 to v2.2 (CRC32 37157331) so a v2.0 dump refuses to generate rather than producing phantom discovery gaps. Measured on Breath of Fire III (SLPS-00990, Psy-Q SDK 3.70), headless, matched frames, PSX_KERNEL_PATCH_RANGES=0 for the OFF side: OpenBIOS SCPH-1001 v2.2 bless mismatch 21 -> 0 74 -> 0 kernel bodies -87.7%/frame -97.8%/frame all interp insns -42.6%/frame -94.2%/frame The residual kernel-body work is exactly declared-words x entries on every range, so nothing leaks past the hook. The A0/B0/C0 vector trampolines are unchanged (+0.2%, noise): the profile excludes them by design and they are a separate job. Card read traces are byte-identical across the A/B on both images, all 32 entries, which exercises the two ranges that replace the kernel's card handler. Rule 18 holds throughout: the guest's patched instructions still execute as written on the dirty-RAM interpreter. Only the code around them changes backend. Nothing is synthesised. Tests: runtime/tests/test_kernel_patch_ranges.c (new, the verifier's accept/reject behaviour and the hand-back predicate) and the install-slot range invariants in recompiler/tests/bios_address_model_test.cpp. Co-Authored-By: Claude Opus 5 --- bios/OpenBIOS.toml | 32 +++- bios/SCPH1001.toml | 63 ++++++- docs/config_schema.md | 35 +++- docs/dynamic_handler_install.md | 83 +++++++- recompiler/src/bios_address_model.cpp | 55 +++++- recompiler/src/bios_address_model.h | 50 ++++- recompiler/src/config_loader.cpp | 27 ++- recompiler/src/config_loader.h | 8 +- recompiler/src/full_function_emitter.cpp | 187 +++++++++++++++---- recompiler/tests/bios_address_model_test.cpp | 97 ++++++++++ runtime/CMakeLists.txt | 11 ++ runtime/include/dirty_ram_interp.h | 7 +- runtime/include/kernel_patch_ranges.h | 58 ++++++ runtime/include/psx_bios_backend.h | 7 + runtime/include/psx_bios_image.h | 22 +++ runtime/runtime.cmake | 1 + runtime/src/debug_server.c | 14 +- runtime/src/dirty_ram_interp.c | 20 +- runtime/src/kernel_patch_ranges.c | 40 ++++ runtime/src/memory.c | 56 +++++- runtime/src/psx_bios_backend.c | 4 + runtime/src/psx_selfcheck.c | 2 +- runtime/tests/test_kernel_patch_ranges.c | 136 ++++++++++++++ 23 files changed, 931 insertions(+), 84 deletions(-) create mode 100644 runtime/include/kernel_patch_ranges.h create mode 100644 runtime/src/kernel_patch_ranges.c create mode 100644 runtime/tests/test_kernel_patch_ranges.c diff --git a/bios/OpenBIOS.toml b/bios/OpenBIOS.toml index de6a781b..141da1a4 100644 --- a/bios/OpenBIOS.toml +++ b/bios/OpenBIOS.toml @@ -81,9 +81,35 @@ runtime_base = "0x80030000" dispatch_key = "rom" kernel_bless = false -# No [[recompiler.install_slots]]: none identified yet. Runtime-installed -# stubs fall to the dirty-RAM interpreter, which is correct (just slower) -# until a slot is identified and added. +# Kernel-RAM RANGES the guest's Psy-Q libapi patchers overwrite at runtime +# (CLAUDE.md rule 18; docs/dynamic_handler_install.md). See SCPH1001.toml for +# what `len` and `resume` mean. OpenBIOS declared none until 2026-09-11; the +# consequence was that its exception handler failed the kernel-bless memcmp +# from the first card access onward and interpreted for the life of the +# process — 1.22 billion instructions in one measured title. +# +# Measured on a live boot of Breath of Fire III (SLPS-00990, Psy-Q SDK 3.70) +# against the ROM source, 30 s in. The shapes are the same three the retail +# image shows, because the patchers are the game's, not the BIOS's. + +[[recompiler.install_slots]] +ram_addr = "0x000027B4" # exceptionHandler: register-save prologue +len = "0x30" # shifted two words + an inserted mfc0 v0,Cause +resume = "fallthrough" + +[[recompiler.install_slots]] +ram_addr = "0x0000281C" # stub into exceptionHandlerCardFastTrack +len = "0x10" # lui/addiu/jalr/nop over ROM zeros; installed +resume = "jalr" # on the first card access, not at boot + +[[recompiler.install_slots]] +ram_addr = "0x0000357C" # exceptionHandlerCardFastTrack: the kernel's +len = "0x10" # card handler replaced by jr into boot EXE +resume = "none" # 0x8017EAA0 (_patch_card payload) + +# NOT declared: the three pad-table words after readPadHighLevel (0x45FC, +# 0x460C, 0x461C). They are data, not instructions, so they unbless nothing +# and a range there would only widen what the verifier stops checking. # HLE anchors: only the boot-skip anchor is exported. deliver_event_ret is # deliberately ABSENT (0) until the B0 event services are validated against diff --git a/bios/SCPH1001.toml b/bios/SCPH1001.toml index 1f95ade7..5f5b39b8 100644 --- a/bios/SCPH1001.toml +++ b/bios/SCPH1001.toml @@ -16,6 +16,20 @@ load_address = "0xBFC00000" entry_pc = "0xBFC00000" text_size = "0x80000" +# Declared identity. The emitter refuses to generate from any other image +# (main_bios.cpp's declared-identity gate), which is the point: the seeds in +# this profile name v2.2 function starts, and generating from a SCPH-1001 +# **v2.0** dump (CRC32 55847D8C, 1994-09-22) produced three boot halts and a +# live A0 table with 90 of 161 ROM targets uncompiled. That looked like +# missing upstream seeds and was actually a revision mismatch. An empty pin +# let it through silently. +# +# This is v2.2, 1995-09-18: CRC32 37157331, 512 KiB. +[program.image] +sha256 = "71af94d1e47a68c11e8fdb9f8368040601514a42a5a399cda48c7d3bff1e99d3" +license = "proprietary" +redistributable = false + [recompiler] seeds = "recompiler/seeds/phase2_ghidra_seeds.json" out_dir = "generated" @@ -60,11 +74,56 @@ kernel_bless = false # class-B bug. Widening this window is a REAL BEHAVIOR CHANGE requiring its # own evidence-backed commit — it is not a parameterization cleanup. -# Kernel-RAM PCs the BIOS overwrites with 4-instruction dispatch stubs at +# Kernel-RAM RANGES the BIOS and the game's Psy-Q libapi patchers overwrite at # runtime (CLAUDE.md rule 18; docs/dynamic_handler_install.md). The emitter -# plants a dirty-check hook at these PCs so the installed stub executes. +# plants a compare-against-ROM hook at each range start so the guest's patched +# instructions execute; the runtime excludes the range from the kernel-bless +# memcmp and resumes native at the range end, so only the patched words +# interpret. +# +# `resume` says how the compiled body picks up again: "jalr" for the classic +# 4-word lui/addiu/jalr/nop stub (the call returns to ram_addr+0x10), +# "fallthrough" when the patched words ARE the function, "none" when the patch +# jumps out and never comes back. +# +# Measured on a live boot of Breath of Fire III (SLPS-00990, Psy-Q SDK 3.70) +# against the ROM source, 30 s in, identical at t+8 s — the patchers are +# _patch_gte / _patch_card / _patch_card2 / _patch_pad. Every SDK title runs +# these, so the ranges are a property of the BIOS revision and libapi release, +# not of one game. Re-measure per SDK version before trusting the numbers: +# the schema tolerates declaring the union. + +[[recompiler.install_slots]] +ram_addr = "0x00000C88" # exception handler: register-save prologue +len = "0x30" # shifted two words + an inserted mfc0 v0,Cause +resume = "fallthrough" # the patched words are the function itself + [[recompiler.install_slots]] ram_addr = "0x00000CF0" # SIO data-byte handler dispatch slot +len = "0x10" # lui/addiu/jalr/nop over ROM zeros +resume = "jalr" # the stub's call returns to 0x0D00 + +[[recompiler.install_slots]] +ram_addr = "0x00004964" # pad driver clear routine, 11 words NOP'd +len = "0x2C" # (RemovePatchPad / ChgclrPAD family) +resume = "fallthrough" + +[[recompiler.install_slots]] +ram_addr = "0x00004D98" # _patch_card2 payload: jalr into boot EXE +len = "0x14" # 0x8017EB6C; the ra landing word is NOP'd too, +resume = "fallthrough" # so the body resumes past it, not at +0x10 + +[[recompiler.install_slots]] +ram_addr = "0x00006444" # _patch_card payload: the kernel's card +len = "0x10" # handler replaced by jr into boot EXE +resume = "none" # 0x8017EAA0 — a jr never returns here + +# NOT declared: RAM 0x0500, one word of kernel_trampoline_0 cleared to zero +# (ROM `lui t2, 0`). It differs on every boot, but what writes it is not yet +# identified, and a range declared on a guess would tell the bless verifier to +# stop checking a word it should be checking. Rule 14: unknown is acceptable, +# guessing is not. Undeclared means that body keeps interpreting, exactly as +# it does today. # Per-image anchors couriered into the generated C (psx_bios_image) for the # runtime HLE tier. Values are FACTS about this image, not tunables: the diff --git a/docs/config_schema.md b/docs/config_schema.md index 5a268708..5c102fae 100644 --- a/docs/config_schema.md +++ b/docs/config_schema.md @@ -554,16 +554,45 @@ dispatch_key = "ram" # "ram": functions keyed by RAM address; # "rom": RAM alias folds back to ROM kernel_bless = true # runtime may byte-verify + run native -[[recompiler.install_slots]] # kernel-RAM PCs the BIOS patches at runtime -ram_addr = "0x00000CF0" +[[recompiler.install_slots]] # kernel-RAM RANGES patched at runtime +ram_addr = "0x00000CF0" # legacy form: len 0x10, resume "jalr" +[[recompiler.install_slots]] +ram_addr = "0x00000C88" +len = "0x30" # bytes; the patched range is [addr, addr+len) +resume = "fallthrough" # "jalr" (default) | "fallthrough" | "none" [recompiler.runtime_exports] # per-image HLE anchors (omit = unavailable) shell_entry_phys = "0x00030000" deliver_event_ret = "0x80001720" ``` +An `install_slots` entry declares kernel-RAM words the guest is EXPECTED to +overwrite at runtime — the BIOS's own install stubs and, far more often, the +Psy-Q libapi patchers every SDK title runs (`_patch_gte`, `_patch_card`, +`_patch_card2`, `_patch_pad`). The emitter plants a compare-against-ROM hook +at the range start, so a live patch dispatches into the interpreter and the +guest's own instructions execute; the runtime excludes the range from the +kernel-bless memcmp and resumes native at the range end. Without the +declaration the patched body fails verification forever and interprets for +the life of the process. + +`resume` says how the compiled body picks up again: + +| `resume` | Continuation PC | Use for | +|---|---|---| +| `"jalr"` (default) | `ram_addr + 0x10` | the classic 4-word `lui/addiu/jalr/nop` stub, whose call returns there | +| `"fallthrough"` | `ram_addr + len` | the patched words ARE the function (a prologue rewrite, a NOP'd routine) | +| `"none"` | — | the patch jumps out and never returns to this body (a `jr` into game text) | + +`ram_addr` and `len` must be 4-aligned, `len` non-zero, ranges must not +overlap, and every range must lie inside the `kernel_bless` window; the +loader refuses the profile otherwise and sorts the list for the runtime. +Finding the ranges for a new image is described in +[`dynamic_handler_install.md`](dynamic_handler_install.md). + Every `copy` entry is a claim that the boot copy is byte-verbatim; the -runtime kernel-bless memcmp enforces it. A BIOS with no copies (runs +runtime kernel-bless memcmp enforces it, minus the declared install-slot +ranges. A BIOS with no copies (runs entirely from ROM) is valid: normalization degenerates to the KSEG mask. Semantic invariants (disjoint windows, no fold-output/input intersection, single bless window) are enforced at load; violations refuse to build. diff --git a/docs/dynamic_handler_install.md b/docs/dynamic_handler_install.md index 17a5c279..5a3e371b 100644 --- a/docs/dynamic_handler_install.md +++ b/docs/dynamic_handler_install.md @@ -115,20 +115,85 @@ control returns to `psx_dispatch` for the next basic block. The combined effect is that BIOS-installed code at RAM 0xCF0 *runs*, correctly, on the same CPU register state as static-recompiled C. +## The patcher is usually the GAME, not the BIOS (measured 2026-09-11) + +The section above describes the BIOS patching itself. In practice the larger +source of patched kernel words is the **game**: every Psy-Q SDK title links +libapi, whose `_patch_gte` / `_patch_card` / `_patch_card2` / `_patch_pad` +routines rewrite kernel RAM during `ResetCallback`. Measured on Breath of Fire +III (SLPS-00990, SDK 3.70) against both BIOS images, 30 s into a boot, five +distinct shapes appear — and only the first fits the original one-PC, +`word != 0` model: + +| Shape | Example | Resume | +|---|---|---| +| 4-word `lui/addiu/jalr/nop` stub over ROM **zeros** | retail `0x0CF0`, OpenBIOS `0x281C` | `"jalr"` — the call returns to `+0x10` | +| 11-word prologue rewrite with an inserted `mfc0 v0, Cause` | retail `0x0C88`, OpenBIOS `0x27B4` | `"fallthrough"` — the patched words ARE the function | +| 11 words NOP'd (a driver routine removed) | retail `0x4964` | `"fallthrough"` | +| 5-word `jalr` into game text, `ra` landing word also NOP'd | retail `0x4D98` | `"fallthrough"`, not `+0x10` | +| 4-word `jr` into game text over **real ROM instructions** | retail `0x6444`, OpenBIOS `0x357C` | `"none"` — a `jr` never comes back | + +The three patched shapes that overwrite real ROM instructions are invisible to +a zero test, and two of them never return to `ram_addr + 0x10`. That is why a +slot is a **range** with an explicit `resume` (see +[`config_schema.md`](config_schema.md) → install slots), detected by comparing +every word against its ROM-baked value. + +## The slot and the bless verifier must compose + +Declaring a slot is necessary but not sufficient, and getting this wrong cost +one title its whole kernel budget. The mechanism has three parts that only +work together: + +1. **The emitted hook** (`full_function_emitter.cpp`) fires when any word in + the declared range differs from ROM, and dispatches to the range start so + the interpreter runs the guest's patched instructions. +2. **The bless verifier** (`memory.c`, via `kernel_patch_ranges.c`) compares a + kernel body in segments that SKIP the declared ranges. Without this the + body's whole-extent memcmp fails on the first patch and the body is + MISMATCH for the life of the process — so native code never reaches the + hook at all. The two halves were written separately and did not compose: + Breath of Fire III's card stub landed at exactly the address + `bios/SCPH1001.toml` already declared, and the exception handler still + interpreted **1.22 billion instructions**, 44.6 % of all interpreted work. +3. **The interpreter hand-back** (`dirty_ram_interp.c`) surfaces to static + dispatch when straight-line flow reaches a range's `hi`, which the emitter + registered as a continuation key. Kernel page 0 is permanently dirty (the + handler saves registers there on every exception), so without this the + interpreter would run the entire function after the patch rather than just + the patched words. + +`PSX_KERNEL_PATCH_RANGES=0` drops the declared ranges at runtime, restoring +the whole-body memcmp. One binary then measures both sides of the change +(companion to `PSX_KERNEL_BLESS=0`, which disables the mechanism outright). + ## Discoverability -Other install slots may exist. The tracker should detect them -mechanically by: +Finding the ranges for a new image is a diff, not a hunt: snapshot the live +kernel RAM copy, compare it word-for-word against the ROM source it was +copied from, and keep the differences that land inside a compiled body. -- Logging every write into `RAM 0x000..0xFFFF` (kernel code area). -- Reporting any PC dispatch that hits a written page. +- Read the kernel-bless window and ROM offset out of the linked image + (`psx_bios_image`), read live RAM over the debug server, and diff. +- Attribute each differing word to a body using the emitted body table; words + outside every body are data (event tables, TCB saves, pad tables) and + unbless nothing — do not declare them. +- Group contiguous differing words into one range, then read the LIVE words to + pick `resume`: a `jalr` returns to `+0x10`, a `jr` never returns, and + anything else falls through to the range end. +- Snapshot twice (early and late). Some patches land at boot and some on the + first card access, and a range that appears only late is still a range. +- Cross-check `kernel_bless` over the debug server: `mismatch` should fall to + the bodies you have NOT declared, and `patch_skips` should be non-zero. -Known/suspected slots to verify after the SIO data handler works: +A word whose writer you cannot identify is a word you should not declare: the +declaration tells the verifier to stop checking it. Leaving it undeclared +costs performance in one body; declaring it wrongly costs correctness +everywhere (CLAUDE.md Rule 14). -- VBlank pad poll fast-path (?) -- Memcard slot-1 detection -- CD-ROM ready handler -- Anything in `RAM 0xC80..0xFFF` (kernel hook region) +Slot addresses are per BIOS revision AND per libapi release. Re-measure on a +second title before assuming a profile's list is complete; the schema +tolerates declaring the union. ## NOT HLE diff --git a/recompiler/src/bios_address_model.cpp b/recompiler/src/bios_address_model.cpp index 9c3d9259..495f224f 100644 --- a/recompiler/src/bios_address_model.cpp +++ b/recompiler/src/bios_address_model.cpp @@ -2,6 +2,7 @@ #include "bios_address_model.h" +#include #include #include "fmt/format.h" @@ -91,10 +92,43 @@ BiosAddressModel BiosAddressModel::from_config(const BiosConfig& cfg) { m.rom_keyed_idx_ = static_cast(i); } - for (uint32_t slot : m.install_slots_) { - if (slot & 3u) - fail(fmt::format("install slot 0x{:08X} must be 4-aligned", slot)); + // Install slots are RANGES now: validate both ends, reject empty and + // overlapping declarations (the runtime's verifier walks them in order + // and the emitter plants one hook per start). + for (size_t i = 0; i < m.install_slots_.size(); ++i) { + const BiosInstallSlot& s = m.install_slots_[i]; + if (s.ram_addr & 3u) + fail(fmt::format("install slot 0x{:08X}: ram_addr must be 4-aligned", + s.ram_addr)); + if (s.len == 0u || (s.len & 3u)) + fail(fmt::format("install slot 0x{:08X}: len must be a non-zero " + "multiple of 4 (got 0x{:X})", s.ram_addr, s.len)); + if (s.hi() < s.ram_addr) + fail(fmt::format("install slot 0x{:08X}: len 0x{:X} wraps", + s.ram_addr, s.len)); + for (size_t j = 0; j < i; ++j) { + const BiosInstallSlot& p = m.install_slots_[j]; + if (s.ram_addr < p.hi() && p.ram_addr < s.hi()) + fail(fmt::format("install slots 0x{:08X} and 0x{:08X} overlap", + p.ram_addr, s.ram_addr)); + } + // A slot outside the kernel-bless window would be published to the + // runtime's verifier as a range it can never reach: a profile defect. + if (m.kbless_idx_ >= 0) { + uint32_t lo = m.copies_[m.kbless_idx_].ram_lo; + uint32_t hi = m.copies_[m.kbless_idx_].ram_hi(); + if (s.ram_addr < lo || s.hi() > hi) + fail(fmt::format("install slot [0x{:08X},0x{:08X}) lies outside " + "the kernel_bless window [0x{:08X},0x{:08X})", + s.ram_addr, s.hi(), lo, hi)); + } } + // Sorted by start: memory.c's segmented verifier relies on the emitted + // table being ordered, and the emitter couriers this vector verbatim. + std::sort(m.install_slots_.begin(), m.install_slots_.end(), + [](const BiosInstallSlot& a, const BiosInstallSlot& b) { + return a.ram_addr < b.ram_addr; + }); return m; } @@ -202,12 +236,23 @@ bool BiosAddressModel::in_kbless(uint32_t norm) const { } bool BiosAddressModel::is_install_slot(uint32_t ram_pc) const { - for (uint32_t slot : install_slots_) { - if (ram_pc == slot) return true; + return install_slot_at(ram_pc) != nullptr; +} + +bool BiosAddressModel::in_install_slot_range(uint32_t ram_pc) const { + for (const BiosInstallSlot& s : install_slots_) { + if (ram_pc >= s.ram_addr && ram_pc < s.hi()) return true; } return false; } +const BiosInstallSlot* BiosAddressModel::install_slot_at(uint32_t ram_pc) const { + for (const BiosInstallSlot& s : install_slots_) { + if (ram_pc == s.ram_addr) return &s; + } + return nullptr; +} + uint32_t BiosAddressModel::rom_keyed_ram_lo() const { return copies_[rom_keyed_idx_].ram_lo; } diff --git a/recompiler/src/bios_address_model.h b/recompiler/src/bios_address_model.h index b21bcb65..29d3dbc0 100644 --- a/recompiler/src/bios_address_model.h +++ b/recompiler/src/bios_address_model.h @@ -48,6 +48,41 @@ struct BiosAddrCopy { uint32_t ram_hi() const { return ram_lo + len(); } // exclusive }; +// One [[recompiler.install_slots]]: a RANGE of kernel-RAM words the guest is +// expected to overwrite at runtime (Psy-Q's _patch_gte / _patch_card / +// _patch_card2 / _patch_pad, and the BIOS's own install stubs). +// +// The original model was a single 4-word `lui/addiu/jalr/nop` stub written +// over ROM zeros, detected by testing the first word against 0. Only one of +// the five shapes observed in the wild fits that (docs/dynamic_handler_install.md): +// patchers also overwrite REAL ROM instructions with a `jr`, rewrite an +// 11-word prologue and insert an `mfc0 Cause`, and NOP out a driver routine. +// So a slot carries a length and is detected against the ROM-BAKED words, +// not against zero. +struct BiosInstallSlot { + // How the compiled body resumes native execution after the interpreter + // has run the patched words. + enum class Resume { + Jalr, // legacy 4-word stub: the jalr's ra = ram_addr + 0x10 + Fallthrough, // the patched words ARE the function: resume at ram_addr + len + None, // the patch never returns here (a `jr` out of the kernel) + }; + + uint32_t ram_addr = 0; // start of the patched range + uint32_t len = 0x10; // bytes; legacy default = 4 words + Resume resume = Resume::Jalr; // legacy default + + uint32_t hi() const { return ram_addr + len; } // exclusive + // RAM PC the compiled body resumes at, or 0 for Resume::None. + uint32_t resume_addr() const { + switch (resume) { + case Resume::Jalr: return ram_addr + 0x10u; + case Resume::Fallthrough: return hi(); + default: return 0u; + } + } +}; + class BiosAddressModel { public: // Empty model: pure KSEG-mask normalization, no copies, no install slots. @@ -99,7 +134,20 @@ public: // --- install slots ---------------------------------------------------- + // True when ram_pc is the START of a declared slot (where the emitter + // plants the hook). Interior words of a range are NOT slot starts. bool is_install_slot(uint32_t ram_pc) const; + // The slot starting at ram_pc, or null. + const BiosInstallSlot* install_slot_at(uint32_t ram_pc) const; + const std::vector& install_slots() const { return install_slots_; } + // Is this RAM PC inside ANY declared range? Such a PC must never become a + // native dispatch key: the compiled bytes there are not what executes — + // the guest's patch is. A key inside a range re-enters the compiled body, + // whose patch-range hook immediately sets pc back to the range start and + // returns, so the dispatch loop spins forever (observed 2026-09-11: a + // pre-existing jal-return continuation at OpenBIOS RAM 0x357C wedged the + // boot at frame 0). + bool in_install_slot_range(uint32_t ram_pc) const; // --- ROM-keyed RAM window (shell-style), for the game-overlap text ---- @@ -116,7 +164,7 @@ public: private: std::vector copies_; - std::vector install_slots_; + std::vector install_slots_; uint32_t rom_base_phys_ = 0x1FC00000u; uint32_t rom_size_ = 0x80000u; int kbless_idx_ = -1; // index into copies_ diff --git a/recompiler/src/config_loader.cpp b/recompiler/src/config_loader.cpp index db8975fe..c82ec734 100644 --- a/recompiler/src/config_loader.cpp +++ b/recompiler/src/config_loader.cpp @@ -1060,14 +1060,31 @@ BiosConfig load_bios_config(const fs::path& config_path_in) { } } - // [[recompiler.install_slots]] — kernel-RAM PCs the BIOS overwrites with - // dispatch stubs at runtime. - std::vector install_slots; + // [[recompiler.install_slots]] — kernel-RAM ranges the BIOS or the game's + // Psy-Q libapi patchers overwrite at runtime. `ram_addr` alone keeps the + // original meaning (a 4-word jalr stub); `len` and `resume` describe the + // other patch shapes (BiosInstallSlot, bios_address_model.h). + std::vector install_slots; if (recomp.contains("install_slots")) { for (const auto& v : recomp.at("install_slots").as_array()) { - install_slots.push_back(parse_hex( + BiosInstallSlot slot; + slot.ram_addr = parse_hex( toml::find(v, "ram_addr"), - "install_slots.ram_addr")); + "install_slots.ram_addr"); + if (v.contains("len")) + slot.len = parse_hex(toml::find(v, "len"), + "install_slots.len"); + if (v.contains("resume")) { + const std::string r = toml::find(v, "resume"); + if (r == "jalr") slot.resume = BiosInstallSlot::Resume::Jalr; + else if (r == "fallthrough") slot.resume = BiosInstallSlot::Resume::Fallthrough; + else if (r == "none") slot.resume = BiosInstallSlot::Resume::None; + else throw std::runtime_error(fmt::format( + "{}: install_slots 0x{:08X}: resume must be \"jalr\", " + "\"fallthrough\" or \"none\", got '{}'", + config_path.string(), slot.ram_addr, r)); + } + install_slots.push_back(slot); } } diff --git a/recompiler/src/config_loader.h b/recompiler/src/config_loader.h index 9457279f..4dbde78b 100644 --- a/recompiler/src/config_loader.h +++ b/recompiler/src/config_loader.h @@ -695,9 +695,11 @@ struct BiosConfig { // copies, semantic validation and consumption in BiosAddressModel // (bios_address_model.h). Empty = the BIOS runs entirely from ROM. std::vector address_copies; - // [[recompiler.install_slots]]: kernel-RAM PCs the BIOS overwrites with - // dispatch stubs at runtime (see docs/dynamic_handler_install.md). - std::vector install_slots; + // [[recompiler.install_slots]]: kernel-RAM RANGES the BIOS (or the game's + // Psy-Q libapi patchers) overwrite at runtime. Layout, defaults and the + // resume shapes are in BiosInstallSlot (bios_address_model.h); see + // docs/dynamic_handler_install.md for how to find new ones. + std::vector install_slots; // [recompiler.runtime_exports]: per-image anchors the emitter couriers // into the generated C (psx_bios_image, runtime/include/psx_bios_image.h) diff --git a/recompiler/src/full_function_emitter.cpp b/recompiler/src/full_function_emitter.cpp index c15f32d4..d201ada2 100644 --- a/recompiler/src/full_function_emitter.cpp +++ b/recompiler/src/full_function_emitter.cpp @@ -1192,56 +1192,90 @@ bool FullFunctionEmitter::emit_function( continue; } - // Install-slot hook (CLAUDE.md Rule 18 / docs/dynamic_handler_install.md). - // The PS1 BIOS overwrites specific kernel-RAM addresses at runtime - // with dispatch stubs (e.g. RAM 0xCF0 for the SIO data-byte handler). - // The recompiler emitted NOPs from the ROM image; if we just run those - // statically, the installed stub never executes. At known install-slot - // PCs, emit a runtime check: if the page is dirty (RAM was written-to), - // dispatch into RAM to run the installed code. Otherwise fall through - // to the static NOP. + // Kernel patch-range hook (CLAUDE.md Rule 18 / docs/dynamic_handler_install.md). + // The BIOS and the game's Psy-Q libapi patchers overwrite specific + // kernel-RAM words at runtime: install stubs written over ROM zeros + // (RAM 0xCF0, the SIO data-byte handler), but also a `jr` planted over + // real ROM instructions, an 11-word prologue rewrite that inserts an + // `mfc0 Cause`, and a driver routine NOP'd out. The recompiler emitted + // the ROM bytes; if we run those statically the guest's patch never + // executes. // - // Install slots come from the profile's [[recompiler.install_slots]] - // (e.g. SCPH1001's SIO data-byte handler slot at RAM 0xCF0). See + // At the START of a declared range, emit a runtime check: if ANY word + // in the range differs from its ROM-baked value, the patch is live — + // dispatch into RAM so the interpreter runs the guest's own + // instructions (Rule 18: we execute the patch as written, we do not + // synthesise its effect). The rest of the body still runs native, + // because the range is also published to the runtime, which excludes + // it from the kernel-bless memcmp (memory.c) and hands straight-line + // flow back at the range end (dirty_ram_interp.c). + // + // Ranges come from the profile's [[recompiler.install_slots]]; see // docs/dynamic_handler_install.md for how to find new ones. uint32_t ram_pc = addr_model().rom_to_ram_phys(addr); - bool is_install_slot = addr_model().is_install_slot(ram_pc); - if (is_install_slot) { - /* The installed stub is 4 instructions: lui, addiu, jalr, nop. - * The jalr captures ra = stub_PC + 8 = install_slot + 0x10. When - * the stub's called function returns via jr ra, control flows back - * to install_slot + 0x10. Register that ROM address as both a - * block leader (so label_: is emitted) and a local - * continuation (so the entry-switch routes to it). Without this, - * external dispatch to the post-stub PC misses and falls into - * interpretation, which doesn't have COP0 and crashes the - * exception handler. */ - uint32_t post_stub_rom = addr + 0x10u; - if (addr_to_raw.count(post_stub_rom)) { - block_leaders.insert(post_stub_rom); - uint32_t post_stub_norm = normalize_address(post_stub_rom); - if (!all_function_entries_norm.count(post_stub_norm)) { - local_continuations.push_back({post_stub_rom, post_stub_norm, norm}); + const BiosInstallSlot* slot = addr_model().install_slot_at(ram_pc); + if (slot) { + /* Where the compiled body picks up again: + * - Jalr (legacy 4-word stub): the stub's jalr captures + * ra = stub_PC + 8 = ram_addr + 0x10, so the called function + * returns there. + * - Fallthrough: the patched words ARE the function; execution + * continues at the end of the range. + * - None: the patch is a `jr` out of the kernel and never comes + * back to this body. + * For the first two, register the resume PC as both a block leader + * (so label_: exists) and a local continuation (so the + * entry-switch and the emitted continuation wrapper route to it). + * Without the continuation key, external dispatch to the resume PC + * misses and falls into interpretation, which has no COP0 and + * crashes the exception handler. */ + uint32_t resume_ram = slot->resume_addr(); + if (resume_ram != 0u) { + uint32_t resume_rom = addr + (resume_ram - ram_pc); + if (addr_to_raw.count(resume_rom)) { + block_leaders.insert(resume_rom); + uint32_t resume_norm = normalize_address(resume_rom); + if (!all_function_entries_norm.count(resume_norm)) { + local_continuations.push_back({resume_rom, resume_norm, norm}); + } } } - /* Hook fires only when the FIRST instruction word at this PC - * differs from the ROM-baked value (= 0x00000000 NOP for these - * slots). A page-level dirty bit is too coarse: the kernel - * handler dirties page 0 on every exception by saving registers, - * which would unconditionally redirect into the interpreter. - * Word-level check is exact: the slot is "live" iff the BIOS - * install function has actually overwritten it. */ + /* Compare every word in the range against the ROM-baked value. + * The old test was `first word != 0`, which only fires for a stub + * written over ROM zeros — a `jr` planted over a real instruction + * and a prologue rewrite both leave non-zero ROM words and were + * never detected. A page-level dirty bit is far too coarse: the + * kernel handler dirties page 0 on every exception by saving + * registers, which would redirect unconditionally. Word-level + * compare is exact: the range is "live" iff the guest has actually + * changed it. */ + std::string cond; + uint32_t base_phys_local = base_addr & 0x1FFFFFFFu; + for (uint32_t off = 0; off < slot->len; off += 4u) { + uint32_t w_rom_addr = addr + off; + uint32_t w_phys = w_rom_addr & 0x1FFFFFFFu; + if (w_phys < base_phys_local || + w_phys + 4u > base_phys_local + rom.size()) { + throw std::runtime_error(fmt::format( + "install slot RAM 0x{:08X} len 0x{:X}: word +0x{:X} " + "lies outside the ROM image", ram_pc, slot->len, off)); + } + uint32_t rom_word = read_u32_le(rom, w_phys - base_phys_local); + if (!cond.empty()) cond += " ||\n "; + cond += fmt::format("cpu->read_word(0x{:08X}u) != 0x{:08X}u", + ram_pc + off, rom_word); + } out += fmt::format( - " /* 0x{:08X}: install-slot hook (RAM 0x{:08X}) — if the BIOS\n" - " * has overwritten this slot with an install stub, dispatch\n" - " * into the stub. Otherwise fall through to static NOP.\n" - " * After the stub's jalr returns, ra=RAM 0x{:08X} routes\n" - " * back here as a registered continuation target. */\n" - " if (cpu->read_word(0x{:08X}u) != 0u) {{\n" + " /* 0x{:08X}: kernel patch-range hook (RAM [0x{:08X},0x{:08X}),\n" + " * resume RAM 0x{:08X}). If the guest has overwritten any word\n" + " * of this range, dispatch into RAM so its own instructions run;\n" + " * otherwise fall through to the statically compiled ROM code. */\n" + " if ({}) {{\n" " psx_check_interrupts_at(cpu, 0x{:08X}u);\n" " cpu->pc = 0x{:08X}u; return;\n" " }}\n", - addr, ram_pc, ram_pc + 0x10u, ram_pc, ram_pc, ram_pc); + addr, ram_pc, ram_pc + slot->len, resume_ram, + cond, ram_pc, ram_pc); } // Non-terminator: emit normally — unless this load's successor reads @@ -1912,6 +1946,40 @@ void FullFunctionEmitter::emit_dispatch( g_sym_prefix, kb_count); } + // --- Kernel patch-range table (runtime bless + interp hand-back) --- + // The profile's [[recompiler.install_slots]], couriered verbatim to the + // runtime. Two consumers read it: memory.c verifies a kernel body in + // segments that SKIP these ranges (so a body with a live install stub is + // still blessed for native execution — before this, the whole-body memcmp + // failed forever on the first patch and the BIOS exception handler + // interpreted for the life of the process), and dirty_ram_interp.c hands + // straight-line flow back to static dispatch at a range's hi, which the + // per-function emitter registered as a continuation key. The guest's + // patched words themselves still execute on the interpreter, as written. + // + // Emitted sorted and non-overlapping (BiosAddressModel::from_config + // validates and sorts), because the runtime's verifier walks them in + // order. + { + std::string pr; + size_t pr_count = 0; + for (const BiosInstallSlot& sl : addr_model().install_slots()) { + pr += fmt::format(" {{ 0x{:08X}u, 0x{:08X}u }},\n", + sl.ram_addr, sl.hi()); + pr_count++; + } + out += fmt::format( + "static const PsxKernelPatchRange {}psx_bios_kernel_patch_ranges[{}] = {{\n", + g_sym_prefix, pr_count ? pr_count : 1); + // A zero-length array is not C; an image with no declared slots emits + // one inert entry and a count of 0, which every consumer range-tests + // against the count before indexing. + out += pr_count ? pr : " { 0u, 0u },\n"; + out += "};\n"; + out += fmt::format("enum {{ {}psx_bios_kernel_patch_range_count = {}u }};\n\n", + g_sym_prefix, pr_count); + } + // --- Image self-description (runtime/include/psx_bios_image.h) --- // The runtime reads these instead of hardcoding per-image constants; // emitted next to the code they describe, so they cannot disagree with @@ -2288,6 +2356,8 @@ void FullFunctionEmitter::emit_dispatch( " {0}psx_dispatch_call," "\n" " {0}psx_bios_kernel_bodies," "\n" " {0}psx_bios_kernel_body_count," "\n" + " {0}psx_bios_kernel_patch_ranges," "\n" + " {0}psx_bios_kernel_patch_range_count," "\n" "}};" "\n", g_sym_prefix); } @@ -2517,12 +2587,47 @@ EmitStats FullFunctionEmitter::emit( // --- Emit continuation wrappers --- // Deduplicate continuations by norm_addr (same label from multiple callers). std::map unique_continuations; + size_t slot_range_keys_dropped = 0; for (const auto& cl : all_continuations) { + // A PC inside a declared install-slot range must NOT become a native + // dispatch key. The compiled bytes there are not what executes — the + // guest's patch is — and the patch-range hook at the range start sets + // cpu->pc back to that same PC and returns, so dispatching into the + // body spins the dispatch loop forever. Dropping the key lets dispatch + // miss through to the dirty-RAM interpreter, which runs the guest's + // patched words: the intended path. + // + // Cost paid 2026-09-11: OpenBIOS RAM 0x357C already carried a + // jal-return continuation from before install slots existed, and + // declaring the card-handler range there wedged the boot at frame 0. + // Four more such keys sit inside retail's NOP'd pad-clear range. + if (addr_model().in_install_slot_range(cl.norm_addr)) { + slot_range_keys_dropped++; + continue; + } // Only add if not already a function entry (shouldn't be, but guard). if (!emitted_normalized.count(cl.norm_addr)) { unique_continuations[cl.norm_addr] = cl; } } + if (slot_range_keys_dropped) { + std::fprintf(stdout, + "psxrecomp-bios: dropped %zu continuation key(s) inside declared install-slot ranges\n", + slot_range_keys_dropped); + } + // A FUNCTION ENTRY inside a declared range is a different matter: dropping + // it would silently remove a callable function from dispatch. That is a + // profile error (the range is too wide, or it covers real code the guest + // does not patch), so refuse to emit rather than produce a build whose + // dispatch quietly misses. + for (uint32_t norm : emitted_normalized) { + if (addr_model().in_install_slot_range(norm)) { + throw std::runtime_error(fmt::format( + "install-slot range covers the function entry at RAM 0x{:08X}: " + "a declared range may only cover words the guest patches, never " + "a dispatch entry. Narrow the range in the BIOS profile.", norm)); + } + } if (!unique_continuations.empty()) { full_c += fmt::format("\n/* --- {} continuation wrappers for jal/jalr return routing --- */\n\n", diff --git a/recompiler/tests/bios_address_model_test.cpp b/recompiler/tests/bios_address_model_test.cpp index 3f8a3cc2..282be397 100644 --- a/recompiler/tests/bios_address_model_test.cpp +++ b/recompiler/tests/bios_address_model_test.cpp @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include "../src/bios_address_model.h" @@ -23,6 +25,7 @@ using PSXRecompV4::BiosAddressModel; using PSXRecompV4::BiosAddrCopy; using PSXRecompV4::BiosConfig; +using PSXRecompV4::BiosInstallSlot; // --------------------------------------------------------------------------- // Reference implementations: verbatim pre-refactor SCPH1001 hardcodes. @@ -175,8 +178,66 @@ int main(int argc, char** argv) { expect_eq(model.kbless_rom_off(), 0x10000u, "kbless_rom_off", 0); expect_eq(model.rom_keyed_ram_lo(), 0x30000u, "rom_keyed_ram_lo", 0); expect_eq(model.rom_keyed_ram_hi_incl(), 0x5AFFFu, "rom_keyed_ram_hi_incl", 0); + + // --- install slots are RANGES ---------------------------------------- + // Only a range START is a slot: that is where the emitter plants the + // compare-against-ROM hook. Interior words are covered by the range, not + // by a second hook. if (!model.is_install_slot(0xCF0u)) { std::fprintf(stderr, "FAIL: 0xCF0 not an install slot\n"); g_failures++; } if (model.is_install_slot(0xCF4u)) { std::fprintf(stderr, "FAIL: 0xCF4 claims install slot\n"); g_failures++; } + // bios/SCPH1001.toml declares the measured Psy-Q patch sites. The legacy + // 0xCF0 form (ram_addr alone) must still mean a 4-word jalr stub. + { + const BiosInstallSlot* sio = model.install_slot_at(0xCF0u); + if (!sio) { std::fprintf(stderr, "FAIL: no slot at 0xCF0\n"); g_failures++; } + else { + expect_eq(sio->len, 0x10u, "slot.0xCF0.len", 0xCF0u); + expect_eq(sio->hi(), 0xD00u, "slot.0xCF0.hi", 0xCF0u); + expect_eq(sio->resume_addr(), 0xD00u, "slot.0xCF0.resume", 0xCF0u); + if (sio->resume != BiosInstallSlot::Resume::Jalr) { + std::fprintf(stderr, "FAIL: 0xCF0 is not a jalr slot\n"); g_failures++; + } + } + // The exception-handler prologue rewrite: a range, resumed by + // fallthrough at its end (the patched words ARE the function). + const BiosInstallSlot* exc = model.install_slot_at(0xC88u); + if (!exc) { std::fprintf(stderr, "FAIL: no slot at 0xC88\n"); g_failures++; } + else { + expect_eq(exc->len, 0x30u, "slot.0xC88.len", 0xC88u); + expect_eq(exc->resume_addr(), 0xCB8u, "slot.0xC88.resume", 0xC88u); + } + // The card-handler replacement is a `jr` out of the kernel: no resume. + const BiosInstallSlot* card = model.install_slot_at(0x6444u); + if (!card) { std::fprintf(stderr, "FAIL: no slot at 0x6444\n"); g_failures++; } + else expect_eq(card->resume_addr(), 0u, "slot.0x6444.resume", 0x6444u); + // Range CONTAINMENT, not just the start. The emitter drops any + // dispatch key inside a range, so this predicate decides which keys + // survive; getting it wrong wedges the boot (the 0x357C loop). + if (!model.in_install_slot_range(0xCF0u) || + !model.in_install_slot_range(0xCF4u) || + !model.in_install_slot_range(0xCFCu)) { + std::fprintf(stderr, "FAIL: 0xCF0 range does not contain its own words\n"); + g_failures++; + } + if (model.in_install_slot_range(0xD00u)) { + std::fprintf(stderr, "FAIL: range hi 0xD00 claimed as inside (must be exclusive)\n"); + g_failures++; + } + if (model.in_install_slot_range(0xCECu)) { + std::fprintf(stderr, "FAIL: 0xCEC below the range claimed as inside\n"); + g_failures++; + } + // Emitted order is the runtime verifier's contract. + uint32_t prev = 0; + for (const BiosInstallSlot& sl : model.install_slots()) { + if (sl.ram_addr < prev) { + std::fprintf(stderr, "FAIL: install slots not sorted at 0x%08X\n", + sl.ram_addr); + g_failures++; + } + prev = sl.ram_addr; + } + } // --- from_config invariants refuse bad profiles ----------------------- expect_throw("overlapping ROM windows", [](BiosConfig& c) { @@ -221,6 +282,42 @@ int main(int argc, char** argv) { }; }, minimal_cfg()); + // --- install-slot invariants refuse bad declarations ------------------ + { + auto with_slots = [](std::vector slots) { + BiosConfig c = minimal_cfg(); + c.address_copies = { + mk("k", 0x1FC10000u, 0x1FC18000u, 0x500u, 0x500u, true, true), + }; + c.install_slots = std::move(slots); + return c; + }; + auto slot = [](uint32_t at, uint32_t len) { + BiosInstallSlot s; s.ram_addr = at; s.len = len; + s.resume = BiosInstallSlot::Resume::Fallthrough; + return s; + }; + expect_throw("misaligned slot start", + [](BiosConfig&) {}, with_slots({slot(0xC8Au, 0x10u)})); + expect_throw("zero-length slot", + [](BiosConfig&) {}, with_slots({slot(0xC88u, 0u)})); + expect_throw("misaligned slot length", + [](BiosConfig&) {}, with_slots({slot(0xC88u, 0x12u)})); + expect_throw("overlapping slots", + [](BiosConfig&) {}, with_slots({slot(0xC88u, 0x30u), slot(0xCB4u, 0x10u)})); + expect_throw("slot outside the bless window", + [](BiosConfig&) {}, with_slots({slot(0x9000u, 0x10u)})); + expect_throw("slot range crossing the bless window end", + [](BiosConfig&) {}, with_slots({slot(0x84F0u, 0x30u)})); + // Declared out of order: accepted, and sorted for the runtime. + { + BiosConfig c = with_slots({slot(0x4964u, 0x2Cu), slot(0xC88u, 0x30u)}); + const auto m = BiosAddressModel::from_config(c); + expect_eq(m.install_slots()[0].ram_addr, 0xC88u, "slots.sorted[0]", 0); + expect_eq(m.install_slots()[1].ram_addr, 0x4964u, "slots.sorted[1]", 0); + } + } + // --- empty model degenerates to the KSEG mask ------------------------- { BiosAddressModel empty; diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 37464aea..46c476a3 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -303,6 +303,17 @@ if(BUILD_TESTING) target_include_directories(overlay_path_canon_test PRIVATE include) add_test(NAME overlay_path_canon_test COMMAND overlay_path_canon_test) + # The kernel-bless verifier must accept a body whose DECLARED patch range + # the guest has overwritten (Psy-Q's install stubs land inside compiled + # kernel bodies) and still reject one changed anywhere else. Before this, + # the whole-body memcmp failed forever on the first install and the BIOS + # exception handler interpreted for the life of the process. + add_executable(kernel_patch_ranges_test + tests/test_kernel_patch_ranges.c + src/kernel_patch_ranges.c) + target_include_directories(kernel_patch_ranges_test PRIVATE include) + add_test(NAME kernel_patch_ranges_test COMMAND kernel_patch_ranges_test) + # The player-facing "skip the BIOS" flag must resolve to the same boot # behaviour on retail SCPH-1001 and on the bundled OpenBIOS, whose # kernel-call HLE is (correctly) refused for want of a DeliverEvent anchor. diff --git a/runtime/include/dirty_ram_interp.h b/runtime/include/dirty_ram_interp.h index 1fb60442..24cfe449 100644 --- a/runtime/include/dirty_ram_interp.h +++ b/runtime/include/dirty_ram_interp.h @@ -139,8 +139,13 @@ int dirty_ram_is_dirty(uint32_t phys); * on writes into the body. Runtime-patched bodies (pad/SIO install stubs) * never verify and keep interpreting — faithful either way. */ int psx_kernel_bless_dispatchable(uint32_t phys); +/* True when a declared kernel patch range ENDS at this RAM address. The + * emitter registered that PC as a continuation key, so the interpreter hands + * straight-line flow back to static dispatch there and only the guest's + * patched words interpret (memory.c psx_bios_kernel_patch_ranges). */ +int psx_kernel_patch_range_ends_at(uint32_t phys); void psx_kernel_bless_note_range(uint32_t phys, uint32_t len); -void psx_kernel_bless_stats(uint64_t out[6]); +void psx_kernel_bless_stats(uint64_t out[8]); void psx_kernel_bless_resync_after_restore(void); /* Soft-return rematch / BIOS switch: drop latched SCPH↔OpenBIOS window + * CLEAN/MISMATCH so the next kbless_on() re-reads psx_bios_image. */ diff --git a/runtime/include/kernel_patch_ranges.h b/runtime/include/kernel_patch_ranges.h new file mode 100644 index 00000000..5d1a3b39 --- /dev/null +++ b/runtime/include/kernel_patch_ranges.h @@ -0,0 +1,58 @@ +/* kernel_patch_ranges.h — the pure half of the kernel patch-range mechanism. + * + * The guest legitimately overwrites words inside compiled BIOS kernel bodies + * at boot: the Psy-Q libapi patchers (_patch_gte / _patch_card / _patch_card2 + * / _patch_pad) and the BIOS's own install stubs. Those ranges are declared + * per image by [[recompiler.install_slots]] and emitted next to the kernel + * body table (psx_bios_image.h, PsxKernelPatchRange). + * + * These two functions are the whole decision, kept free of runtime globals so + * they can be tested directly (runtime/tests/test_kernel_patch_ranges.c). + * memory.c wraps them with its snapshotted window constants; dirty_ram_interp.c + * reads the second one through that wrapper. + * + * Rule 18: the patched words still execute, on the dirty-RAM interpreter, + * exactly as the guest wrote them. Nothing here synthesises behaviour — it + * only decides which backend runs which words. + */ +#ifndef PSX_KERNEL_PATCH_RANGES_H +#define PSX_KERNEL_PATCH_RANGES_H + +#include + +#include "psx_bios_image.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Compare live kernel RAM [lo, hi) against its ROM source, SKIPPING every + * declared patch range that intersects it. Returns 0 when every compared + * segment matches (memcmp's "equal" sense), non-zero otherwise. + * + * ram/rom point at the two images; `ram_lo` is the RAM address that + * rom[rom_off] corresponds to, so a byte at RAM address a is ram[a] and + * rom[rom_off + (a - ram_lo)]. `ranges` must be sorted by lo and + * non-overlapping (BiosAddressModel::from_config validates and sorts, so a + * malformed profile cannot reach here). `skips_out`, when non-null, is + * incremented once per skipped segment. + * + * A body with NO intersecting range degenerates to a single memcmp of the + * whole extent — the original behaviour, unchanged. */ +int psx_kernel_patch_cmp(const PsxKernelPatchRange *ranges, uint32_t n, + const uint8_t *ram, const uint8_t *rom, + uint32_t ram_lo, uint32_t rom_off, + uint32_t lo, uint32_t hi, + uint64_t *skips_out); + +/* Does a declared range END at this RAM address? The emitter registered each + * range's hi as a continuation key, so the interpreter hands straight-line + * flow back to static dispatch there and only the patched words interpret. */ +int psx_kernel_patch_ends_at(const PsxKernelPatchRange *ranges, uint32_t n, + uint32_t phys); + +#ifdef __cplusplus +} +#endif + +#endif /* PSX_KERNEL_PATCH_RANGES_H */ diff --git a/runtime/include/psx_bios_backend.h b/runtime/include/psx_bios_backend.h index c5e1842a..62398197 100644 --- a/runtime/include/psx_bios_backend.h +++ b/runtime/include/psx_bios_backend.h @@ -49,6 +49,13 @@ typedef struct PsxBiosBackend { * the generated dispatch itself, so it stays static there. */ const PsxKernelBody *kernel_bodies; uint32_t kernel_body_count; + + /* Kernel-RAM ranges the guest legitimately patches at runtime, from this + * image's [[recompiler.install_slots]] (psx_bios_image.h). The bless + * verifier skips them and the dirty-RAM interpreter resumes native at + * each range's hi. Null/0 for an image that declares no slots. */ + const PsxKernelPatchRange *kernel_patch_ranges; + uint32_t kernel_patch_range_count; } PsxBiosBackend; /* The backend in use. Null before psx_bios_select() runs; every forwarder and diff --git a/runtime/include/psx_bios_image.h b/runtime/include/psx_bios_image.h index e5301ef6..b2ae401d 100644 --- a/runtime/include/psx_bios_image.h +++ b/runtime/include/psx_bios_image.h @@ -28,12 +28,34 @@ typedef struct { uint32_t body_hi; /* exclusive */ } PsxKernelBody; +/* Kernel-RAM ranges the guest is EXPECTED to overwrite at runtime: the Psy-Q + * libapi patchers (_patch_gte / _patch_card / _patch_card2 / _patch_pad) and + * the BIOS's own install stubs rewrite words inside compiled kernel bodies at + * boot. Declared per image by [[recompiler.install_slots]] and emitted next to + * the body table. + * + * Two consumers, one table: + * - memory.c's bless verifier compares a body in segments that SKIP these + * ranges, so a body with a live patch inside it can still run native. + * - dirty_ram_interp.c surfaces back to static dispatch when straight-line + * interpretation reaches a range's `hi`, which the emitter registered as a + * continuation key. Only the patched words themselves interpret. + * + * This is not HLE: the guest's patched instructions still execute as written, + * on the dirty-RAM interpreter. Only the code AROUND them changes backend. */ +typedef struct { + uint32_t lo; /* RAM extent of the patched words [lo, hi) */ + uint32_t hi; /* exclusive; also the native resume key */ +} PsxKernelPatchRange; + /* Published from the ACTIVE backend at selection (psx_bios_backend.c), * hence a pointer rather than an array: a build links more than one * recompiled BIOS and each carries its own table. Indexing is unchanged * for consumers. */ extern const PsxKernelBody *psx_bios_kernel_bodies; extern uint32_t psx_bios_kernel_body_count; +extern const PsxKernelPatchRange *psx_bios_kernel_patch_ranges; +extern uint32_t psx_bios_kernel_patch_range_count; /* Native call-stub extents (A0/B0/C0). Layout shared with the generated * dispatch, which keeps its own static table. */ diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index dc2fc5d2..77b89e97 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -322,6 +322,7 @@ set(PSXRECOMP_RUNTIME_SOURCES ${PSXRECOMP_ROOT}/runtime/src/psx_sdl_audio.cpp ${PSXRECOMP_ROOT}/runtime/src/psx_stick.c ${PSXRECOMP_ROOT}/runtime/src/memory.c + ${PSXRECOMP_ROOT}/runtime/src/kernel_patch_ranges.c ${PSXRECOMP_ROOT}/runtime/src/gpu.c ${PSXRECOMP_ROOT}/runtime/src/ws_ui_group.c ${PSXRECOMP_ROOT}/runtime/src/ws_aspect_cone_math.c diff --git a/runtime/src/debug_server.c b/runtime/src/debug_server.c index 48ab208f..cb9dda37 100644 --- a/runtime/src/debug_server.c +++ b/runtime/src/debug_server.c @@ -7735,20 +7735,24 @@ static void handle_ws_hud_mode(int id, const char *json) } /* Kernel-image bless state: {"cmd":"kernel_bless"} -> - * entries/clean/mismatch/native_hits/verifies/invalidations. + * entries/clean/mismatch/native_hits/verifies/invalidations, plus the + * declared kernel patch ranges and the segments they made the verifier + * skip (psx_bios_kernel_patch_ranges). * (memory.c psx_kernel_bless_*; PSX_KERNEL_BLESS=0 disables the mechanism.) */ static void handle_kernel_bless(int id, const char *json) { - extern void psx_kernel_bless_stats(uint64_t out[6]); + extern void psx_kernel_bless_stats(uint64_t out[8]); (void)json; - uint64_t s[6]; + uint64_t s[8]; psx_kernel_bless_stats(s); send_fmt("{\"id\":%d,\"ok\":true,\"entries\":%llu,\"clean\":%llu," "\"mismatch\":%llu,\"native_hits\":%llu,\"verifies\":%llu," - "\"invalidations\":%llu}", + "\"invalidations\":%llu,\"patch_ranges\":%llu," + "\"patch_skips\":%llu}", id, (unsigned long long)s[0], (unsigned long long)s[1], (unsigned long long)s[2], (unsigned long long)s[3], - (unsigned long long)s[4], (unsigned long long)s[5]); + (unsigned long long)s[4], (unsigned long long)s[5], + (unsigned long long)s[6], (unsigned long long)s[7]); } static void handle_ws_margin(int id, const char *json) diff --git a/runtime/src/dirty_ram_interp.c b/runtime/src/dirty_ram_interp.c index dbdfdb72..7ba7498f 100644 --- a/runtime/src/dirty_ram_interp.c +++ b/runtime/src/dirty_ram_interp.c @@ -3322,9 +3322,27 @@ static int dirty_ram_dispatch_inner(CPUState* cpu, uint32_t addr, uint32_t stop_ g_dirty_interp_chain_target = pc; OV_FPLOG_RET1(); } + uint32_t next_phys = pc & 0x1FFFFFFFu; + /* A declared kernel patch range ends here (Rule 18 + the profile's + * [[recompiler.install_slots]]). The emitted body dispatched us in at + * the range's lo to run the guest's patched words; the emitter + * registered this PC as a continuation key and the bless verifier + * skips the patched words, so the rest of the body runs native. + * Without this hand-back the kernel page stays dirty and straight- + * line flow would interpret the whole function — which is why + * declaring the slot alone never paid off. */ + if (next_phys < DIRTY_RAM_KERNEL_WINDOW_END && + psx_kernel_patch_range_ends_at(next_phys) && + psx_kernel_bless_dispatchable(next_phys)) { + cpu->pc = pc; + g_dirty_ram_native_handoffs++; + g_dirty_ram_blocks_run++; + if (pc_entry) pc_entry->insns += (uint64_t)insns_executed; + g_dirty_interp_chain_target = pc; + OV_FPLOG_RET1(); + } /* Straight-line code that left the dirty page — hand back to * static dispatch by setting cpu->pc and returning. */ - uint32_t next_phys = pc & 0x1FFFFFFFu; uint32_t next_page = next_phys >> 12; if ((!current_page_dirty || next_page != current_page) && !dirty_ram_is_dirty(next_phys)) { diff --git a/runtime/src/kernel_patch_ranges.c b/runtime/src/kernel_patch_ranges.c new file mode 100644 index 00000000..4937f5d5 --- /dev/null +++ b/runtime/src/kernel_patch_ranges.c @@ -0,0 +1,40 @@ +/* kernel_patch_ranges.c — see kernel_patch_ranges.h for the contract. */ + +#include "kernel_patch_ranges.h" + +#include + +int psx_kernel_patch_cmp(const PsxKernelPatchRange *ranges, uint32_t n, + const uint8_t *ram, const uint8_t *rom, + uint32_t ram_lo, uint32_t rom_off, + uint32_t lo, uint32_t hi, + uint64_t *skips_out) +{ + uint32_t at = lo; + if (hi <= lo) return 0; + for (uint32_t i = 0; i < n && at < hi; i++) { + uint32_t plo = ranges[i].lo, phi = ranges[i].hi; + if (phi <= at) continue; /* range lies behind the cursor */ + if (plo >= hi) break; /* sorted: no later range intersects */ + if (plo > at && + memcmp(ram + at, rom + rom_off + (at - ram_lo), plo - at) != 0) + return 1; + /* Skip the patched words. A range may start below `lo` or end above + * `hi`; clamp so the cursor only ever advances inside the body. */ + at = phi > hi ? hi : phi; + if (skips_out) (*skips_out)++; + } + if (at < hi && + memcmp(ram + at, rom + rom_off + (at - ram_lo), hi - at) != 0) + return 1; + return 0; +} + +int psx_kernel_patch_ends_at(const PsxKernelPatchRange *ranges, uint32_t n, + uint32_t phys) +{ + for (uint32_t i = 0; i < n; i++) { + if (ranges[i].hi == phys) return 1; + } + return 0; +} diff --git a/runtime/src/memory.c b/runtime/src/memory.c index 75f08675..f731d25b 100644 --- a/runtime/src/memory.c +++ b/runtime/src/memory.c @@ -193,9 +193,15 @@ static inline void dirty_ram_mark_page(uint32_t phys) { * through the extern each time. A BIOS with no bless window exports all * zeros: the span-0 range test then rejects every address. */ #include "psx_bios_image.h" +#include "kernel_patch_ranges.h" static uint32_t s_kb_lo = 0, s_kb_span = 0, s_kb_rom_off = 0; +static const PsxKernelPatchRange* s_kb_pr = 0; +static uint32_t s_kb_pr_n = 0; +static uint64_t kbless_patch_skips = 0; /* segments skipped, TCP counter */ + + #define KBLESS_UNKNOWN 0u #define KBLESS_CLEAN 1u #define KBLESS_MISMATCH 2u @@ -216,6 +222,19 @@ static int kbless_on(void) { s_kb_lo = psx_bios_image.kbless_ram_lo; s_kb_span = psx_bios_image.kbless_ram_hi - psx_bios_image.kbless_ram_lo; s_kb_rom_off = psx_bios_image.kbless_rom_off; + s_kb_pr = psx_bios_kernel_patch_ranges; + s_kb_pr_n = psx_bios_kernel_patch_ranges ? + psx_bios_kernel_patch_range_count : 0u; + /* PSX_KERNEL_PATCH_RANGES=0 drops the declared ranges, restoring the + * whole-body memcmp. Same purpose as PSX_KERNEL_BLESS=0 one level up: + * an A/B instrument. With the ranges dropped a patched body mismatches + * forever and its emitted hook is unreachable, which is exactly the + * behaviour before the ranges existed — so one binary measures both + * sides. */ + { + const char* pe = getenv("PSX_KERNEL_PATCH_RANGES"); + if (pe && pe[0] == '0') s_kb_pr_n = 0; + } if (s_kb_span == 0) kbless_enabled = 0; /* BIOS with no bless window */ /* The emitted constants must agree with each other and the ROM * array: a window whose ROM source exceeds the image is a build @@ -231,6 +250,29 @@ static int kbless_on(void) { return kbless_enabled; } +/* Declared patch ranges (psx_bios_kernel_patch_ranges, emitted from the + * profile's [[recompiler.install_slots]]): kernel-RAM words the guest is + * EXPECTED to overwrite at boot. They sit inside compiled bodies, so the + * whole-body memcmp below used to fail forever on the first install — the + * reason Breath of Fire III's BIOS exception handler interpreted 1.22 billion + * instructions with its card stub sitting in the profile's declared slot. + * The body is verified in segments that skip them instead: CLEAN now means + * every byte OUTSIDE every declared range still matches the ROM source, which + * is exactly the claim the native code depends on. The patched words + * themselves still execute on the dirty-RAM interpreter (Rule 18), entered + * through the emitted patch-range hook. + * + * Snapshotted on the same latch as the window constants. The decision itself + * lives in kernel_patch_ranges.c so it can be unit-tested without the + * runtime's globals. */ +/* Does a declared patch range END at this RAM address? The emitter registered + * that PC as a continuation key, so the dirty-RAM interpreter hands straight- + * line flow back to static dispatch there and only the patched words + * interpret (dirty_ram_interp.c). */ +int psx_kernel_patch_range_ends_at(uint32_t phys) { + return psx_kernel_patch_ends_at(s_kb_pr, s_kb_pr_n, phys); +} + /* Binary search the (key-sorted) body table. -1 if absent. */ static int kbless_find(uint32_t phys) { uint32_t lo = 0, hi = psx_bios_kernel_body_count; @@ -255,9 +297,10 @@ int psx_kernel_bless_dispatchable(uint32_t phys) { if (st == KBLESS_MISMATCH) return 0; const PsxKernelBody* b = &psx_bios_kernel_bodies[i]; kbless_verifies++; - if (memcmp(ram + b->body_lo, - bios_rom + s_kb_rom_off + (b->body_lo - s_kb_lo), - b->body_hi - b->body_lo) == 0) { + if (psx_kernel_patch_cmp(s_kb_pr, s_kb_pr_n, ram, bios_rom, + s_kb_lo, s_kb_rom_off, + b->body_lo, b->body_hi, + &kbless_patch_skips) == 0) { kbless_state[i] = KBLESS_CLEAN; kbless_native_hits++; return 1; @@ -308,7 +351,7 @@ void psx_kernel_bless_note_range(uint32_t phys, uint32_t len) { } } -void psx_kernel_bless_stats(uint64_t out[6]) { +void psx_kernel_bless_stats(uint64_t out[8]) { uint32_t n = psx_bios_kernel_body_count; uint32_t clean = 0, mism = 0; if (n > KBLESS_MAX_ENTRIES) n = KBLESS_MAX_ENTRIES; @@ -322,6 +365,11 @@ void psx_kernel_bless_stats(uint64_t out[6]) { out[3] = kbless_native_hits; out[4] = kbless_verifies; out[5] = kbless_invalidations; + /* Declared patch ranges, and how many segments the verifier skipped + * because of them: the proof that a body with a live install stub is + * being blessed rather than failing forever. */ + out[6] = s_kb_pr_n; + out[7] = kbless_patch_skips; } void psx_kernel_bless_resync_after_restore(void) { diff --git a/runtime/src/psx_bios_backend.c b/runtime/src/psx_bios_backend.c index c6eb0ba5..d00340be 100644 --- a/runtime/src/psx_bios_backend.c +++ b/runtime/src/psx_bios_backend.c @@ -31,6 +31,8 @@ PsxBiosImageInfo psx_bios_image; const PsxKernelBody *psx_bios_kernel_bodies = 0; uint32_t psx_bios_kernel_body_count = 0; +const PsxKernelPatchRange *psx_bios_kernel_patch_ranges = 0; +uint32_t psx_bios_kernel_patch_range_count = 0; /* Dispatch nesting depth. Shared dispatch state, not per-image: each generated * dispatch used to define its own copy, which is precisely why two of them @@ -84,6 +86,8 @@ int psx_bios_activate(const PsxBiosBackend *backend) psx_bios_image = *backend->image; psx_bios_kernel_bodies = backend->kernel_bodies; psx_bios_kernel_body_count = backend->kernel_body_count; + psx_bios_kernel_patch_ranges = backend->kernel_patch_ranges; + psx_bios_kernel_patch_range_count = backend->kernel_patch_range_count; /* Soft-return rematch can switch OPENBIOS ↔ SCPH without process exit. * Drop the prior image's call-HLE / boot-skip hook immediately so a * sticky SCPH DeliverEvent path cannot run against OpenBIOS ROM bytes diff --git a/runtime/src/psx_selfcheck.c b/runtime/src/psx_selfcheck.c index c6203333..a35f840c 100644 --- a/runtime/src/psx_selfcheck.c +++ b/runtime/src/psx_selfcheck.c @@ -753,7 +753,7 @@ static void sc_do_load(struct CPUState *cpu, const char *via) extern uint64_t g_guest_store_count; extern uint64_t g_irqctx_seq; extern uint32_t dirty_ram_text_diverged_pages(void); - uint64_t kb[6]; + uint64_t kb[8]; const int next_pass = s_warming ? 0 : (s_pass == 0 ? 1 : s_pass + 1); psx_kernel_bless_stats(kb); fprintf(stderr, diff --git a/runtime/tests/test_kernel_patch_ranges.c b/runtime/tests/test_kernel_patch_ranges.c new file mode 100644 index 00000000..e315e770 --- /dev/null +++ b/runtime/tests/test_kernel_patch_ranges.c @@ -0,0 +1,136 @@ +/* test_kernel_patch_ranges.c — the kernel-bless verifier must accept a body + * whose DECLARED patch range has been overwritten, and still reject one + * changed anywhere else. + * + * The regression this pins is the whole point of the mechanism. The verifier + * used to memcmp a body's entire extent against the ROM source, so the + * instant the guest's Psy-Q patcher wrote its install stub, the body + * mismatched and every dispatch to it ran on the interpreter for the life of + * the process. On Breath of Fire III that was the BIOS exception handler: + * 1.22 billion interpreted instructions, 44.6 % of all interpreted work, with + * the stub sitting at exactly the address the BIOS profile already declared. + * Declaring a slot changed nothing because the verifier had never heard of + * slots (`grep install_slot runtime/` returned nothing). + * + * Rule 18 is not relaxed here: a patched word is skipped by the VERIFIER, not + * by execution. The emitted hook still dispatches those words to the + * dirty-RAM interpreter, which runs the guest's own instructions. + */ +#include "kernel_patch_ranges.h" + +#include +#include + +static int s_fails = 0; + +static void expect(int cond, const char *what) { + if (!cond) { + fprintf(stderr, "FAIL: %s\n", what); + s_fails++; + } +} + +/* A synthetic 256-byte kernel body at RAM 0x500, sourced from ROM offset + * 0x10000 — the same shape as SCPH-1001's "Kernel Part 2" copy. */ +#define RAM_LO 0x500u +#define ROM_OFF 0x10000u +#define BODY_LO 0x500u +#define BODY_HI 0x600u + +static uint8_t ram[0x800]; +static uint8_t rom[0x11000]; + +static void reset(void) { + /* Byte-verbatim copy: the claim every kernel body depends on. */ + for (unsigned i = 0; i < sizeof ram; i++) ram[i] = (uint8_t)(i * 7u + 3u); + memset(rom, 0xAA, sizeof rom); + memcpy(rom + ROM_OFF, ram + RAM_LO, sizeof ram - RAM_LO); +} + +static int cmp(const PsxKernelPatchRange *r, uint32_t n, uint64_t *skips) { + return psx_kernel_patch_cmp(r, n, ram, rom, RAM_LO, ROM_OFF, + BODY_LO, BODY_HI, skips); +} + +int main(void) { + /* ---- 1. An untouched body verifies, with and without ranges -------- */ + const PsxKernelPatchRange one[] = { { 0x540u, 0x550u } }; + reset(); + expect(cmp(0, 0, 0) == 0, "clean body, no ranges declared"); + expect(cmp(one, 1, 0) == 0, "clean body, one range declared"); + + /* ---- 2. A word inside a DECLARED range does not unbless the body --- */ + reset(); + ram[0x544u] ^= 0xFFu; /* inside [0x540, 0x550) */ + expect(cmp(one, 1, 0) == 0, "patched word inside a declared range: CLEAN"); + expect(cmp(0, 0, 0) != 0, + "the same word with NO range declared: MISMATCH (the old behaviour)"); + + /* Every word of the range, and both its edges. */ + reset(); + for (uint32_t a = 0x540u; a < 0x550u; a++) ram[a] ^= 0xFFu; + expect(cmp(one, 1, 0) == 0, "whole declared range overwritten: CLEAN"); + + /* ---- 3. A word OUTSIDE the range still unblesses ------------------- */ + reset(); + ram[0x53Cu] ^= 0xFFu; /* one byte below the range */ + expect(cmp(one, 1, 0) != 0, "word just below the range: MISMATCH"); + reset(); + ram[0x550u] ^= 0xFFu; /* first byte above the range */ + expect(cmp(one, 1, 0) != 0, "word just above the range: MISMATCH"); + reset(); + ram[BODY_LO] ^= 0xFFu; + expect(cmp(one, 1, 0) != 0, "first word of the body: MISMATCH"); + reset(); + ram[BODY_HI - 1u] ^= 0xFFu; + expect(cmp(one, 1, 0) != 0, "last word of the body: MISMATCH"); + + /* ---- 4. Several ranges, and ranges outside this body --------------- */ + /* Sorted and non-overlapping, as the emitter guarantees. 0x700 lies past + * the body: it must neither be skipped into nor stop the walk early. */ + const PsxKernelPatchRange many[] = { + { 0x510u, 0x520u }, { 0x540u, 0x550u }, { 0x5F0u, 0x600u }, + { 0x700u, 0x710u }, + }; + reset(); + ram[0x514u] ^= 0xFFu; + ram[0x548u] ^= 0xFFu; + ram[0x5F8u] ^= 0xFFu; /* a range flush against body_hi */ + expect(cmp(many, 4, 0) == 0, "three declared ranges patched: CLEAN"); + ram[0x530u] ^= 0xFFu; /* in the gap between ranges */ + expect(cmp(many, 4, 0) != 0, "a gap between declared ranges still checked"); + + /* A range that starts before the body still clamps correctly. */ + const PsxKernelPatchRange spanning[] = { { 0x4F0u, 0x510u } }; + reset(); + ram[0x504u] ^= 0xFFu; + expect(cmp(spanning, 1, 0) == 0, "range starting below body_lo: CLEAN"); + + /* ---- 5. The skip counter reports real work ------------------------- */ + { + uint64_t skips = 0; + reset(); + (void)cmp(many, 4, &skips); + expect(skips == 3u, "skipped exactly the three intersecting ranges"); + skips = 0; + (void)cmp(0, 0, &skips); + expect(skips == 0u, "no ranges declared: nothing skipped"); + } + + /* ---- 6. The interpreter hand-back predicate ------------------------ */ + expect(psx_kernel_patch_ends_at(many, 4, 0x550u) == 1, + "a range end is a resume PC"); + expect(psx_kernel_patch_ends_at(many, 4, 0x540u) == 0, + "a range START is not a resume PC"); + expect(psx_kernel_patch_ends_at(many, 4, 0x548u) == 0, + "an interior word is not a resume PC"); + expect(psx_kernel_patch_ends_at(0, 0, 0x550u) == 0, + "no ranges: no resume PCs"); + + if (s_fails == 0) { + printf("test_kernel_patch_ranges: all checks passed\n"); + return 0; + } + fprintf(stderr, "test_kernel_patch_ranges: %d failure(s)\n", s_fails); + return 1; +} From baca0a8a4274f71669b5f10ad0c6802a30d98694 Mon Sep 17 00:00:00 2001 From: Kevin <9866117+kerokline@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:57:01 -0400 Subject: [PATCH 3/9] tools: find install-slot ranges and A/B what they buy, for any title The install-slot range work is only trustworthy if a second title can reproduce it, and the two tools that produced its evidence lived in one game repo with that game's paths baked in. Moved into the framework and generalised: every path defaults off --project-root (cwd) and the BIOS profile's out_stem, and the exe is found by globbing the build tree, because its name is derived from the window title and so differs per title. Run either from a game project root. kernel_patch_diff.py boots headless and diffs live kernel RAM against the ROM copy it came from, joining each differing word to the compiled kernel bodies. It reads the emitted PsxKernelPatchRange table, so it separates a DECLARED patch (the verifier skips it, the body still runs native) from one that genuinely unblesses a body. Without that split its verdict line calls every declared range "these interpret", which is exactly backwards once the ranges work. This is the tool you use to find ranges for a new image or a new Psy-Q release. kernel_patch_ab.py answers "what did declaring them buy". It boots the same binary twice to the same frame, once with the ranges live and once with PSX_KERNEL_PATCH_RANGES=0, which drops them and restores the whole-body bless memcmp. Holding the binary constant holds the overlays, the codegen and the disc constant too, so only the bless verdict moves. Headless runs uncapped, so it normalises per frame rather than comparing raw totals at different frame counts. Two things it gets right that cost real time to learn: - It does NOT discount the A0/B0/C0 vectors from the headline. An earlier version did, reasoning that they interpret on both sides and would mask the result. True on one title, false in general: on Mega Man X6 with OpenBIOS the B0 vector burns 2,361,470 instructions over 1,180,735 entries with the ranges off and vanishes entirely with them on, because a clean bless verdict lets the vector body dispatch native. Discounting it hid a 130 insn/frame win. Kernel and all-interp are the headline; the bodies/vectors split is printed underneath as diagnosis. - It fails fast on a wedge, detected as a frame counter that stops advancing while the debug server still answers, instead of burning the whole timeout. That is how a dispatch key inside a declared range was caught spinning the dispatch loop at frame 0. debug_client.query grew an optional timeout so these can use it instead of carrying their own transport; the 10 s default is too short for dirty_ram_stats and a whole-window read_ram. Co-Authored-By: Claude Opus 5 --- tools/debug_client.py | 11 +- tools/kernel_patch_ab.py | 195 +++++++++++++++++++++++++++++ tools/kernel_patch_common.py | 160 ++++++++++++++++++++++++ tools/kernel_patch_diff.py | 231 +++++++++++++++++++++++++++++++++++ 4 files changed, 594 insertions(+), 3 deletions(-) create mode 100644 tools/kernel_patch_ab.py create mode 100644 tools/kernel_patch_common.py create mode 100644 tools/kernel_patch_diff.py diff --git a/tools/debug_client.py b/tools/debug_client.py index 4d57997c..60459fb1 100644 --- a/tools/debug_client.py +++ b/tools/debug_client.py @@ -143,9 +143,14 @@ def send_cmd(sock, cmd_dict): return json.loads(buf.decode().strip()) -def query(host, port, cmd_dict): - """One-shot: connect, send, receive, close.""" - s = connect(host, port) +def query(host, port, cmd_dict, timeout=10.0): + """One-shot: connect, send, receive, close. + + `timeout` is per-socket-operation. The 10 s default suits the small + commands; bulk ones (dirty_ram_stats walks a large PC table, read_ram of a + whole window) need more, so callers that ask for those must raise it. + """ + s = connect(host, port, timeout) try: return send_cmd(s, cmd_dict) finally: diff --git a/tools/kernel_patch_ab.py b/tools/kernel_patch_ab.py new file mode 100644 index 00000000..764639ac --- /dev/null +++ b/tools/kernel_patch_ab.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""A/B the declared kernel install-slot ranges: what do they buy this game? + +Boots the game twice against the SAME binary, to the same frame, once with the +profile's [[recompiler.install_slots]] ranges live and once with +PSX_KERNEL_PATCH_RANGES=0, which drops them and restores the whole-body +kernel-bless memcmp - the behaviour before the ranges existed. One binary +measures both sides, so the overlays, the codegen and the disc are held +constant and only the bless verdict moves. + +Run from a game project root: + + python psxrecomp/tools/kernel_patch_ab.py + python psxrecomp/tools/kernel_patch_ab.py --bios psxrecomp/bios/SCPH1001.BIN + python psxrecomp/tools/kernel_patch_ab.py --frames 20000 + +Set PSX_BIOS_HLE=0 to isolate this change from the kernel-call HLE tier. That +matters on an image exporting a DeliverEvent anchor (retail SCPH-1001); +OpenBIOS has none, so its kernel calls are LLE either way. + +Headless runs uncapped, so a frame is a unit of guest work rather than wall +time, and the two sides reach slightly different frames in the same wall +budget - the report normalises per frame. + +Read "kernel" and "all" as the result. The bodies/vectors split underneath is +diagnosis: whether the A0/B0/C0 vectors interpret on both sides is +title-dependent (see kernel_patch_common.VECTOR_STUBS), so they are shown but +never discounted. +""" +import argparse +import json +import os +import subprocess +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import kernel_patch_common as kp + + +def wait_port(host, port, secs=180): + t0 = time.time() + while time.time() - t0 < secs: + try: + r = kp.q("frame", host, port, timeout=30.0) + if r.get("ok") or "frame" in r: + return + except Exception: + pass + time.sleep(1) + raise SystemExit("debug port never came up") + + +def run_side(a, root, ranges_on): + env = dict(os.environ, PSX_STARVATION_TIMEOUT_US="0") + env.pop("PSX_LOAD_SLOT", None) + if ranges_on: + env.pop("PSX_KERNEL_PATCH_RANGES", None) + else: + env["PSX_KERNEL_PATCH_RANGES"] = "0" + argv = [os.path.join(root, a.exe), "--game", a.game, "--no-launcher", + "--headless", "--debug-port", str(a.port)] + if a.bios: + argv += ["--bios", os.path.join(root, a.bios)] + proc = subprocess.Popen(argv, cwd=root, env=env, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0)) + try: + wait_port(a.host, a.port) + t0 = time.time() + # A wedge shows up as a frame counter that stops advancing while the + # debug server still answers, so watch for STALL as well as the overall + # timeout - a boot that hangs at frame 0 should not burn the whole + # budget before saying so. It did once: a dispatch key inside a + # declared range spun the dispatch loop forever. + last_fr, last_move = -1, time.time() + while True: + try: + fr = kp.q("frame", a.host, a.port, timeout=30.0).get("frame") or 0 + except Exception: + fr = last_fr # transient; the stall check still applies + if fr >= a.frames: + break + if fr != last_fr: + last_fr, last_move = fr, time.time() + elif time.time() - last_move > a.stall: + raise SystemExit("WEDGED: frame stuck at %s for %ds (target %d)" + % (fr, a.stall, a.frames)) + if time.time() - t0 > a.timeout: + raise SystemExit("only reached frame %s of %d in %ds" + % (fr, a.frames, a.timeout)) + time.sleep(1.0) + kb = kp.q("kernel_bless", a.host, a.port) + st = kp.q("dirty_ram_stats", a.host, a.port) + rows = [r for r in st.get("per_pc", []) + if int(r["pc"], 16) < kp.KERNEL_WINDOW_END] + kernel = sum(r["insns"] for r in rows) + tramp = sum(r["insns"] for r in rows + if int(r["pc"], 16) in kp.VECTOR_STUBS) + top = sorted(rows, key=lambda r: -r["insns"])[:12] + return { + "ranges_on": ranges_on, + "frame": fr, + "wall_s": round(time.time() - t0, 1), + "bless_entries": kb.get("entries"), + "bless_clean": kb.get("clean"), + "bless_mismatch": kb.get("mismatch"), + "bless_native_hits": kb.get("native_hits"), + "patch_ranges": kb.get("patch_ranges"), + "patch_skips": kb.get("patch_skips"), + "interp_insns_total": st.get("insns_run"), + "interp_insns_kernel": kernel, + "interp_insns_vectors": tramp, + "interp_insns_kernel_bodies": kernel - tramp, + "native_handoffs": st.get("native_handoffs"), + "top_kernel_pcs": [{"pc": r["pc"], "insns": r["insns"], + "entries": r["entries"]} for r in top], + } + finally: + proc.kill() + time.sleep(2) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split(chr(10) + chr(10), 1)[0]) + kp.add_project_args(ap, default_port=4372) + ap.add_argument("--frames", type=int, default=11000, + help="frame target for both sides (default 11000)") + ap.add_argument("--timeout", type=int, default=900) + ap.add_argument("--stall", type=int, default=90, + help="fail if the frame counter does not advance for this long") + ap.add_argument("--out", default="analysis/kernel_patch_ab.json") + a = ap.parse_args() + root = kp.resolve(a) + + pranges = kp.parse_patch_ranges(os.path.join(root, a.dispatch)) + print("%s declares %d install-slot range(s): %s" + % (os.path.basename(a.dispatch), len(pranges), + ", ".join("0x%04X..0x%04X" % r for r in pranges) or "none"), flush=True) + if not pranges: + raise SystemExit("nothing to A/B: this backend declares no ranges. Add " + "[[recompiler.install_slots]] to the BIOS profile and " + "regenerate (see docs/dynamic_handler_install.md).") + + rows = [] + for ranges_on in (False, True): + print("=== %s" % ("ranges ON (declared install slots)" if ranges_on else + "ranges OFF (PSX_KERNEL_PATCH_RANGES=0, pre-change " + "behaviour)"), flush=True) + r = run_side(a, root, ranges_on) + rows.append(r) + print(" frame %(frame)s in %(wall_s)ss bless: %(bless_clean)s clean / " + "%(bless_mismatch)s mismatch of %(bless_entries)s " + "ranges=%(patch_ranges)s skips=%(patch_skips)s" % r, flush=True) + print(" interpreted: %(interp_insns_total)s total, " + "%(interp_insns_kernel)s kernel RAM" % r, flush=True) + for t in r["top_kernel_pcs"][:6]: + print(" %s insns=%-10s entries=%s" + % (t["pc"], t["insns"], t["entries"]), flush=True) + + off, on = rows[0], rows[1] + print() + print("frames: OFF %s, ON %s; counts below are PER FRAME, since headless " + "runs uncapped" % (off["frame"], on["frame"])) + print("%-30s %14s %14s %9s" % ("", "ranges OFF", "ranges ON", "change")) + + def line(label, key, per_frame=True): + o, n = off.get(key) or 0, on.get(key) or 0 + if per_frame: + o = o / float(off["frame"] or 1) + n = n / float(on["frame"] or 1) + os_, ns_ = "%.1f" % o, "%.1f" % n + else: + os_, ns_ = str(o), str(n) + ch = ("%+.1f%%" % (100.0 * (n - o) / o)) if o else "n/a" + print("%-30s %14s %14s %9s" % (label, os_, ns_, ch)) + + line("kernel-bless mismatch", "bless_mismatch", False) + line("kernel-bless clean", "bless_clean", False) + line("interp insns / frame, kernel", "interp_insns_kernel") + line("interp insns / frame, all", "interp_insns_total") + print(" of which (diagnostic, not discounted):") + line(" kernel bodies", "interp_insns_kernel_bodies") + line(" A0/B0/C0 vectors", "interp_insns_vectors") + + out = os.path.join(root, a.out) + os.makedirs(os.path.dirname(out), exist_ok=True) + json.dump({"frames": a.frames, "bios": a.bios, "dispatch": a.dispatch, + "declared_ranges": ["0x%04X..0x%04X" % r for r in pranges], + "sides": rows}, + open(out, "w", encoding="utf-8"), indent=1) + print(chr(10) + "wrote " + a.out) + + +if __name__ == "__main__": + main() diff --git a/tools/kernel_patch_common.py b/tools/kernel_patch_common.py new file mode 100644 index 00000000..958ba023 --- /dev/null +++ b/tools/kernel_patch_common.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""kernel_patch_common.py - shared plumbing for the kernel install-slot tools. + +Both kernel_patch_diff.py (which kernel-RAM words does this game patch?) and +kernel_patch_ab.py (what do the declared ranges buy?) run a GAME project +headless and talk to its debug server. This module holds the three things they +would otherwise each get subtly wrong: locating the project's exe, reading the +BIOS profile, and reading the emitted tables back out of the generated C. + +Run both tools from a game project root (the directory holding game.toml and +the framework checkout), not from the framework: + + python psxrecomp/tools/kernel_patch_diff.py + python psxrecomp/tools/kernel_patch_ab.py --bios psxrecomp/bios/SCPH1001.BIN + +See docs/dynamic_handler_install.md for what the ranges are and how to find +them for a new image. +""" +import glob +import os +import re +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import debug_client + +# Kernel RAM below this is the relocated BIOS copy (dirty_ram_interp.h's +# DIRTY_RAM_KERNEL_WINDOW_END). +KERNEL_WINDOW_END = 0x10000 + +# The four 16-byte kernel call vectors, reported on their own line by the A/B. +# +# They are NOT discounted from its headline. An earlier version subtracted them +# on the grounds that they interpret on both sides and would mask the result. +# That is true of some titles and false in general: on Mega Man X6 with +# OpenBIOS the B0 vector burns 2,361,470 instructions over 1,180,735 entries +# with the ranges off and vanishes entirely with them on, because once the +# bless verdict is clean the vector body dispatches native. Subtracting it hid +# a 130 insn/frame win. +VECTOR_STUBS = (0x80, 0xA0, 0xB0, 0xC0) + + +def add_project_args(ap, default_port): + """The arguments every kernel-patch tool takes.""" + ap.add_argument("--project-root", default=".", + help="game project root (holds game.toml); default cwd") + ap.add_argument("--framework", default="psxrecomp", + help="framework checkout, relative to --project-root") + ap.add_argument("--build-dir", default="build-relprof", + help="build tree to launch from, relative to --project-root") + ap.add_argument("--exe", default=None, + help="runtime exe; default: the single .exe in --build-dir") + ap.add_argument("--game", default="game.toml", + help="game config passed to --game") + ap.add_argument("--profile", default=None, + help="BIOS profile toml; default /bios/OpenBIOS.toml") + ap.add_argument("--dispatch", default=None, + help="generated _dispatch.c; default derived from --profile") + ap.add_argument("--bios", default=None, + help="BIOS image to launch with; default = the runtime's own pick") + ap.add_argument("--port", type=int, default=default_port) + ap.add_argument("--host", default="127.0.0.1") + + +def resolve(a): + """Fill in the derived paths on a parsed args namespace. Returns the root.""" + root = os.path.abspath(a.project_root) + fw = os.path.join(root, a.framework) + if not os.path.isdir(fw): + raise SystemExit("no framework checkout at %s (pass --framework)" % fw) + if a.profile is None: + a.profile = os.path.join(a.framework, "bios", "OpenBIOS.toml") + if a.dispatch is None: + stem = read_out_stem(os.path.join(root, a.profile)) + a.dispatch = os.path.join(a.framework, "generated", stem + "_dispatch.c") + if a.exe is None: + a.exe = find_exe(os.path.join(root, a.build_dir)) + for key in ("profile", "dispatch", "exe"): + p = os.path.join(root, getattr(a, key)) + if not os.path.exists(p): + raise SystemExit("--%s not found: %s" % (key, p)) + return root + + +def find_exe(build_dir): + """The game's runtime exe. Its name is derived from the window title, so it + differs per title; glob rather than guess, and refuse to choose.""" + if not os.path.isdir(build_dir): + raise SystemExit("no build dir at %s (pass --build-dir or --exe)" % build_dir) + found = [p for p in glob.glob(os.path.join(build_dir, "*.exe")) + if "_oracle" not in os.path.basename(p)] + if not found: + raise SystemExit("no .exe in %s - build psx-runtime first" % build_dir) + if len(found) > 1: + names = "\n ".join(os.path.basename(p) for p in found) + raise SystemExit("several exes in %s; pass --exe:\n %s" % (build_dir, names)) + return found[0] + + +def read_out_stem(profile_path): + """[recompiler] out_stem from a BIOS profile ("OpenBIOS", "SCPH1001").""" + txt = open(profile_path, encoding="utf-8").read() + m = re.search(r'^\s*out_stem\s*=\s*"([^"]+)"', txt, re.M) + if not m: + raise SystemExit("%s has no [recompiler] out_stem" % profile_path) + return m.group(1) + + +def parse_profile(path): + """(rom_path, [(name, rom_lo, ram_lo, length, kernel_bless)]) from a profile.""" + txt = open(path, encoding="utf-8").read() + rom = re.search(r'^rom\s*=\s*"([^"]+)"', txt, re.M).group(1) + copies = [] + for m in re.finditer(r'\[\[recompiler\.address_model\.copy\]\](.*?)(?=\n\[|\Z)', + txt, re.S): + blk = m.group(1) + g = lambda k: re.search(r'^\s*%s\s*=\s*"?([^"\n]+)"?' % k, + blk, re.M).group(1).strip() + rom_lo = int(g("rom_lo"), 16) + rom_hi = int(g("rom_hi"), 16) + copies.append((g("name"), rom_lo, int(g("ram_lo"), 16), + rom_hi - rom_lo, "true" in g("kernel_bless"))) + return rom, copies + + +def parse_bodies(dispatch_c): + """[(key, body_lo, body_hi)] from the emitted PsxKernelBody table.""" + txt = open(dispatch_c, encoding="utf-8", errors="replace").read() + m = re.search(r'psx_bios_kernel_bodies\[\d+\]\s*=\s*\{(.*?)\};', txt, re.S) + if not m: + return [] + return [(int(a, 16), int(b, 16), int(c, 16)) for a, b, c in + re.findall(r'\{\s*0x([0-9A-Fa-f]+)u,\s*0x([0-9A-Fa-f]+)u,' + r'\s*0x([0-9A-Fa-f]+)u\s*\}', m.group(1))] + + +def parse_patch_ranges(dispatch_c): + """[(lo, hi)] from the emitted PsxKernelPatchRange table. + + The profile's [[recompiler.install_slots]], couriered into the generated C. + A differing word INSIDE one of these is a declared patch: the bless + verifier skips it and the body still runs native. A differing word outside + every range is what actually unblesses a body. Empty for a backend + generated before install-slot ranges existed. + """ + txt = open(dispatch_c, encoding="utf-8", errors="replace").read() + m = re.search(r'psx_bios_kernel_patch_ranges\[\d+\]\s*=\s*\{(.*?)\};', txt, re.S) + if not m: + return [] + pairs = [(int(a, 16), int(b, 16)) for a, b in + re.findall(r'\{\s*0x([0-9A-Fa-f]+)u,\s*0x([0-9A-Fa-f]+)u\s*\}', + m.group(1))] + # An image with no slots emits one inert { 0, 0 } entry and a count of 0. + cnt = re.search(r'psx_bios_kernel_patch_range_count\s*=\s*(\d+)u', txt) + return pairs[:int(cnt.group(1))] if cnt else pairs + + +def q(cmd, host, port, timeout=120.0, **kw): + """One debug-server command. Bulk commands need the long default.""" + return debug_client.query(host, port, dict(cmd=cmd, **kw), timeout=timeout) diff --git a/tools/kernel_patch_diff.py b/tools/kernel_patch_diff.py new file mode 100644 index 00000000..0989fbdc --- /dev/null +++ b/tools/kernel_patch_diff.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Which kernel-RAM words does this game patch at runtime? + +Boots the game headless, then diffs live kernel RAM against the BIOS image's +boot-time copy (the [[recompiler.address_model.copy]] window with +kernel_bless = true) and reports every differing word, joined to the compiled +kernel bodies from the generated _dispatch.c. The output says which +FUNCTIONS carry a patch, and which of those still fail the bless check and so +run interpreted. + +This is how you find [[recompiler.install_slots]] ranges for a new image or a +new SDK version. Run from a game project root: + + python psxrecomp/tools/kernel_patch_diff.py # OpenBIOS + python psxrecomp/tools/kernel_patch_diff.py --at 5 --at 30 # two snapshots + python psxrecomp/tools/kernel_patch_diff.py --no-launch --port 4370 + python psxrecomp/tools/kernel_patch_diff.py \ + --bios psxrecomp/bios/SCPH1001.BIN \ + --profile psxrecomp/bios/SCPH1001.toml + +Snapshot twice. Some patches land at boot and some on the first card access, +and a range that only appears late is still a range. + +Reading the output: a differing word inside a compiled body and OUTSIDE every +declared range is what unblesses that body. A word inside a declared range is +already handled - the verifier skips it and the body runs native anyway. A +word in no body at all is data (event tables, TCB saves, pad tables); it +unblesses nothing, so do not declare it. A word whose writer you cannot +identify is a word you should not declare either: the declaration tells the +verifier to stop checking it (CLAUDE.md rule 14). + +See docs/dynamic_handler_install.md for the patch shapes, the resume kinds, +and how the three halves of the mechanism compose. +""" +import argparse +import json +import os +import struct +import subprocess +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import kernel_patch_common as kp + + +def wait_port(host, port, secs=180): + t0 = time.time() + while time.time() - t0 < secs: + try: + r = kp.q("frame", host, port, timeout=30.0) + if r.get("ok") or "frame" in r: + return time.time() - t0 + except Exception: + pass + time.sleep(1) + raise SystemExit("debug port never came up") + + +def parse_seeds(path): + """[(ram_addr, label)] for seeds, accepting the ROM-LMA form too.""" + d = json.load(open(path, encoding="utf-8")) + rows = d if isinstance(d, list) else (d.get("functions") or d.get("seeds") or []) + return [(int(r["address"], 16) & 0x1FFFFFFF, r.get("label", "?")) + for r in rows if isinstance(r, dict)] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split(chr(10) + chr(10), 1)[0]) + kp.add_project_args(ap, default_port=4371) + ap.add_argument("--seeds", default=None, + help="seeds json for function names; default from the profile") + ap.add_argument("--at", type=float, action="append", default=None, + help="seconds after the port is up to snapshot " + "(repeatable; default 8 and 30)") + ap.add_argument("--no-launch", action="store_true", + help="diff against an already-running game on --port") + ap.add_argument("--out", default="analysis/kernel_patch_diff.json") + a = ap.parse_args() + root = kp.resolve(a) + ats = a.at or [8.0, 30.0] + + rom_rel, copies = kp.parse_profile(os.path.join(root, a.profile)) + rom = open(os.path.join(root, a.framework, rom_rel), "rb").read() + bodies = kp.parse_bodies(os.path.join(root, a.dispatch)) + pranges = kp.parse_patch_ranges(os.path.join(root, a.dispatch)) + declared = lambda ram: any(lo <= ram < hi for lo, hi in pranges) + if pranges: + print("declared install-slot ranges (%s): %s" + % (os.path.basename(a.dispatch), + ", ".join("0x%04X..0x%04X" % r for r in pranges)), flush=True) + else: + print("no install-slot ranges declared in %s - every patched word " + "unblesses its body" % os.path.basename(a.dispatch), flush=True) + + if a.seeds is None: + import re + txt = open(os.path.join(root, a.profile), encoding="utf-8").read() + m = re.search(r'^\s*seeds\s*=\s*"([^"]+)"', txt, re.M) + a.seeds = os.path.join(a.framework, m.group(1)) if m else None + seeds = parse_seeds(os.path.join(root, a.seeds)) if a.seeds else [] + + # Map ROM-LMA seeds into RAM for each copy so a patched word can be named. + ram_names = {} + for name, rom_lo, ram_lo, ln, bless in copies: + for addr, label in seeds: + if rom_lo <= addr < rom_lo + ln: + ram_names[addr - rom_lo + ram_lo] = label + elif ram_lo <= addr < ram_lo + ln: + ram_names[addr] = label + + def fn_of(ram): + best = [k for k in ram_names if k <= ram] + return ("%s+0x%X" % (ram_names[max(best)], ram - max(best))) if best else "?" + + proc = None + if not a.no_launch: + env = dict(os.environ, PSX_STARVATION_TIMEOUT_US="0") + env.pop("PSX_LOAD_SLOT", None) + argv = [os.path.join(root, a.exe), "--game", a.game, "--no-launcher", + "--headless", "--debug-port", str(a.port)] + if a.bios: + argv += ["--bios", os.path.join(root, a.bios)] + proc = subprocess.Popen(argv, cwd=root, env=env, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0)) + print("launched pid %d" % proc.pid, flush=True) + t_up = wait_port(a.host, a.port) + print("port up after %.1fs" % t_up, flush=True) + t0 = time.time() + + report = {"copies": [], "snapshots": [], + "declared_ranges": ["0x%04X..0x%04X" % r for r in pranges]} + try: + for at in sorted(ats): + while time.time() - t0 < at: + time.sleep(0.5) + fr = kp.q("frame", a.host, a.port) + kb = kp.q("kernel_bless", a.host, a.port) + snap = {"t": round(time.time() - t0, 1), "frame": fr.get("frame"), + "kernel_bless": kb, "diffs": []} + print("\n=== t+%.0fs frame %s kernel_bless: %s" + % (at, fr.get("frame"), + {k: v for k, v in kb.items() if k not in ("id", "ok")}), + flush=True) + for name, rom_lo, ram_lo, ln, bless in copies: + if not bless: + continue # only the bless window gates native dispatch + live = bytes.fromhex(kp.q("read_ram", a.host, a.port, + addr="0x%08X" % ram_lo, len=ln)["hex"]) + src = rom[rom_lo - 0x1FC00000: rom_lo - 0x1FC00000 + ln] + words = [] + for off in range(0, ln - 3, 4): + if live[off:off + 4] != src[off:off + 4]: + ram = ram_lo + off + inb = [b for b in bodies if b[1] <= ram < b[2]] + words.append({ + "ram": "0x%04X" % ram, + "rom": struct.unpack(" live %08X %s" + % (w["ram"], w["rom"], w["live"], + "declared" if w["declared"] + else "NOT DECLARED"), flush=True) + snap["diffs"].append({"copy": name, "words": words, "runs": runs}) + report["snapshots"].append(snap) + + # Only an UNDECLARED patched word unblesses a body. + dirty, saved = set(), set() + for sd in snap["diffs"]: + for w in sd["words"]: + r = int(w["ram"], 16) + for key, lo, hi in bodies: + if lo <= r < hi: + (saved if w["declared"] else dirty).add((lo, hi)) + saved -= dirty # one undeclared word is enough to unbless + if saved: + print("compiled kernel bodies patched INSIDE a declared range " + "(these still run native):", flush=True) + for lo, hi in sorted(saved): + print(" 0x%04X..0x%04X %s" % (lo, hi, fn_of(lo)), flush=True) + if dirty: + print("compiled kernel bodies containing an UNDECLARED patched " + "word (these interpret):", flush=True) + for lo, hi in sorted(dirty): + print(" 0x%04X..0x%04X %s" % (lo, hi, fn_of(lo)), flush=True) + else: + print("no compiled kernel body carries an undeclared patched word", + flush=True) + snap["dirty_bodies"] = ["0x%04X..0x%04X %s" % (lo, hi, fn_of(lo)) + for lo, hi in sorted(dirty)] + snap["blessed_patched_bodies"] = ["0x%04X..0x%04X %s" % (lo, hi, fn_of(lo)) + for lo, hi in sorted(saved)] + finally: + if proc: + proc.kill() + out = os.path.join(root, a.out) + os.makedirs(os.path.dirname(out), exist_ok=True) + json.dump(report, open(out, "w", encoding="utf-8"), indent=1) + print("\nwrote " + a.out) + + +if __name__ == "__main__": + main() From ac0e40628af2aea415d9fc7a6b1af945dcce77ac Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Fri, 11 Sep 2026 23:39:11 -0700 Subject: [PATCH 4/9] Match BIOS setup detection to configured linkable backends --- host/psxrecomp_codegen_host.c | 90 +++++++++---------- runtime/runtime.cmake | 18 +++- runtime/tests/test_cli_retail_bios_profile.py | 58 ++++++++++++ runtime/tests/test_codegen_host_bios_stems.py | 40 ++++++--- 4 files changed, 143 insertions(+), 63 deletions(-) create mode 100644 runtime/tests/test_cli_retail_bios_profile.py diff --git a/host/psxrecomp_codegen_host.c b/host/psxrecomp_codegen_host.c index bd66f4d1..33f3af39 100644 --- a/host/psxrecomp_codegen_host.c +++ b/host/psxrecomp_codegen_host.c @@ -1137,72 +1137,66 @@ static int resolve_build_paths(void) { return join_path(g_exe_path, sizeof(g_exe_path), g_build_dir, exe_name); } -/* Pair a _dispatch.c with its _full.c, the same pairing - * runtime.cmake requires before it will link a BIOS backend. */ -static int dispatch_has_full(const char* dir, const char* name) { - static const char suffix[] = "_dispatch.c"; - char stem[512], full[600], path[1200]; - size_t len = strlen(name); - size_t slen = sizeof(suffix) - 1; - if (len <= slen || strcmp(name + len - slen, suffix) != 0) - return 0; - if (len - slen >= sizeof(stem)) - return 0; - memcpy(stem, name, len - slen); - stem[len - slen] = '\0'; +#ifndef PSX_SETUP_BIOS_STEMS +#define PSX_SETUP_BIOS_STEMS "OpenBIOS|SCPH1001" +#endif +#ifndef PSX_SETUP_FRAMEWORK_REL +#define PSX_SETUP_FRAMEWORK_REL "psxrecomp" +#endif + +/* Match runtime.cmake: a requested pair with its backend descriptor. A stale + * pre-descriptor pair or unrelated game dispatch cannot complete BIOS setup. */ +static int generated_bios_backend_linkable(const char* dir, const char* stem) { + char full[600], path[1200], descriptor[600], line[4096]; + FILE* f; + int found = 0; if ((size_t)snprintf(full, sizeof(full), "%s_full.c", stem) >= sizeof(full)) return 0; + if (!join_path(path, sizeof(path), dir, full) || !path_is_file(path)) + return 0; + snprintf(full, sizeof(full), "%s_dispatch.c", stem); if (!join_path(path, sizeof(path), dir, full)) return 0; - return path_is_file(path); + f = fopen(path, "r"); + if (!f) return 0; + snprintf(descriptor, sizeof(descriptor), "%s_psx_bios_backend", stem); + while (fgets(line, sizeof(line), f)) { + if (strstr(line, descriptor)) { found = 1; break; } + } + fclose(f); + return found; } -/* Does psxrecomp/generated/ hold ANY recompiled BIOS backend? +/* Does the configured framework hold a linkable requested BIOS backend? * * This used to probe two hardcoded names, OpenBIOS_dispatch.c and * SCPH1001_dispatch.c. A port that pins a different image — via * PSXRECOMP_BIOS_STEMS / game.toml recompiler.bios_config, as every wave-3 * kit does with SCPH5552 — emits its backend under that other stem, so the * probe was permanently unsatisfied: Generate kept succeeding, the wizard - * kept reopening, and first-run setup could never complete. Accept any stem - * that has both halves instead of naming images here. */ + * kept reopening, and first-run setup could never complete. CMake supplies + * the requested stems; accept any linkable member, not an unrelated pair. */ static int generated_has_bios_backend(const char* dir) { -#if defined(_WIN32) - char pat[1200]; - WIN32_FIND_DATAA fd; - HANDLE h; - int found = 0; - if (!join_path(pat, sizeof(pat), dir, "*_dispatch.c")) - return 0; - h = FindFirstFileA(pat, &fd); - if (h == INVALID_HANDLE_VALUE) - return 0; - do { - if (dispatch_has_full(dir, fd.cFileName)) { - found = 1; - break; + const char* next = PSX_SETUP_BIOS_STEMS; + while (*next) { + char stem[512]; + const char* end = strchr(next, '|'); + size_t len = end ? (size_t)(end - next) : strlen(next); + if (len && len < sizeof(stem)) { + memcpy(stem, next, len); + stem[len] = 0; + if (generated_bios_backend_linkable(dir, stem)) return 1; } - } while (FindNextFileA(h, &fd)); - FindClose(h); - return found; -#else - DIR* d = opendir(dir); - struct dirent* e; - int found = 0; - if (!d) - return 0; - while (!found && (e = readdir(d)) != NULL) { - if (dispatch_has_full(dir, e->d_name)) - found = 1; + if (!end) break; + next = end + 1; } - closedir(d); - return found; -#endif + return 0; } static int bios_backends_missing(void) { - char gen[1100]; - if (!join_path(gen, sizeof(gen), g_project_root, "psxrecomp/generated")) + char framework[1100], gen[1200]; + if (!join_path(framework, sizeof(framework), g_project_root, PSX_SETUP_FRAMEWORK_REL) || + !join_path(gen, sizeof(gen), framework, "generated")) return 1; return !generated_has_bios_backend(gen); } diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 8f103309..25e34500 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -618,6 +618,20 @@ set(PSXRECOMP_BIOS_STEM "${PSXRECOMP_BIOS_STEM_PRIMARY}" CACHE STRING set(PSXRECOMP_BIOS_PROFILE "${PSXRECOMP_ROOT}/bios/${PSXRECOMP_BIOS_STEM}.toml" CACHE FILEPATH "BIOS profile TOML this build regenerates from (staleness stamp input)") +# The setup host must test the same requested backends as the linker, including +# a relocated framework checkout. Keep the path relative for portable kits. +file(RELATIVE_PATH PSXRECOMP_SETUP_FRAMEWORK_REL "${CMAKE_SOURCE_DIR}" "${PSXRECOMP_ROOT}") +string(REPLACE ";" "|" PSXRECOMP_SETUP_BIOS_STEMS "${PSXRECOMP_BIOS_STEMS}") +set(PSXRECOMP_EXPECTED_RETAIL_STEM "") +foreach(_stem IN LISTS PSXRECOMP_BIOS_STEMS) + if(NOT _stem MATCHES "^[A-Za-z_][A-Za-z0-9_]*$") + message(FATAL_ERROR "Invalid BIOS backend stem: ${_stem}") + endif() + if(NOT _stem STREQUAL "OpenBIOS" AND NOT PSXRECOMP_EXPECTED_RETAIL_STEM) + set(PSXRECOMP_EXPECTED_RETAIL_STEM "${_stem}") + endif() +endforeach() + # Link a stem only if its generated sources are actually present. # # PSXRECOMP_BIOS_STEMS lists every stem this build WOULD like. SCPH1001 is in @@ -1728,7 +1742,9 @@ function(psxrecomp_add_runtime_target target) # The retail stem this build pins. A setup host has no linked # backend to ask, so this is how it knows which image to look for # and name (psx_bios_known_images.h) instead of assuming SCPH-1001. - PSX_EXPECTED_BIOS_STEM="${PSXRECOMP_BIOS_STEM}" + PSX_EXPECTED_BIOS_STEM="${PSXRECOMP_EXPECTED_RETAIL_STEM}" + PSX_SETUP_BIOS_STEMS="${PSXRECOMP_SETUP_BIOS_STEMS}" + PSX_SETUP_FRAMEWORK_REL="${PSXRECOMP_SETUP_FRAMEWORK_REL}" # Where the shipped redistributable image lives, relative to the exe. # This is what a player gets when they choose no BIOS. PSX_BUNDLED_BIOS_PATH="${PSXRECOMP_BUNDLED_BIOS_PATH}" diff --git a/runtime/tests/test_cli_retail_bios_profile.py b/runtime/tests/test_cli_retail_bios_profile.py new file mode 100644 index 00000000..20ff52f8 --- /dev/null +++ b/runtime/tests/test_cli_retail_bios_profile.py @@ -0,0 +1,58 @@ +"""CLI half of the BIOS setup contract (registered by PR #343).""" +import importlib.util +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import Mock, patch + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) +spec = importlib.util.spec_from_file_location("psxrecomp_cli_under_test", ROOT / "psxrecomp_cli.py") +cli = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = cli +spec.loader.exec_module(cli) + + +class RetailBiosProfileTests(unittest.TestCase): + def test_pairs_require_matching_descriptor(self): + with tempfile.TemporaryDirectory() as temporary: + fw = Path(temporary) + generated = fw / "generated" + generated.mkdir() + for stem in ("OpenBIOS", "SCPH1001", "SCPH5552"): + with self.subTest(stem=stem): + dispatch = generated / f"{stem}_dispatch.c" + full = generated / f"{stem}_full.c" + self.assertFalse(cli.bios_backend_present(fw, stem)) + dispatch.write_text(f"const PsxBiosBackend {stem}_psx_bios_backend;", encoding="utf-8") + self.assertFalse(cli.bios_backend_present(fw, stem)) + full.write_text("", encoding="utf-8") + self.assertTrue(cli.bios_backend_present(fw, stem)) + dispatch.write_text("const PsxBiosBackend unrelated_psx_bios_backend;", encoding="utf-8") + self.assertFalse(cli.bios_backend_present(fw, stem)) + + def test_profile_and_utf8_are_forwarded_and_failures_propagate(self): + with tempfile.TemporaryDirectory() as temporary: + fw = Path(temporary) + (fw / "bios").mkdir() + (fw / "bios/SCPH5552.toml").write_text('[program]\nid="SCPH-5552"\n', encoding="utf-8") + progress = Mock() + with patch.object(cli, "framework_root", return_value=fw), \ + patch.object(cli, "find_psxrecomp_bios", return_value=fw / "bios-tool"), \ + patch.object(cli.subprocess, "run") as run: + run.return_value = subprocess.CompletedProcess([], 0, "BIOS → generated", "") + cli.regen_bios_profile(fw, "bios/SCPH5552.toml", progress=progress) + args, kwargs = run.call_args + self.assertEqual(args[0][1:], ["--config", "bios/SCPH5552.toml"]) + self.assertEqual(kwargs["cwd"], str(fw)) + self.assertEqual(kwargs["encoding"], "utf-8") + self.assertEqual(kwargs["errors"], "replace") + run.return_value = subprocess.CompletedProcess([], 2, "", "wrong image") + with self.assertRaisesRegex(RuntimeError, "SCPH5552.*exit 2"): + cli.regen_bios_profile(fw, "bios/SCPH5552.toml", progress=progress) + + +if __name__ == "__main__": + unittest.main() diff --git a/runtime/tests/test_codegen_host_bios_stems.py b/runtime/tests/test_codegen_host_bios_stems.py index 5633273b..66d023b3 100644 --- a/runtime/tests/test_codegen_host_bios_stems.py +++ b/runtime/tests/test_codegen_host_bios_stems.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Regression: the codegen host must accept ANY recompiled BIOS stem. +"""Regression: the codegen host must accept any configured, linkable BIOS stem. bios_backends_missing() in host/psxrecomp_codegen_host.c used to probe two hardcoded filenames, psxrecomp/generated/OpenBIOS_dispatch.c and @@ -54,16 +54,20 @@ def find_recomp_ui() -> Path | None: return None -def make_project(root: Path, bios_files: list[str]) -> None: +def make_project(root: Path, bios_files: list[str], framework: str, descriptor: bool) -> None: """A project tree the host recognises, with the given BIOS artefacts.""" (root / "generated").mkdir(parents=True, exist_ok=True) (root / "generated" / "SCUS_943.51_dispatch.c").write_text("", encoding="utf-8") - fw_gen = root / "psxrecomp" / "generated" + fw_gen = root / framework / "generated" fw_gen.mkdir(parents=True, exist_ok=True) (root / "game.toml").write_text("[game]\n", encoding="utf-8") + (root / "psxrecomp").mkdir(exist_ok=True) (root / "psxrecomp" / "psxrecomp_cli.py").write_text("", encoding="utf-8") for name in bios_files: - (fw_gen / name).write_text("", encoding="utf-8") + text = "" + if descriptor and name.endswith("_dispatch.c"): + text = "const PsxBiosBackend " + name.removesuffix("_dispatch.c") + "_psx_bios_backend = {};\n" + (fw_gen / name).write_text(text, encoding="utf-8") def main() -> int: @@ -71,7 +75,7 @@ def main() -> int: if ui is None: print("SKIP: recomp-ui not found (set RECOMP_UI_ROOT)") return 0 - cc = shutil.which("cc") or shutil.which("gcc") or shutil.which("clang") + cc = os.environ.get("CC") or shutil.which("cc") or shutil.which("gcc") or shutil.which("clang") if cc is None: print("SKIP: no C compiler on PATH") return 0 @@ -81,8 +85,11 @@ def main() -> int: probe_c = tmp / "probe.c" probe_c.write_text(PROBE, encoding="utf-8") exe = tmp / ("probe.exe" if os.name == "nt" else "probe") + framework = "framework checkout" build = subprocess.run( [cc, "-std=c11", "-o", str(exe), str(probe_c), str(HOST_C), + '-DPSX_SETUP_BIOS_STEMS="OpenBIOS|SCPH5552"', + f'-DPSX_SETUP_FRAMEWORK_REL="{framework}"', "-I", str(ROOT / "host"), "-I", str(ROOT / "runtime" / "include"), "-I", str(ui / "src"), "-I", str(ui / "src" / "common")], @@ -93,20 +100,25 @@ def main() -> int: cases = [ # (BIOS artefacts present, expected sources_missing, why) - (["SCPH5552_dispatch.c", "SCPH5552_full.c"], 0, + (["SCPH5552_dispatch.c", "SCPH5552_full.c"], True, 0, "a pinned non-SCPH1001 stem must satisfy the host"), - (["OpenBIOS_dispatch.c", "OpenBIOS_full.c"], 0, + (["OpenBIOS_dispatch.c", "OpenBIOS_full.c"], True, 0, "the bundled stem must still satisfy the host"), - (["SCPH5552_dispatch.c"], 1, + (["SCPH5552_dispatch.c"], True, 1, "a dispatch with no _full.c is not a linkable backend"), - ([], 1, + (["SCPH5552_dispatch.c", "SCPH5552_full.c"], False, 1, + "a pre-descriptor pair must still require generation"), + (["SCPH1001_dispatch.c", "SCPH1001_full.c"], True, 1, + "an unrequested backend must not bypass setup"), + (["SCUS_943.51_dispatch.c", "SCUS_943.51_full.c"], True, 1, + "game sources cannot masquerade as a configured BIOS"), + ([], False, 1, "no BIOS backend at all means setup is genuinely incomplete"), ] failures = 0 - for artefacts, expected, why in cases: - project = tmp / ("case%d" % len(artefacts + [why])) - shutil.rmtree(project, ignore_errors=True) - make_project(project, artefacts) + for index, (artefacts, descriptor, expected, why) in enumerate(cases): + project = tmp / f"case{index}" + make_project(project, artefacts, framework, descriptor) env = dict(os.environ, PSXRECOMP_PROJECT_ROOT=str(project)) run = subprocess.run([str(exe)], capture_output=True, text=True, env=env) got = run.stdout.strip() @@ -119,7 +131,7 @@ def main() -> int: if failures: return 1 - print("PASS: codegen host accepts any paired BIOS stem") + print("PASS: codegen host accepts configured, linkable BIOS stems in relocated frameworks") return 0 From fd84134c193c4eebbeb151c08975f11ea6684897 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Fri, 11 Sep 2026 23:39:33 -0700 Subject: [PATCH 5/9] Guard patched BIOS ranges before control flow and cycle charging --- recompiler/src/full_function_emitter.cpp | 215 +++++++++++------- .../tests/full_function_emitter_test.cpp | 65 +++++- runtime/src/dirty_ram_interp.c | 2 +- runtime/src/memory.c | 2 + 4 files changed, 197 insertions(+), 87 deletions(-) diff --git a/recompiler/src/full_function_emitter.cpp b/recompiler/src/full_function_emitter.cpp index 702a8a5e..157f1b30 100644 --- a/recompiler/src/full_function_emitter.cpp +++ b/recompiler/src/full_function_emitter.cpp @@ -251,6 +251,17 @@ bool FullFunctionEmitter::emit_function( // dominated by kernel poll-loop leaders 0x45FC/0x4614 inside func_00004498 — // both had labels/gotos but no dispatch entries (only jal+8 conts were registered). // Entry block is already a function dispatch key; skip it. Dedup later. + // Split before computing cycle totals and load-delay boundaries. Guards + // precede all replaced instructions, including interior branch targets. + for (const auto& [pc, word] : addr_to_raw) { + (void)word; + const uint32_t ram_pc = addr_model().rom_to_ram_phys(pc); + for (const auto& slot : addr_model().install_slots()) { + if ((ram_pc >= slot.ram_addr && ram_pc < slot.hi()) || + ram_pc == slot.resume_addr()) + block_leaders.insert(pc); + } + } for (uint32_t leader : block_leaders) { if (!addr_to_raw.count(leader)) continue; uint32_t leader_norm = normalize_address(leader); @@ -355,6 +366,24 @@ bool FullFunctionEmitter::emit_function( } } + // A guard cannot unwind a live branch/load delay using only a PC. Leave + // such a RAM-backed body on the existing fail-closed interpreter path; + // never widen the declared bytes to conceal an unsafe boundary. + for (const auto& [pc, word] : addr_to_raw) { + (void)word; + const uint32_t ram_pc = addr_model().rom_to_ram_phys(pc); + if (!addr_model().is_install_slot(ram_pc)) continue; + const auto previous = addr_to_raw.find(pc - 4u); + const bool prior_load = previous != addr_to_raw.end() && + (previous->second >> 26) >= 0x20u && (previous->second >> 26) <= 0x26u; + if (pending_at.count(pc) || prior_load) { + if (out_interpreter_reason) + *out_interpreter_reason = fmt::format( + "patch range at RAM 0x{:08X} crosses a branch/load delay boundary", ram_pc); + return false; + } + } + // Detect delay slots that fall outside this function's address range. // These will never be visited by the emit loop, so their PendingBranch // would silently vanish. Collect them for immediate resolution. @@ -754,6 +783,106 @@ bool FullFunctionEmitter::emit_function( relocate_ra(rom_addr)); }; + auto emit_patch_range_guard = [&](uint32_t addr) { + // Kernel patch-range hook (CLAUDE.md Rule 18 / docs/dynamic_handler_install.md). + // The BIOS and the game's Psy-Q libapi patchers overwrite specific + // kernel-RAM words at runtime: install stubs written over ROM zeros + // (RAM 0xCF0, the SIO data-byte handler), but also a `jr` planted over + // real ROM instructions, an 11-word prologue rewrite that inserts an + // `mfc0 Cause`, and a driver routine NOP'd out. The recompiler emitted + // the ROM bytes; if we run those statically the guest's patch never + // executes. + // + // At every possible entry into a declared range, emit a runtime check: if ANY word + // in the range differs from its ROM-baked value, the patch is live — + // dispatch into RAM so the interpreter runs the guest's own + // instructions (Rule 18: we execute the patch as written, we do not + // synthesise its effect). The rest of the body still runs native, + // because the range is also published to the runtime, which excludes + // it from the kernel-bless memcmp (memory.c) and hands straight-line + // flow back at the range end (dirty_ram_interp.c). + // + // Ranges come from the profile's [[recompiler.install_slots]]; see + // docs/dynamic_handler_install.md for how to find new ones. + uint32_t ram_pc = addr_model().rom_to_ram_phys(addr); + const BiosInstallSlot* slot = nullptr; + for (const auto& range : addr_model().install_slots()) { + if (ram_pc >= range.ram_addr && ram_pc < range.hi()) { slot = ⦥ break; } + } + if (slot) { + /* Where the compiled body picks up again: + * - Jalr (legacy 4-word stub): the stub's jalr captures + * ra = stub_PC + 8 = ram_addr + 0x10, so the called function + * returns there. + * - Fallthrough: the patched words ARE the function; execution + * continues at the end of the range. + * - None: the patch is a `jr` out of the kernel and never comes + * back to this body. + * For the first two, register the resume PC as both a block leader + * (so label_: exists) and a local continuation (so the + * entry-switch and the emitted continuation wrapper route to it). + * Without the continuation key, external dispatch to the resume PC + * misses and falls into interpretation, which has no COP0 and + * crashes the exception handler. */ + uint32_t resume_ram = slot->resume_addr(); + if (resume_ram != 0u) { + uint32_t resume_rom = addr + (resume_ram - ram_pc); + if (addr_to_raw.count(resume_rom)) { + block_leaders.insert(resume_rom); + uint32_t resume_norm = normalize_address(resume_rom); + if (!all_function_entries_norm.count(resume_norm)) { + local_continuations.push_back({resume_rom, resume_norm, norm}); + } + } + } + /* Compare every word in the range against the ROM-baked value. + * The old test was `first word != 0`, which only fires for a stub + * written over ROM zeros — a `jr` planted over a real instruction + * and a prologue rewrite both leave non-zero ROM words and were + * never detected. A page-level dirty bit is far too coarse: the + * kernel handler dirties page 0 on every exception by saving + * registers, which would redirect unconditionally. Word-level + * compare is exact: the range is "live" iff the guest has actually + * changed it. */ + std::string cond; + uint32_t base_phys_local = base_addr & 0x1FFFFFFFu; + for (uint32_t off = 0; off < slot->len; off += 4u) { + uint32_t w_rom_addr = addr - (ram_pc - slot->ram_addr) + off; + uint32_t w_phys = w_rom_addr & 0x1FFFFFFFu; + if (w_phys < base_phys_local || + w_phys + 4u > base_phys_local + rom.size()) { + throw std::runtime_error(fmt::format( + "install slot RAM 0x{:08X} len 0x{:X}: word +0x{:X} " + "lies outside the ROM image", ram_pc, slot->len, off)); + } + uint32_t rom_word = read_u32_le(rom, w_phys - base_phys_local); + if (!cond.empty()) cond += " ||\n "; + cond += fmt::format("cpu->read_word(0x{:08X}u) != 0x{:08X}u", + slot->ram_addr + off, rom_word); + } + // A normal delay-slot execution belongs to the guarded branch. + // A direct branch TO this label has no pending branch flag. + auto pending = pending_at.find(addr); + if (pending != pending_at.end()) + cond = fmt::format("!psx_delay_{:08X} && ({})", + pending->second.terminator_addr, cond); + out += fmt::format( + " /* 0x{:08X}: kernel patch-range hook (RAM [0x{:08X},0x{:08X}),\n" + " * resume RAM 0x{:08X}). If the guest has overwritten any word\n" + " * of this range, dispatch into RAM so its own instructions run;\n" + " * otherwise fall through to the statically compiled ROM code. */\n" + " if ({}) {{\n" + "#ifdef PSX_ENABLE_BLOCK_CYCLES\n" + " psx_cyc_bb_defer_flush();\n" + "#endif\n" + " psx_check_interrupts_at(cpu, 0x{:08X}u);\n" + " cpu->pc = 0x{:08X}u; return;\n" + " }}\n", + addr, slot->ram_addr, slot->hi(), resume_ram, + cond, ram_pc, ram_pc); + } + }; + for (auto it = addr_to_raw.begin(); it != addr_to_raw.end(); ++it) { uint32_t addr = it->first; uint32_t raw = it->second; @@ -763,6 +892,7 @@ bool FullFunctionEmitter::emit_function( // can service vblank and other hardware interrupts. if (block_leaders.count(addr)) { out += fmt::format("label_{:08X}:\n", addr); + emit_patch_range_guard(addr); // Per-block-leader cycle observe (cyc_watch ruler). Sampled BEFORE // this block's cycle advance, so it reports cumulative cycles for // all PRIOR blocks — matching Beetle's before-instruction sample @@ -1192,91 +1322,6 @@ bool FullFunctionEmitter::emit_function( continue; } - // Kernel patch-range hook (CLAUDE.md Rule 18 / docs/dynamic_handler_install.md). - // The BIOS and the game's Psy-Q libapi patchers overwrite specific - // kernel-RAM words at runtime: install stubs written over ROM zeros - // (RAM 0xCF0, the SIO data-byte handler), but also a `jr` planted over - // real ROM instructions, an 11-word prologue rewrite that inserts an - // `mfc0 Cause`, and a driver routine NOP'd out. The recompiler emitted - // the ROM bytes; if we run those statically the guest's patch never - // executes. - // - // At the START of a declared range, emit a runtime check: if ANY word - // in the range differs from its ROM-baked value, the patch is live — - // dispatch into RAM so the interpreter runs the guest's own - // instructions (Rule 18: we execute the patch as written, we do not - // synthesise its effect). The rest of the body still runs native, - // because the range is also published to the runtime, which excludes - // it from the kernel-bless memcmp (memory.c) and hands straight-line - // flow back at the range end (dirty_ram_interp.c). - // - // Ranges come from the profile's [[recompiler.install_slots]]; see - // docs/dynamic_handler_install.md for how to find new ones. - uint32_t ram_pc = addr_model().rom_to_ram_phys(addr); - const BiosInstallSlot* slot = addr_model().install_slot_at(ram_pc); - if (slot) { - /* Where the compiled body picks up again: - * - Jalr (legacy 4-word stub): the stub's jalr captures - * ra = stub_PC + 8 = ram_addr + 0x10, so the called function - * returns there. - * - Fallthrough: the patched words ARE the function; execution - * continues at the end of the range. - * - None: the patch is a `jr` out of the kernel and never comes - * back to this body. - * For the first two, register the resume PC as both a block leader - * (so label_: exists) and a local continuation (so the - * entry-switch and the emitted continuation wrapper route to it). - * Without the continuation key, external dispatch to the resume PC - * misses and falls into interpretation, which has no COP0 and - * crashes the exception handler. */ - uint32_t resume_ram = slot->resume_addr(); - if (resume_ram != 0u) { - uint32_t resume_rom = addr + (resume_ram - ram_pc); - if (addr_to_raw.count(resume_rom)) { - block_leaders.insert(resume_rom); - uint32_t resume_norm = normalize_address(resume_rom); - if (!all_function_entries_norm.count(resume_norm)) { - local_continuations.push_back({resume_rom, resume_norm, norm}); - } - } - } - /* Compare every word in the range against the ROM-baked value. - * The old test was `first word != 0`, which only fires for a stub - * written over ROM zeros — a `jr` planted over a real instruction - * and a prologue rewrite both leave non-zero ROM words and were - * never detected. A page-level dirty bit is far too coarse: the - * kernel handler dirties page 0 on every exception by saving - * registers, which would redirect unconditionally. Word-level - * compare is exact: the range is "live" iff the guest has actually - * changed it. */ - std::string cond; - uint32_t base_phys_local = base_addr & 0x1FFFFFFFu; - for (uint32_t off = 0; off < slot->len; off += 4u) { - uint32_t w_rom_addr = addr + off; - uint32_t w_phys = w_rom_addr & 0x1FFFFFFFu; - if (w_phys < base_phys_local || - w_phys + 4u > base_phys_local + rom.size()) { - throw std::runtime_error(fmt::format( - "install slot RAM 0x{:08X} len 0x{:X}: word +0x{:X} " - "lies outside the ROM image", ram_pc, slot->len, off)); - } - uint32_t rom_word = read_u32_le(rom, w_phys - base_phys_local); - if (!cond.empty()) cond += " ||\n "; - cond += fmt::format("cpu->read_word(0x{:08X}u) != 0x{:08X}u", - ram_pc + off, rom_word); - } - out += fmt::format( - " /* 0x{:08X}: kernel patch-range hook (RAM [0x{:08X},0x{:08X}),\n" - " * resume RAM 0x{:08X}). If the guest has overwritten any word\n" - " * of this range, dispatch into RAM so its own instructions run;\n" - " * otherwise fall through to the statically compiled ROM code. */\n" - " if ({}) {{\n" - " psx_check_interrupts_at(cpu, 0x{:08X}u);\n" - " cpu->pc = 0x{:08X}u; return;\n" - " }}\n", - addr, ram_pc, ram_pc + slot->len, resume_ram, - cond, ram_pc, ram_pc); - } // Non-terminator: emit normally — unless this load's successor reads // its destination (MIPS-I load-delay pair): then defer the register diff --git a/recompiler/tests/full_function_emitter_test.cpp b/recompiler/tests/full_function_emitter_test.cpp index 747e977d..f30f49af 100644 --- a/recompiler/tests/full_function_emitter_test.cpp +++ b/recompiler/tests/full_function_emitter_test.cpp @@ -62,6 +62,7 @@ std::string read_file(const std::filesystem::path& path) { struct RunResult { EmitStats stats; std::string dispatch; + std::string full; }; RunResult run_case(const char* name, const std::vector& words, @@ -86,6 +87,7 @@ RunResult run_case(const char* name, const std::vector& words, rom, kBase, kBase + static_cast(rom.size()) - 1u, discovery, "synthetic", out_dir.string(), stem); result.dispatch = read_file(out_dir / (stem + "_dispatch.c")); + result.full = read_file(out_dir / (stem + "_full.c")); std::filesystem::remove_all(out_dir); return result; } @@ -102,7 +104,10 @@ void expect_dispatch_key_absent(const RunResult& result, uint32_t normalized, const char* message) { char needle[32]; std::snprintf(needle, sizeof(needle), "{ 0x%08Xu,", normalized); - expect(result.dispatch.find(needle) == std::string::npos, message); + const auto begin = result.dispatch.find("static const DispatchEntry dispatch_table["); + const auto end = result.dispatch.find("};", begin); + expect(begin != std::string::npos && + result.dispatch.substr(begin, end - begin).find(needle) == std::string::npos, message); } void delay_slot_load_falls_back() { @@ -199,6 +204,62 @@ void complementary_lwl_lwr_stays_native() { } // namespace +void patch_range_guards(BiosConfig config) { + PSXRecompV4::BiosInstallSlot slot; + slot.ram_addr = 0x508u; + slot.len = 8u; + slot.resume = PSXRecompV4::BiosInstallSlot::Resume::Fallthrough; + config.install_slots.push_back(slot); + config.address_copies[0].kernel_bless = true; + const auto model = BiosAddressModel::from_config(config); + FullFunctionEmitter::set_address_model(&model); + const auto result = run_case("patch-terminator", { + 0x10000002u, // branch directly into the range interior at +12 + 0x00000000u, + 0x03E00008u, // jr ra at the range START must also be guarded + 0x00000000u, + 0x03E00008u, + 0x00000000u, + }, {function_at(kBase, kBase + 20u, {kBase, kBase + 8u, kBase + 12u})}); + const auto start = result.full.find("label_BFC00008:"); + const auto interior = result.full.find("label_BFC0000C:"); + expect(start != std::string::npos && + result.full.find("kernel patch-range hook", start) < interior, + "a terminator at range start is guarded"); + expect(interior != std::string::npos && + result.full.find("kernel patch-range hook", interior) != std::string::npos, + "a branch into a range interior cannot bypass the guard"); + const auto guard = result.full.find("kernel patch-range hook", start); + expect(guard < result.full.find("psx_cyc_step", start), + "patched instructions are not charged before interpreter handoff"); + expect_dispatch_key_absent(result, 0x508u, "range start not externally dispatched"); + expect_dispatch_key_absent(result, 0x50Cu, "range interior not externally dispatched"); +} + +void patch_range_delay_boundaries(BiosConfig config) { + PSXRecompV4::BiosInstallSlot slot; + slot.ram_addr = 0x504u; + slot.len = 4u; + slot.resume = PSXRecompV4::BiosInstallSlot::Resume::Fallthrough; + config.install_slots.push_back(slot); + config.address_copies[0].kernel_bless = true; + const auto model = BiosAddressModel::from_config(config); + FullFunctionEmitter::set_address_model(&model); + for (uint32_t predecessor : {0x10000001u, 0x8D280000u}) { + const auto result = run_case("patch-delay-boundary", { + predecessor, // branch or CPU load immediately before the range + 0x00000000u, + 0x03E00008u, + 0x00000000u, + }, {function_at(kBase, kBase + 12u, {kBase})}); + expect(result.stats.functions_interpreted == 1, + "a patch guard cannot discard a pending branch/load delay"); + expect(result.stats.functions_emitted == 0, + "an unsafe patch boundary is fail-closed"); + expect_entry_absent(result, 0x500u, "unsafe patch owner not dispatched"); + } +} + int main() { BiosConfig config{}; config.config_path = "test://full-function-emitter"; @@ -221,6 +282,8 @@ int main() { fragment_split_load_falls_back(); noncomplementary_lwl_falls_back(); complementary_lwl_lwr_stays_native(); + patch_range_guards(config); + patch_range_delay_boundaries(config); if (failures != 0) { std::fprintf(stderr, "%d full-function emitter test(s) failed\n", failures); diff --git a/runtime/src/dirty_ram_interp.c b/runtime/src/dirty_ram_interp.c index 7ba7498f..922c9fe8 100644 --- a/runtime/src/dirty_ram_interp.c +++ b/runtime/src/dirty_ram_interp.c @@ -3331,7 +3331,7 @@ static int dirty_ram_dispatch_inner(CPUState* cpu, uint32_t addr, uint32_t stop_ * Without this hand-back the kernel page stays dirty and straight- * line flow would interpret the whole function — which is why * declaring the slot alone never paid off. */ - if (next_phys < DIRTY_RAM_KERNEL_WINDOW_END && + if (!s_ld_pend_armed && next_phys < DIRTY_RAM_KERNEL_WINDOW_END && psx_kernel_patch_range_ends_at(next_phys) && psx_kernel_bless_dispatchable(next_phys)) { cpu->pc = pc; diff --git a/runtime/src/memory.c b/runtime/src/memory.c index f731d25b..f4b3cb78 100644 --- a/runtime/src/memory.c +++ b/runtime/src/memory.c @@ -389,6 +389,8 @@ void psx_kernel_bless_reset_for_boot(void) { s_kb_lo = 0; s_kb_span = 0; s_kb_rom_off = 0; + s_kb_pr = NULL; + s_kb_pr_n = 0; memset(kbless_state, KBLESS_UNKNOWN, sizeof(kbless_state)); } From 63b205df7862cf66024b8fd3210e978a00963d94 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Fri, 11 Sep 2026 23:39:46 -0700 Subject: [PATCH 6/9] Index immutable resident dispatch keys and clarify owner telemetry --- recompiler/CMakeLists.txt | 1 + recompiler/src/main_psx.cpp | 53 +++++++++--- .../tests/test_kuseg_dispatch_lookup.py | 80 +++++++++++++++++-- runtime/include/overlay_loader.h | 5 +- runtime/src/main.cpp | 2 +- 5 files changed, 125 insertions(+), 16 deletions(-) diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index 90b22cff..dae9d0bf 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -569,6 +569,7 @@ if(BUILD_TESTING) COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_kuseg_dispatch_lookup.py --recompiler $ + --compiler ${CMAKE_C_COMPILER} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) add_test( NAME overlay_shim_compile_contract diff --git a/recompiler/src/main_psx.cpp b/recompiler/src/main_psx.cpp index bf276000..a2649d50 100644 --- a/recompiler/src/main_psx.cpp +++ b/recompiler/src/main_psx.cpp @@ -1602,6 +1602,34 @@ int main(int argc, char** argv) { } ds << "};\n"; ds << fmt::format("#define PSX_GAME_DISPATCH_COUNT {}u\n\n", records.size()); + // WO-6: resident dispatch keys are immutable. Index them once at build + // time instead of binary-searching on every overlay-to-EXE call and + // every text-validity query. This caches resolution, NEVER validity: + // callers below still check live instruction ranges on every dispatch. + // Bound storage to one PS1 RAM image; unusual sparse/non-RAM tables + // retain the existing binary search. No mutable cache or new flag. + const uint32_t lookup_lo = records.empty() ? 0u : (records.front().addr & 0x1FFFFFFFu); + const uint32_t lookup_hi = records.empty() ? 0u : (records.back().addr & 0x1FFFFFFFu); + bool indexed_lookup = !records.empty() && lookup_hi - lookup_lo < 0x200000u; + for (size_t i = 0; i < records.size(); ++i) { + const uint32_t key = records[i].addr & 0x1FFFFFFFu; + if ((key & 3u) || (i && key == (records[i - 1].addr & 0x1FFFFFFFu))) + indexed_lookup = false; + } + if (indexed_lookup) { + std::vector index((lookup_hi - lookup_lo) / 4u + 1u, 0u); + for (size_t i = 0; i < records.size(); ++i) + index[((records[i].addr & 0x1FFFFFFFu) - lookup_lo) / 4u] = (uint32_t)i + 1u; + ds << "/* Immutable physical-word index; zero denotes a dispatch miss. */\n"; + ds << "static const " << (records.size() <= 65535u ? "uint16_t" : "uint32_t") + << " k_psx_game_dispatch_index[] = {\n"; + for (size_t i = 0; i < index.size(); ++i) { + if ((i & 15u) == 0) ds << " "; + ds << index[i] << "u,"; + ds << (((i & 15u) == 15u || i + 1 == index.size()) ? "\n" : " "); + } + ds << "};\n\n"; + } ds << "/* PS1 segments alias the same physical RAM. A game whose PS-X EXE\n"; ds << " * header carries KUSEG addresses (load address and entry PC without the\n"; ds << " * KSEG bit) executes with a KUSEG PC, while this table is keyed by the\n"; @@ -1612,15 +1640,22 @@ int main(int argc, char** argv) { ds << " * the table is sorted by the same masked key. */\n"; ds << "static const PsxGameDispatchEntry* psx_game_find_entry(uint32_t addr) {\n"; ds << " const uint32_t want = addr & 0x1FFFFFFFu;\n"; - ds << " uint32_t lo = 0, hi = PSX_GAME_DISPATCH_COUNT;\n"; - ds << " while (lo < hi) {\n"; - ds << " uint32_t mid = lo + (hi - lo) / 2;\n"; - ds << " uint32_t key = k_psx_game_dispatch[mid].addr & 0x1FFFFFFFu;\n"; - ds << " if (want < key) hi = mid;\n"; - ds << " else if (want > key) lo = mid + 1;\n"; - ds << " else return &k_psx_game_dispatch[mid];\n"; - ds << " }\n"; - ds << " return 0;\n"; + if (indexed_lookup) { + ds << fmt::format(" const uint32_t offset = want - 0x{:08X}u;\n", lookup_lo); + ds << fmt::format(" if ((want & 3u) || offset > 0x{:X}u) return 0;\n", lookup_hi - lookup_lo); + ds << " const uint32_t index = k_psx_game_dispatch_index[offset >> 2];\n"; + ds << " return index ? &k_psx_game_dispatch[index - 1u] : 0;\n"; + } else { + ds << " uint32_t lo = 0, hi = PSX_GAME_DISPATCH_COUNT;\n"; + ds << " while (lo < hi) {\n"; + ds << " uint32_t mid = lo + (hi - lo) / 2;\n"; + ds << " uint32_t key = k_psx_game_dispatch[mid].addr & 0x1FFFFFFFu;\n"; + ds << " if (want < key) hi = mid;\n"; + ds << " else if (want > key) lo = mid + 1;\n"; + ds << " else return &k_psx_game_dispatch[mid];\n"; + ds << " }\n"; + ds << " return 0;\n"; + } ds << "}\n\n"; ds << "/* Exact static-code validity for this entry's emitted CFG ranges. */\n"; diff --git a/recompiler/tests/test_kuseg_dispatch_lookup.py b/recompiler/tests/test_kuseg_dispatch_lookup.py index cb14c4b7..96854c75 100644 --- a/recompiler/tests/test_kuseg_dispatch_lookup.py +++ b/recompiler/tests/test_kuseg_dispatch_lookup.py @@ -20,7 +20,7 @@ key so the binary-search invariant holds. Usage: python test_kuseg_dispatch_lookup.py [--recompiler ] Exit 0 = PASS. """ -import argparse, os, re, struct, subprocess, sys, tempfile +import argparse, os, re, shutil, struct, subprocess, sys, tempfile LOAD = 0x00010000 # KUSEG: no KSEG bit, the case under test PHYS_MASK = 0x1FFFFFFF @@ -80,6 +80,7 @@ def main(): ap.add_argument("--recompiler", default=os.path.normpath(os.path.join(here, "..", "build", "psxrecomp-game"))) + ap.add_argument("--compiler", default=os.environ.get("CC") or shutil.which("cc") or shutil.which("gcc")) args = ap.parse_args() if not os.path.isfile(args.recompiler): raise SystemExit("recompiler not found: %s (build it first)" % args.recompiler) @@ -98,10 +99,8 @@ def main(): raise SystemExit( "psx_game_find_entry does not mask to a physical address; a KUSEG " "guest PC can never match KSEG-normalized table keys") - # ... and so must the stored key, otherwise the comparison is still mixed. - if not re.search(r"k_psx_game_dispatch\[mid\]\.addr\s*&\s*0x1FFFFFFFu", body): - raise SystemExit( - "psx_game_find_entry compares against an unmasked table key") + if "k_psx_game_dispatch_index" not in body: + raise SystemExit("small resident tables must use the immutable word index") # The binary search requires the table to be ordered by the same key it # compares. Parse the emitted entries and assert monotonic physical order. @@ -116,6 +115,77 @@ def main(): raise SystemExit("dispatch table is not sorted by physical address; " "the masked binary search would miss entries") + # Compile the REAL emitted lookup, validity gates and dispatch function. + # Only peripheral callbacks and generated MIPS bodies are fixture doubles. + if not args.compiler: + raise SystemExit("a C compiler is required for the dispatch regression") + begin = src.index("typedef struct { uint32_t lo; uint32_t len; } PsxGameCodeRange;") + end = src.index("/* 1 iff addr is a re-enterable", begin) + fragment = src[begin:end] + funcs = sorted(set(re.findall(r"func_[0-9A-Fa-f]{8}", fragment))) + harness = r''' +#include +#include +#include +typedef struct { uint32_t pc; } CPUState; +static int allowed = 1, calls = 0, irqs = 0; +static uint32_t resumed; +static void dummy(CPUState* cpu) { calls++; resumed = cpu->pc; } +static int dirty_ram_text_native_ok_ranges_from(const uint32_t* r, uint32_t n, uint32_t a) { + (void)r; (void)n; (void)a; return allowed; +} +static int dirty_ram_text_native_ok_ranges(const uint32_t* r, uint32_t n) { + return dirty_ram_text_native_ok_ranges_from(r, n, 0); +} +static void psx_check_interrupts_dispatch_entry(CPUState* cpu, uint32_t a) { + (void)cpu; (void)a; irqs++; +} +int psx_vsync_query_hle_try(CPUState* cpu, uint32_t a) { (void)cpu; (void)a; return 0; } +''' + harness += "\n".join(f"#define {fn} dummy" for fn in funcs) + "\n" + fragment + harness += r''' +int main(void) { + const uint32_t aliases[] = {0, 0x80000000u, 0xA0000000u}; + CPUState cpu = {0}; + for (unsigned a = 0; a < 3; ++a) { + for (uint32_t phys = 0xFFF0u; phys < 0x10080u; ++phys) { + const PsxGameDispatchEntry* expected = 0; + for (unsigned i = 0; i < PSX_GAME_DISPATCH_COUNT; ++i) + if ((k_psx_game_dispatch[i].addr & 0x1FFFFFFFu) == phys) + expected = &k_psx_game_dispatch[i]; + uint32_t addr = aliases[a] | phys; + assert(psx_game_find_entry(addr) == expected); + if (!expected) continue; + // A previously resolved PC must NOT cache its live-byte verdict. + allowed = 0; cpu.pc = 0xDEADBEEFu; + int old_calls = calls, old_irqs = irqs; + assert(!psx_game_text_native_ok(addr)); + assert(!psx_dispatch_game_compiled(&cpu, addr)); + assert(calls == old_calls && irqs == old_irqs && cpu.pc == 0xDEADBEEFu); + allowed = 1; + assert(psx_dispatch_game_compiled(&cpu, addr)); + assert(calls == old_calls + 1 && irqs == old_irqs + 1); + assert(resumed == expected->resume_pc); + } + } + assert(!psx_game_find_entry(0xFFFFFFFFu)); + assert(!psx_game_find_entry(0)); + return 0; +} +''' + with tempfile.TemporaryDirectory() as tmp: + source = os.path.join(tmp, "lookup.c") + binary = os.path.join(tmp, "lookup.exe" if os.name == "nt" else "lookup") + with open(source, "w", encoding="utf-8") as f: + f.write(harness) + compiler_name = os.path.basename(args.compiler).lower() + if compiler_name in ("cl", "cl.exe", "clang-cl", "clang-cl.exe"): + command = [args.compiler, "/nologo", "/Od", source, "/Fe:" + binary] + else: + command = [args.compiler, "-std=c11", "-O2", source, "-o", binary] + subprocess.run(command, check=True, cwd=tmp) + subprocess.run([binary], check=True, cwd=tmp) + print("KUSEG dispatch lookup test passed (%d entries, physical-keyed)" % len(keys)) diff --git a/runtime/include/overlay_loader.h b/runtime/include/overlay_loader.h index 122f3dc7..5cd8c46e 100644 --- a/runtime/include/overlay_loader.h +++ b/runtime/include/overlay_loader.h @@ -113,7 +113,10 @@ void overlay_loader_get_counters(uint32_t *loads, uint32_t *invalidations, void overlay_loader_get_load_timing(uint64_t *total_us, uint64_t *max_us, uint64_t *last_us); /* Opt-in PSX_RUNTIME_PERF_DIAG sampler: returns and clears the hottest native - * owner since the preceding call. Disabled runs pay no table update cost. */ + * owner since the preceding call. Counts owner activations, INCLUDING CPS + * continuations, not guest function invocations (use fntrace for entries). + * The bounded direct-mapped sampler can undercount on collision; it cannot + * inflate a count. Disabled runs pay no table update cost. */ void overlay_loader_take_hot_native(uint32_t *pc, uint64_t *calls); /* Exact shadow-differential summary for opt-in perf diagnostics. */ void overlay_loader_get_shadow_summary(uint64_t *calls, uint64_t *divergences, diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index 25d89d29..cfcf7534 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -3600,7 +3600,7 @@ static void runtime_perf_diag_tick() { "cpu=%.1f tex=%.1f draw=%.1f ms/s; " "work guest=%.1f pacer=%.1f autocapture=%.1f provider_poll=%.1f ms/s, " "dirty=%.0f insn/s %.0f dispatch/s; " - "overlay native=+%llu interp=+%llu hot_native=0x%08X/+%llu " + "overlay native=+%llu interp=+%llu hot_native_owner=0x%08X/activations>=+%llu " "shadow=+%llu div=+%llu first_div=0x%08X " "loads=+%u revalidations=+%u " "load_wall=%.1f ms max=%.1f last=%.1f ms; " From fae85cc886dcbcc33e4f79624bc19610b511fcdf Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Fri, 11 Sep 2026 23:39:47 -0700 Subject: [PATCH 7/9] Repair Windows and stale regression test harness assumptions --- runtime/CMakeLists.txt | 9 +++++++ .../test_dirty_text_continuation_guards.py | 6 +++-- runtime/tests/test_launcher_vulkan_option.py | 8 +++---- .../tests/test_overlay_pair_dedup_runtime.py | 24 +++++++------------ runtime/tests/test_release_zip.py | 4 ++-- 5 files changed, 27 insertions(+), 24 deletions(-) diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 46c476a3..9b74e42c 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -393,6 +393,15 @@ if(BUILD_TESTING) include ../recompiler/lib/toml11) target_link_libraries(mod_runtime_test PRIVATE chdr-static) + # The real disc hasher uses a 1 MiB scratch buffer. Match the runtime's + # stack allowance instead of overflowing MinGW's 1 MiB executable default. + if(WIN32) + if(MSVC) + target_link_options(mod_runtime_test PRIVATE /STACK:16777216) + else() + target_link_options(mod_runtime_test PRIVATE -Wl,--stack,16777216) + endif() + endif() add_test(NAME mod_runtime_test COMMAND mod_runtime_test) if(WIN32) diff --git a/runtime/tests/test_dirty_text_continuation_guards.py b/runtime/tests/test_dirty_text_continuation_guards.py index 56aa21ea..751fef44 100644 --- a/runtime/tests/test_dirty_text_continuation_guards.py +++ b/runtime/tests/test_dirty_text_continuation_guards.py @@ -18,13 +18,15 @@ def main() -> int: required_range_fragments = ( "uint32_t exec_pc", - "if (phys + len <= at) continue;", - "len -= at - phys;", + "(void)exec_pc;", + "memcmp(ram + phys, text_ref_image + (phys - text_ref_lo), len)", "if (!any)", ) for fragment in required_range_fragments: if fragment not in range_guard: raise AssertionError(f"missing continuation range guard: {fragment}") + if "len -= at - phys;" in range_guard or "if (phys + len <= at) continue;" in range_guard: + raise AssertionError("continuation guard clips away code reachable by a backward edge") if "text_diverged_bitmap" in range_guard: raise AssertionError("exact-range mismatch still sticky-poisons a whole page") if "dirty_ram_text_native_ok_ranges_from(lo_len_pairs, count, 0u)" not in memory: diff --git a/runtime/tests/test_launcher_vulkan_option.py b/runtime/tests/test_launcher_vulkan_option.py index ad460ece..06d51d37 100644 --- a/runtime/tests/test_launcher_vulkan_option.py +++ b/runtime/tests/test_launcher_vulkan_option.py @@ -2,10 +2,10 @@ from pathlib import Path root = Path(__file__).resolve().parents[2] -main = (root / "runtime" / "src" / "main.cpp").read_text() -config_h = (root / "recompiler" / "src" / "config_loader.h").read_text() -config_cpp = (root / "recompiler" / "src" / "config_loader.cpp").read_text() -runtime_cmake = (root / "runtime" / "runtime.cmake").read_text() +main = (root / "runtime" / "src" / "main.cpp").read_text(encoding="utf-8") +config_h = (root / "recompiler" / "src" / "config_loader.h").read_text(encoding="utf-8") +config_cpp = (root / "recompiler" / "src" / "config_loader.cpp").read_text(encoding="utf-8") +runtime_cmake = (root / "runtime" / "runtime.cmake").read_text(encoding="utf-8") assert "runtime/" + "launcher" not in runtime_cmake assert "Rml" + "Ui" not in runtime_cmake diff --git a/runtime/tests/test_overlay_pair_dedup_runtime.py b/runtime/tests/test_overlay_pair_dedup_runtime.py index 957dfba8..187d8612 100644 --- a/runtime/tests/test_overlay_pair_dedup_runtime.py +++ b/runtime/tests/test_overlay_pair_dedup_runtime.py @@ -7,10 +7,11 @@ import argparse import os import pathlib import platform -import re import shutil import subprocess +import sys import tempfile +from unittest.mock import patch import zlib @@ -23,21 +24,12 @@ RESIDENT = "psxrecomp bios resident shard v1\n" def codegen_leaf() -> str: - api = (RUNTIME / "include" / "overlay_api.h") - hash_header = (RUNTIME / "include" / "overlay_codegen_hash.h") - version = "0" - code_hash = "00000000" - if api.is_file(): - match = re.search(r"PSX_OVERLAY_CODEGEN_VER\s+(\d+)", - api.read_text(encoding="utf-8")) - if match: - version = match.group(1) - if hash_header.is_file(): - match = re.search(r"PSX_OVERLAY_CODEGEN_HASH\s+0x([0-9A-Fa-f]{8})", - hash_header.read_text(encoding="utf-8")) - if match: - code_hash = match.group(1).lower() - return f"cg{version}_{code_hash}_gc00000000_f0" + sys.path.insert(0, str(ROOT / "tools")) + import compile_overlays + # The synthetic loader uses a zero config hash; serialize its namespace + # with the production owner so new namespace fields cannot go stale here. + with patch.object(compile_overlays, "overlay_config_hash", return_value=0): + return compile_overlays.cache_tag(str(RUNTIME / "include"), "", "") def arch_abi() -> str: diff --git a/runtime/tests/test_release_zip.py b/runtime/tests/test_release_zip.py index 2431963a..73131af2 100644 --- a/runtime/tests/test_release_zip.py +++ b/runtime/tests/test_release_zip.py @@ -16,8 +16,8 @@ with tempfile.TemporaryDirectory() as temporary: stage = base / "stage" (stage / "mods" / "package").mkdir(parents=True) (stage / "Tomba Recompiled.exe").write_bytes(b"exe") - (stage / "mods" / "package" / "manifest.toml").write_text( - 'id = "test"\n', encoding="utf-8" + (stage / "mods" / "package" / "manifest.toml").write_bytes( + b'id = "test"\n' ) output = base / "release.zip" subprocess.run( From b9e49074ea881f424309b8d947ad5d0b2812f628 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Fri, 11 Sep 2026 23:50:15 -0700 Subject: [PATCH 8/9] Record combined Tomba and MMX6 BIOS regression evidence --- docs/internal/FAITHFUL_TIMING_PLAN.md | 20 ++++ docs/internal/WO6_BIOS_INTEGRATION_REVIEW.md | 105 +++++++++++++++++++ recompiler/src/main_psx.cpp | 4 +- 3 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 docs/internal/WO6_BIOS_INTEGRATION_REVIEW.md diff --git a/docs/internal/FAITHFUL_TIMING_PLAN.md b/docs/internal/FAITHFUL_TIMING_PLAN.md index 72a2f20e..108fd06c 100644 --- a/docs/internal/FAITHFUL_TIMING_PLAN.md +++ b/docs/internal/FAITHFUL_TIMING_PLAN.md @@ -213,6 +213,26 @@ on a fixed region -> next. ## 5. Status / Log (update every session) +- **2026-09-11 (WO-6/WO-7 and BIOS #343/#346 combined validation):** + Branch `integrate/wo6-bios343-346` preserves both contributor PR histories. + Resident game dispatch now uses an immutable physical-word index, retaining + live-byte validation, flat CPS returns and interrupt service. WO-7's sampled + owner activations include continuations, unlike function-entry counts; the + telemetry label now states that distinction and its collision lower bound. + BIOS setup checks configured, linkable backend pairs in the configured + framework location. Patch-range guards run before cycles and terminators, + cover interior entry labels, and fail closed at unsafe delay boundaries. + Fresh Tomba/MMX6 baseline and integration builds each passed 11,000-frame + OpenBIOS and SCPH-1001 LLE runs with screenshot and native-coverage checks. + The first baseline Tomba screenshot request and first integration MMX6 cold + startup exceeded harness timeouts; retries completed. Recompiler tests pass + (65 enabled); runtime tests pass (78 executed, one explicit skip, two disabled). + The disabled overlay-pair executable regression also passes when run directly. + Refresh against master `b4ea4c37` and all eight final live reruns pass, with + clean exits and completed card reads. See + [the integration review](WO6_BIOS_INTEGRATION_REVIEW.md) for evidence and + coverage limits. Tracked by `beads-eio.3.140`. + - **2026-08-31 (GPU DMA2 review correction — source gate passed):** The first fork review found two valid timing defects in the DMA2 candidate. The linked-list engine now reads and emits one live payload word at each diff --git a/docs/internal/WO6_BIOS_INTEGRATION_REVIEW.md b/docs/internal/WO6_BIOS_INTEGRATION_REVIEW.md new file mode 100644 index 00000000..b07ef45d --- /dev/null +++ b/docs/internal/WO6_BIOS_INTEGRATION_REVIEW.md @@ -0,0 +1,105 @@ +# WO-6 / WO-7 and BIOS #343 / #346 integration + +Tracked by `beads-eio.3.140`. Integration branch: +`integrate/wo6-bios343-346`. Contributor heads `c2f9c3d5` (#343) and +`baca0a8a` (#346) are retained as ancestors, not copied or squashed. +Initial baseline: `736c002a`; refreshed baseline: `b4ea4c37`. + +## Changes and conclusions + +- WO-6: generate an unconditional immutable physical-word index for resident + dispatch tables spanning less than 2 MiB. Wide/ambiguous tables retain the + original binary search. Keep CPS returns, live-code validation, interrupt + checks and mod hooks. This accelerates address resolution; it does not turn + overlay calls into nested C calls or promise fewer dispatcher activations. + Tomba's 18,130-entry index is 265,706 bytes; MMX6's 18,858-entry index is + 255,918 bytes. An isolated benchmark of the actual emitted lookup with + simplified table records and 4,096 repeated mixed-hit queries measured + roughly 54-61 ns/query for binary search and 1.2-1.6 ns/query for the index. + This is **not** a measurement of total dispatch overhead or game speedup. +- WO-7: the native hot-owner sampler counts owner activations, including CPS + continuation returns. Function-entry tracing counts invocations. Its + direct-mapped bucket resets on a collision, so it can undercount but does not + accumulate another owner's calls. Clarify the telemetry label and API docs; + no counter algorithm change or confirmed counter corruption. The supplied + brief's 3.2 microseconds divides total guest work by activations and therefore + does not isolate dispatch cost. Its branch at `0x80191BC0` with immediate + `FFFB` targets `0x80191BB0`, not `0x80191BB4`. +- #343: setup detection now requires a configured dispatch/full pair and its + matching backend descriptor, and uses the configured framework location. + Preserve expected-retail discovery for `OpenBIOS;SCPH1001` and alternate + retail stems. Add the CLI test file registered but missing from the PR. +- #346: guard patched BIOS ranges before instruction charges and terminators, + including entry through interior labels. Compare the whole declared range + and hand off at the requested RAM PC. Reject native emission when the range + begins across a live branch/load-delay boundary. Do not hand back from the + interpreter with a pending load. Clear range metadata on memory reset. + No patch ranges were widened and no guest patch effects were synthesized. + +## Validation + +Windows native Clang 22, Release recompiler / RelWithDebInfo runtimes; fresh +game and BIOS generation, isolated builds, overlay caches and memory cards. +Real title-owned mods were built/staged; no original game output or save was +modified. Live runs use LLE BIOS boot/calls, headless framebuffer readback, +debug diagnostics and native GCC/Clang overlay compilation. UI, Vulkan, +netplay, rewind and PGXP are disabled in this regression configuration. + +Recompiler CTest: 65 enabled tests pass, including the executable generated +lookup check (aliases, holes, alignment, bounds, stale-code rejection and +interrupt/call behavior). Runtime CTest: 78 executed tests pass; one explicit +skip and two disabled. Recompiler has three disabled tests; its disabled +overlay-pair test was run directly and passed all five executable scenarios. +The #343 host probe was also run with real recomp-ui headers and native CC: +seven positive/negative cases pass. Upstream relocation/AOT tests: 4 + 24 pass. + +The test-harness repairs include two Windows text/byte fixtures, a stale +assertion that demanded unsafe continuation-range clipping, a cache-tag +fixture now using the canonical serializer, and a Windows stack allowance for +the mod-runtime test's real 1 MiB disc-hashing scratch buffer. None changes +production guest behavior. + +Both linked BIOS images were regenerated. OpenBIOS emits 651 functions with +zero tagged interpreter fallbacks and four existing unsupported-instruction +skips; retail emits 1,309 with its one existing load-delay fallback and one +existing unsupported-instruction skip. +Retail SHA-256: +`71af94d1e47a68c11e8fdb9f8368040601514a42a5a399cda48c7d3bff1e99d3`. + +Initial matrix: all eight combinations of Tomba/MMX6, OpenBIOS/SCPH-1001 and +baseline/integration completed at least 11,000 frames. Screenshots show real +FMV and MMX6's gameplay demo; integration has native overlay/kernel execution, +zero remaining kernel-body mismatches, and completed 128-byte card reads. +The first baseline Tomba screenshot exceeded a five-second TCP deadline; +its rerun passed. The first integration MMX6 run spent most of its budget in +host startup and reached only 1,414 frames; a longer-budget retry passed. +The baseline also exhibited slow host startup. Do not present either failed +attempt as a successful run or interpret cold-start times as dispatch cost. + +The rebuilt eight-way matrix against `b4ea4c37` passed, with clean process +exits and 32 completed 128-byte card transactions per run: + +| Game | BIOS | Baseline frames | Integration frames | Kernel mismatches, baseline -> integration | +| --- | --- | ---: | ---: | ---: | +| Tomba | OpenBIOS | 11,005 | 11,015 | 22 -> 0 | +| Tomba | SCPH-1001 | 11,009 | 11,008 | 11 -> 0 | +| MMX6 | OpenBIOS | 11,002 | 11,013 | 7 -> 0 | +| MMX6 | SCPH-1001 | 11,014 | 11,015 | 11 -> 0 | + +Each integration run has native overlays plus native kernel execution and +11-33 patch-skip events. Observed warm wall times were 82.5/86.2 seconds for +Tomba/OpenBIOS, 71.6/74.2 for Tomba/retail, 70.2/69.3 for MMX6/OpenBIOS and +70.6/70.8 for MMX6/retail (integration/baseline). These runs share a busy host +and are not controlled FPS benchmarks. Retail Tomba's frame-1,500 checkpoint +falls in a display-disabled boot interval on both builds; subsequent +checkpoints show the same FMV. Local raw reports/screenshots are under +`F:/Projects/psxrecomp/_validation-wo6-bios/{baseline,integration}/{tomba,mmx6}/results/`; +`final-openbios` and `final-SCPH1001` identify the refreshed runs. + +## Scope of confidence + +This is regression evidence for the tested boot/FMVs/attract paths, generated +dispatch contracts, BIOS patch boundaries and card protocol, not a full +playthrough, hardware differential proof, audible-audio assessment, UI test, +save/load round trip, cross-platform certification or Parasite Eve FPS result. +Tomba 2 and alternate retail BIOS images were not part of this matrix. diff --git a/recompiler/src/main_psx.cpp b/recompiler/src/main_psx.cpp index a2649d50..25e7f72c 100644 --- a/recompiler/src/main_psx.cpp +++ b/recompiler/src/main_psx.cpp @@ -1606,8 +1606,8 @@ int main(int argc, char** argv) { // time instead of binary-searching on every overlay-to-EXE call and // every text-validity query. This caches resolution, NEVER validity: // callers below still check live instruction ranges on every dispatch. - // Bound storage to one PS1 RAM image; unusual sparse/non-RAM tables - // retain the existing binary search. No mutable cache or new flag. + // Bound the indexed span to one PS1 RAM image; wider or ambiguous + // tables retain the existing binary search. No mutable cache or flag. const uint32_t lookup_lo = records.empty() ? 0u : (records.front().addr & 0x1FFFFFFFu); const uint32_t lookup_hi = records.empty() ? 0u : (records.back().addr & 0x1FFFFFFFu); bool indexed_lookup = !records.empty() && lookup_hi - lookup_lo < 0x200000u; From 1c7e62a58a4bd89c4af9dc7a5e9c5eaf871aa510 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Fri, 11 Sep 2026 23:51:43 -0700 Subject: [PATCH 9/9] Record matched MMX6 demo fade-out validation --- docs/internal/WO6_BIOS_INTEGRATION_REVIEW.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/internal/WO6_BIOS_INTEGRATION_REVIEW.md b/docs/internal/WO6_BIOS_INTEGRATION_REVIEW.md index b07ef45d..f231a394 100644 --- a/docs/internal/WO6_BIOS_INTEGRATION_REVIEW.md +++ b/docs/internal/WO6_BIOS_INTEGRATION_REVIEW.md @@ -96,6 +96,14 @@ checkpoints show the same FMV. Local raw reports/screenshots are under `F:/Projects/psxrecomp/_validation-wo6-bios/{baseline,integration}/{tomba,mmx6}/results/`; `final-openbios` and `final-SCPH1001` identify the refreshed runs. +An additional OpenBIOS/MMX6 run on each side captured the demo fade-out at +roughly 20-frame intervals through frame 11,150. Both show the same fade, +brief display-disable interval, and return to the 512-wide Capcom screen. +This explains the darker initial integration screenshot sampled 11 frames +later than baseline; it is not missing background rendering. + +Integration PR: [#348](https://github.com/RetroPortingToolKit/psxrecomp/pull/348). + ## Scope of confidence This is regression evidence for the tested boot/FMVs/attract paths, generated