From 6b6e2c712e491ac1f36987e2927eb4cf4866b804 Mon Sep 17 00:00:00 2001 From: Shokunin <30949000+tetrisgm@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:56:55 -0700 Subject: [PATCH] overlay: run shards only for the segment they were compiled for Overlay shards are compiled at the KSEG0 address of their bytes: captures record 0x80000000|offset and the manifest parser keys entries there. The emitter bakes that segment into every PC a shard produces: the $ra a jal/jalr writes, jump and exception-resume PCs, and I-cache fetch tags. Dispatch matched candidates by the segment-stripped address, so a KUSEG or KSEG1 alias of the same bytes ran the KSEG0 body at the wrong architectural PC. Found in R4 (SLUS-00797) with OpenBIOS: the exception handler enters its RAM patch slots at KUSEG (0x0000281C, 0x0000357C). The kernel-page shard linked $ra=0x8000282C where the interpreter (and hardware) gives 0x0000282C, and fetched 0x8000281C/0x8000357C, missing lines the kernel had just filled with KUSEG tags: +4 cycles per slot, so the next VBlank IRQ landed at a different instruction and warm (native) vs cold (interpreted) overlay-cache runs saved different registers from frame 194 on. A KSEG1 entry would also be charged as cached. PSX_OVERLAY_CODE_SEGMENT (overlay_loader.h) names the rule for capture, the manifest parser, overlay_loader_dispatch and the generated static overlay table: a PC outside it falls to the interpreter, which runs at the PC it was handed. CPS continuations already failed closed this way (their entry switch only knows KSEG0 PCs); function entries now match. overlay_loader_status reports segment_alias_interp. Static game code has the same shape for KUSEG-linked EXEs (the parser normalizes their headers to KSEG0); that needs segment-relative emission and is not changed here. Shard cache tags are unchanged. Tests: test_overlay_segment_gate.py drives the real loader (pair-dedup harness, which also gains the RAM-geometry globals it no longer linked without) with a KSEG0 fixture shard: KUSEG and KSEG1 dispatches stay interpreted and never call it, KSEG0 still runs native. It fails with the gate removed. test_compile_overlays_static_split.py covers the generated static table and rejects a non-KSEG0 entry. Co-Authored-By: Claude Opus 5.5 --- recompiler/CMakeLists.txt | 1 + runtime/include/overlay_loader.h | 16 +++++ runtime/src/debug_server.c | 6 +- runtime/src/overlay_capture.c | 8 +-- runtime/src/overlay_loader.c | 13 ++++- runtime/tests/overlay_pair_dedup_harness.c | 38 ++++++++++++ runtime/tests/test_overlay_segment_gate.py | 60 +++++++++++++++++++ tools/compile_overlays.py | 14 ++++- tools/test_compile_overlays_static_split.py | 65 +++++++++++++++++++++ 9 files changed, 213 insertions(+), 8 deletions(-) create mode 100644 runtime/tests/test_overlay_segment_gate.py diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index b58a379d..c6658b07 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -829,6 +829,7 @@ if(BUILD_TESTING) release_zip overlay_decodable_fallback overlay_init_guard + overlay_segment_gate recomp_ui_root_override pal_cadence_host_refresh sdl3_main_single_include diff --git a/runtime/include/overlay_loader.h b/runtime/include/overlay_loader.h index 5cd8c46e..abb94489 100644 --- a/runtime/include/overlay_loader.h +++ b/runtime/include/overlay_loader.h @@ -19,6 +19,19 @@ extern "C" { #endif +/* The segment every overlay shard is compiled for. Captures record their + * bytes at the KSEG0 address (overlay_capture.c) and the manifest parser keys + * entries there; the emitter bakes that segment into every PC a shard makes: + * the $ra a jal/jalr writes, its jump and exception-resume PCs, and its I-cache + * fetch tags. The same bytes reached through KUSEG or KSEG1 run at a different + * architectural PC (a different link value; KSEG1 is uncached), so a shard runs + * only for PCs in this segment and any other alias is interpreted at its own + * PC. (OpenBIOS's exception path enters its RAM patch slots at KUSEG.) */ +#define PSX_OVERLAY_CODE_SEGMENT 0x80000000u +static inline int psx_overlay_code_segment_pc(uint32_t pc) { + return (pc & 0xE0000000u) == PSX_OVERLAY_CODE_SEGMENT; +} + /* Called at game handoff to set the cache root directory and game ID. * cache_dir: absolute path to the cache root (e.g. "build-dev/cache") * game_id: product code (e.g. "SCUS-94236") */ @@ -138,6 +151,9 @@ int overlay_loader_lazy_manifest_count(void); int overlay_loader_lazy_manifest_overflow(void); uint64_t overlay_loader_candidate_overflow(void); uint64_t overlay_loader_pair_aliases(void); +/* Dispatches sent to the interpreter because the PC is a KUSEG/KSEG1 alias of + * code compiled for PSX_OVERLAY_CODE_SEGMENT. */ +uint64_t overlay_loader_segment_alias_interp(void); int overlay_loader_dump_lazy_at(uint32_t addr, char *out, int cap); /* Overlay CI wrapper early-return attribution (PSX_POST_LOAD_PROBE). */ diff --git a/runtime/src/debug_server.c b/runtime/src/debug_server.c index 655d03f4..e54d47e4 100644 --- a/runtime/src/debug_server.c +++ b/runtime/src/debug_server.c @@ -12262,7 +12262,8 @@ static void handle_overlay_loader_status(int id, const char *json) "\"reval_attempts\":%u,\"reval_crc_miss\":%u,\"last_reval_crc\":\"0x%08X\"," "\"gen_fastpath\":%llu,\"range_links\":%d,\"range_index_overflow\":%d," "\"lazy_manifests\":%d,\"lazy_manifest_overflow\":%d," - "\"candidate_overflow\":%llu,\"pair_aliases\":%llu", + "\"candidate_overflow\":%llu,\"pair_aliases\":%llu," + "\"segment_alias_interp\":%llu", r0v, r0w, r0lo, r0hi, r0crc, ratt, rmiss, rlast, (unsigned long long)overlay_loader_gen_fastpath(), overlay_loader_range_link_count(), @@ -12270,7 +12271,8 @@ static void handle_overlay_loader_status(int id, const char *json) overlay_loader_lazy_manifest_count(), overlay_loader_lazy_manifest_overflow(), (unsigned long long)overlay_loader_candidate_overflow(), - (unsigned long long)overlay_loader_pair_aliases()); + (unsigned long long)overlay_loader_pair_aliases(), + (unsigned long long)overlay_loader_segment_alias_interp()); uint64_t nd=0, ni=0, sn=0, ss=0, sc=0, sx=0; psx_interrupt_delivery_diag(&nd, &ni, &sn, &ss, &sc, &sx); n += snprintf(buf + n, sizeof(buf) - n, diff --git a/runtime/src/overlay_capture.c b/runtime/src/overlay_capture.c index 6db3fa61..293a5c6c 100644 --- a/runtime/src/overlay_capture.c +++ b/runtime/src/overlay_capture.c @@ -362,7 +362,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page, size += 4u; guard_bytes = 4u; } - uint32_t virt = 0x80000000u | psx_ram_canonical_offset(phys); + uint32_t virt = PSX_OVERLAY_CODE_SEGMENT | psx_ram_canonical_offset(phys); in_run = 0; /* Seeds: only per-PC interpreter hits — execution-verified. */ @@ -417,7 +417,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page, uint32_t wi = ep >> 2; if (!((exec_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u)) continue; - uint32_t seed_virt = 0x80000000u | ep; + uint32_t seed_virt = PSX_OVERLAY_CODE_SEGMENT | ep; if (emitted_exec++) fprintf(f, ", "); fprintf(f, "\"0x%08X\"", seed_virt); } @@ -429,7 +429,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page, uint32_t wi = ep >> 2; if (!((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u)) continue; - uint32_t seed_virt = 0x80000000u | ep; + uint32_t seed_virt = PSX_OVERLAY_CODE_SEGMENT | ep; if (emitted_dispatch++) fprintf(f, ", "); fprintf(f, "\"0x%08X\"", seed_virt); } @@ -443,7 +443,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page, uint32_t wi = ep >> 2; if (!((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u)) continue; - uint32_t seed_virt = 0x80000000u | ep; + uint32_t seed_virt = PSX_OVERLAY_CODE_SEGMENT | ep; if (emitted_dispatch++) fprintf(f, ", "); fprintf(f, "\"0x%08X\"", seed_virt); } diff --git a/runtime/src/overlay_loader.c b/runtime/src/overlay_loader.c index cac04bb4..b85309cd 100644 --- a/runtime/src/overlay_loader.c +++ b/runtime/src/overlay_loader.c @@ -501,6 +501,7 @@ static uint32_t s_rehash_miss = 0; /* hashes that didn't match crc_code */ static uint64_t s_gen_fastpath = 0; /* dispatches that skipped the crc32 */ /* via the unchanged page-generation */ /* fast path (overlay-cache v2 P2) */ +static uint64_t s_segment_alias_interp = 0; /* KUSEG/KSEG1 PCs kept off shards */ static uint64_t s_diffgate_interp = 0; /* CPS interior re-entries sent to the */ /* interp because their candidate is */ /* still inside the diff verify budget */ @@ -938,7 +939,7 @@ static ManFn *parse_manifest(const char *path, int *out_n, arr = na; } cur = &arr[n++]; - cur->entry = ((uint32_t)parsed_e & 0x1FFFFFFFu) | 0x80000000u; + cur->entry = ((uint32_t)parsed_e & 0x1FFFFFFFu) | PSX_OVERLAY_CODE_SEGMENT; cur->crc = (uint32_t)parsed_crc; cur->has_crc = 1; cur->n = 0; @@ -3660,6 +3661,15 @@ int overlay_loader_dispatch(CPUState *cpu, uint32_t addr) { * overlay-off + CPS game hit this and wedged at boot before any game code ran * (found via Ape Escape, the only overlay-off title). Fail closed here. */ if (!s_active) return 0; + /* A shard's body bakes the KSEG0 PCs it was compiled for; a KUSEG or KSEG1 + * alias of its bytes must run at its own PC ($ra, EPC, I-cache tags). The + * CPS continuation switch already rejects such a PC (bad entry); this + * keeps function entries to the same rule. */ + if (!psx_overlay_code_segment_pc(addr)) { + s_segment_alias_interp++; + s_disp_interp++; + return 0; + } if (overlay_cache_window_contains(phys) && lazy_miss_cached(phys)) { s_disp_interp++; return 0; @@ -4096,6 +4106,7 @@ int overlay_loader_lazy_manifest_count(void) { return s_lazy_man_n; } int overlay_loader_lazy_manifest_overflow(void) { return s_lazy_man_overflow; } uint64_t overlay_loader_candidate_overflow(void) { return s_cand_overflow; } uint64_t overlay_loader_pair_aliases(void) { return s_pair_aliases; } +uint64_t overlay_loader_segment_alias_interp(void) { return s_segment_alias_interp; } /* (sljit removed 2026-07-15: overlay_loader_sljit_obsoleted and the * overlay_loader_sljit_probe one-shot JIT helper lived here.) */ diff --git a/runtime/tests/overlay_pair_dedup_harness.c b/runtime/tests/overlay_pair_dedup_harness.c index 4fc6e554..4805e77b 100644 --- a/runtime/tests/overlay_pair_dedup_harness.c +++ b/runtime/tests/overlay_pair_dedup_harness.c @@ -23,6 +23,10 @@ _Static_assert(PSX_OVERLAY_TEST_CANDIDATE_CAP == 4, static uint8_t s_ram[RAM_SIZE]; static uint8_t s_scratch[1024]; +/* Live RAM geometry (psx_memory.h): retail 2 MiB. */ +uint32_t g_psx_ram_size = RAM_SIZE; +uint32_t g_psx_ram_mask = RAM_SIZE - 1u; + uint32_t g_debug_current_func_addr; uint32_t g_debug_last_store_pc; uint32_t g_overlay_region_floor; @@ -302,6 +306,39 @@ static int reveal_second_pair(const char *second) { return 1; } +/* A shard compiled for KSEG0 runs only for KSEG0 PCs. The same bytes reached + * through KUSEG or KSEG1 are a different architectural PC ($ra, EPC, I-cache + * tag), so those dispatches must fall to the interpreter without touching the + * shard (overlay_loader.h, PSX_OVERLAY_CODE_SEGMENT). */ +static int segment_alias(const char *first) { + CPUState cpu; + int ok = 1; + ok &= expect_int("registered", overlay_loader_registered_count(), 2); + static const uint32_t aliases[] = { 0x00010000u, 0xA0010000u }; + for (unsigned i = 0; i < sizeof(aliases) / sizeof(aliases[0]); i++) { + memset(&cpu, 0, sizeof(cpu)); + ok &= expect_int("alias dispatch", + overlay_loader_dispatch(&cpu, aliases[i]), 0); + ok &= expect_int("alias left registers", cpu.gpr[2], 0); + } + ok &= expect_int("alias calls", counter_value(first, "test_call_count"), 0); + ok &= expect_int("alias counter", + (long long)overlay_loader_segment_alias_interp(), 2); + memset(&cpu, 0, sizeof(cpu)); + ok &= expect_int("kseg0 dispatch", + overlay_loader_dispatch(&cpu, 0x80010000u), 1); + ok &= expect_int("kseg0 marker", cpu.gpr[2], TEST_MARKER); + ok &= expect_int("kseg0 calls", counter_value(first, "test_call_count"), 1); + ok &= expect_int("kseg0 counter", + (long long)overlay_loader_segment_alias_interp(), 2); + if (!ok) { + fprintf(stderr, "loader: %s\n", overlay_loader_last_msg()); + return 1; + } + printf("PASS segment-alias\n"); + return 0; +} + int main(int argc, char **argv) { if (argc != 5) { fprintf(stderr, "usage: %s \n", @@ -313,6 +350,7 @@ int main(int argc, char **argv) { const char *second = argv[4]; memset(s_ram, 0, sizeof(s_ram)); overlay_loader_init(argv[1], "PAIR-TEST", 0); + if (strcmp(scenario, "segment-alias") == 0) return segment_alias(first); int alias = strcmp(scenario, "alias-at-cap") == 0; int partial = strcmp(scenario, "partial-first") == 0; diff --git a/runtime/tests/test_overlay_segment_gate.py b/runtime/tests/test_overlay_segment_gate.py new file mode 100644 index 00000000..2449b806 --- /dev/null +++ b/runtime/tests/test_overlay_segment_gate.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Build the real overlay loader and prove shards run only for their segment. + +Overlay shards are compiled for the KSEG0 address of their bytes and bake that +segment into every PC they produce: jal/jalr link values, jump and exception +resume PCs, I-cache fetch tags. OpenBIOS's exception path enters its RAM patch +slots at KUSEG (0x0000281C), and a KSEG0 shard run there wrote $ra=0x8000282C +instead of 0x0000282C and missed I-cache lines the kernel had just filled with +KUSEG tags: R4 warm (native) and cold (interpreted) overlay-cache runs took the +next VBlank IRQ at different instructions and split at guest frame 194. + +This drives the loader harness (overlay_pair_dedup_harness.c) with a fixture +shard at 0x80010000: KUSEG and KSEG1 aliases must fall to the interpreter +without calling the shard, and the KSEG0 PC must still run native. + +Usage: python runtime/tests/test_overlay_segment_gate.py +Exit 0 = PASS. +""" + +from __future__ import annotations + +import argparse +import os +import pathlib +import platform +import shutil +import sys +import tempfile + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +import test_overlay_pair_dedup_runtime as pair # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--gcc", default=shutil.which("gcc") or shutil.which("cc")) + args = parser.parse_args() + if not args.gcc: + raise SystemExit("gcc/cc is required") + ext = ".dll" if platform.system() == "Windows" else ".so" + exe = ".exe" if platform.system() == "Windows" else "" + with tempfile.TemporaryDirectory( + prefix="psx-segment-gate-", ignore_cleanup_errors=True) as raw: + tmp = pathlib.Path(raw) + shard = tmp / f"shard{ext}" + harness = tmp / f"segment-harness{exe}" + pair.compile_fixture(args.gcc, shard, instance=1) + pair.compile_harness(args.gcc, harness) + cache = tmp / "cache" + first = pair.publish(cache, "gcc", f"00010000_11111111{ext}", shard, + pair.manifest([(0x80010000, 4), (0x80010004, 4)])) + os.environ.pop("PSX_PAIR_TEST_TRACE", None) + pair.run([str(harness), str(cache), "segment-alias", str(first), + str(tmp / f"unused{ext}")]) + print("PASS: overlay shards run only for the segment they were compiled for") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index 2a83a394..105407e4 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -2532,6 +2532,10 @@ def generate_overlay_dispatch(variants: list) -> str: unique = [] seen = set() for variant in variants: + # The lookup keys on the exact PC; every body is compiled for KSEG0. + if variant['addr'] & 0xE0000000 != 0x80000000: + raise ValueError(f"static overlay entry 0x{variant['addr']:08X} " + "is not a KSEG0 PC") ranges = tuple((lo & 0x1FFFFFFF, length) for lo, length in variant['ranges']) key = (variant['addr'], variant['crc'], ranges) @@ -2712,7 +2716,15 @@ def generate_overlay_dispatch(variants: list) -> str: 'static uint32_t psx_ov_found_entry = 0;', '', 'static const PsxOvVariant *psx_overlay_static_find_variant(uint32_t addr) {', - ' const uint32_t key = (addr & 0x1FFFFFFFu) | 0x80000000u;', + ' /* Entries are the KSEG0 PCs their bodies were compiled for', + ' * (PSX_OVERLAY_CODE_SEGMENT, overlay_loader.h). A KUSEG or KSEG1', + ' * alias of the same bytes is a different architectural PC and', + ' * stays in the interpreter. */', + ' if ((addr & 0xE0000000u) != 0x80000000u) {', + ' psx_ov_static_address_misses++;', + ' return 0;', + ' }', + ' const uint32_t key = addr;', ] if not flat_variants: diff --git a/tools/test_compile_overlays_static_split.py b/tools/test_compile_overlays_static_split.py index 2357cbe5..f00a2265 100644 --- a/tools/test_compile_overlays_static_split.py +++ b/tools/test_compile_overlays_static_split.py @@ -225,6 +225,71 @@ class StaticDispatchApiTests(unittest.TestCase): subprocess.run([exe], check=True) + def test_segment_aliases_stay_interpreted(self): + # Bodies bake the KSEG0 PCs they were compiled for ($ra, EPC, I-cache + # tags); a KUSEG/KSEG1 alias of the same bytes must not reach them. + cc = shutil.which('gcc') or shutil.which('clang') + if cc is None: + self.skipTest('no C compiler available') + variants = [{'addr': 0x8000281C, 'symbol': 'ov_func_k', 'crc': 0x11111111, + 'ranges': ((0x0000281C, 16),)}] + dispatch = compile_overlays.generate_overlay_dispatch(variants) + with tempfile.TemporaryDirectory() as tmp: + Path(tmp, 'psx_runtime.h').write_text( + '#include \n' + 'typedef struct CPUState { uint32_t pc; } CPUState;\n', + encoding='utf-8') + exe = os.path.join(tmp, 'segment.exe' if os.name == 'nt' + else 'segment') + src = os.path.join(tmp, 'segment.c') + Path(src).write_text(textwrap.dedent(f''' + #include "psx_runtime.h" + + void ov_func_k(CPUState *cpu); + + {dispatch} + + static int calls; + + int psx_overlay_static_code_matches( + const uint32_t *lo_len_pairs, uint32_t count, + uint32_t expected_crc) + {{ + (void)lo_len_pairs; + (void)count; + return expected_crc == 0x11111111u; + }} + + void ov_func_k(CPUState *cpu) {{ (void)cpu; calls++; }} + + int main(void) + {{ + CPUState cpu = {{0}}; + uint64_t checks = 0, hits = 0, vm = 0, am = 0; + if (psx_overlay_static_can_dispatch(0x0000281Cu)) return 1; + if (psx_overlay_static_can_dispatch(0xA000281Cu)) return 2; + if (psx_overlay_dispatch(&cpu, 0x0000281Cu)) return 3; + if (psx_overlay_dispatch(&cpu, 0xA000281Cu)) return 4; + if (calls != 0) return 5; + psx_overlay_static_get_stats(&checks, &hits, &vm, &am); + if (checks != 0 || hits != 0 || am != 4) return 6; + if (!psx_overlay_static_can_dispatch(0x8000281Cu)) return 7; + if (!psx_overlay_dispatch(&cpu, 0x8000281Cu)) return 8; + if (calls != 1) return 9; + return 0; + }} + '''), encoding='utf-8') + subprocess.run([cc, src, '-I', tmp, '-std=c99', '-Wall', '-Werror', + '-o', exe], check=True) + subprocess.run([exe], check=True) + + def test_non_kseg0_entry_is_rejected(self): + with self.assertRaises(ValueError): + compile_overlays.generate_overlay_dispatch( + [{'addr': 0x0000281C, 'symbol': 'ov_func_k', 'crc': 1, + 'ranges': ((0x0000281C, 16),)}]) + + class StaticWorkerContractTests(unittest.TestCase): def test_worker_is_picklable_for_a_process_pool(self): # ProcessPoolExecutor pickles the callable by qualified name; a nested