mirror/psxrecomp
1
0
Fork 0
mirror of https://github.com/RetroPortingToolKit/psxrecomp synced 2026-10-26 02:11:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #417 from RetroPortingToolKit/fix/overlay-segment-alias

Browse Source 
overlay: run shards only for the segment they were compiled for
This commit is contained in:
Matt Stanley
2026-09-29 11:19:48 -07:00
committed by GitHub
parent d6dc76e149 6b6e2c712e
commit 3a7ea5e890
9 changed files with 213 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
recompiler/CMakeLists.txt
+1
View File
@@ -829,6 +829,7 @@ if(BUILD_TESTING)
release_zip
overlay_decodable_fallback
overlay_init_guard
overlay_segment_gate
recomp_ui_root_override
pal_cadence_host_refresh
sdl3_main_single_include
runtime/include/overlay_loader.h
+16
View File
@@ -19,6 +19,19 @@
extern "C" {
#endif
/* The segment every overlay shard is compiled for. Captures record their
* bytes at the KSEG0 address (overlay_capture.c) and the manifest parser keys
* entries there; the emitter bakes that segment into every PC a shard makes:
* the $ra a jal/jalr writes, its jump and exception-resume PCs, and its I-cache
* fetch tags. The same bytes reached through KUSEG or KSEG1 run at a different
* architectural PC (a different link value; KSEG1 is uncached), so a shard runs
* only for PCs in this segment and any other alias is interpreted at its own
* PC. (OpenBIOS's exception path enters its RAM patch slots at KUSEG.) */
#define PSX_OVERLAY_CODE_SEGMENT 0x80000000u
static inline int psx_overlay_code_segment_pc(uint32_t pc) {
return (pc & 0xE0000000u) == PSX_OVERLAY_CODE_SEGMENT;
}
/* Called at game handoff to set the cache root directory and game ID.
* cache_dir: absolute path to the cache root (e.g. "build-dev/cache")
* game_id: product code (e.g. "SCUS-94236") */
@@ -138,6 +151,9 @@ int overlay_loader_lazy_manifest_count(void);
int overlay_loader_lazy_manifest_overflow(void);
uint64_t overlay_loader_candidate_overflow(void);
uint64_t overlay_loader_pair_aliases(void);
/* Dispatches sent to the interpreter because the PC is a KUSEG/KSEG1 alias of
* code compiled for PSX_OVERLAY_CODE_SEGMENT. */
uint64_t overlay_loader_segment_alias_interp(void);
int overlay_loader_dump_lazy_at(uint32_t addr, char *out, int cap);
/* Overlay CI wrapper early-return attribution (PSX_POST_LOAD_PROBE). */
runtime/src/debug_server.c
+4 -2
View File
@@ -12262,7 +12262,8 @@ static void handle_overlay_loader_status(int id, const char *json)
"\"reval_attempts\":%u,\"reval_crc_miss\":%u,\"last_reval_crc\":\"0x%08X\","
"\"gen_fastpath\":%llu,\"range_links\":%d,\"range_index_overflow\":%d,"
"\"lazy_manifests\":%d,\"lazy_manifest_overflow\":%d,"
"\"candidate_overflow\":%llu,\"pair_aliases\":%llu",
"\"candidate_overflow\":%llu,\"pair_aliases\":%llu,"
"\"segment_alias_interp\":%llu",
r0v, r0w, r0lo, r0hi, r0crc, ratt, rmiss, rlast,
(unsigned long long)overlay_loader_gen_fastpath(),
overlay_loader_range_link_count(),
@@ -12270,7 +12271,8 @@ static void handle_overlay_loader_status(int id, const char *json)
overlay_loader_lazy_manifest_count(),
overlay_loader_lazy_manifest_overflow(),
(unsigned long long)overlay_loader_candidate_overflow(),
(unsigned long long)overlay_loader_pair_aliases());
(unsigned long long)overlay_loader_pair_aliases(),
(unsigned long long)overlay_loader_segment_alias_interp());
uint64_t nd=0, ni=0, sn=0, ss=0, sc=0, sx=0;
psx_interrupt_delivery_diag(&nd, &ni, &sn, &ss, &sc, &sx);
n += snprintf(buf + n, sizeof(buf) - n,
runtime/src/overlay_capture.c
+4 -4
View File
@@ -362,7 +362,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
size += 4u;
guard_bytes = 4u;
}
uint32_t virt = 0x80000000u | psx_ram_canonical_offset(phys);
uint32_t virt = PSX_OVERLAY_CODE_SEGMENT | psx_ram_canonical_offset(phys);
in_run = 0;
/* Seeds: only per-PC interpreter hits — execution-verified. */
@@ -417,7 +417,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
uint32_t wi = ep >> 2;
if (!((exec_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u))
continue;
uint32_t seed_virt = 0x80000000u | ep;
uint32_t seed_virt = PSX_OVERLAY_CODE_SEGMENT | ep;
if (emitted_exec++) fprintf(f, ", ");
fprintf(f, "\"0x%08X\"", seed_virt);
}
@@ -429,7 +429,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
uint32_t wi = ep >> 2;
if (!((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u))
continue;
uint32_t seed_virt = 0x80000000u | ep;
uint32_t seed_virt = PSX_OVERLAY_CODE_SEGMENT | ep;
if (emitted_dispatch++) fprintf(f, ", ");
fprintf(f, "\"0x%08X\"", seed_virt);
}
@@ -443,7 +443,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
uint32_t wi = ep >> 2;
if (!((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u))
continue;
uint32_t seed_virt = 0x80000000u | ep;
uint32_t seed_virt = PSX_OVERLAY_CODE_SEGMENT | ep;
if (emitted_dispatch++) fprintf(f, ", ");
fprintf(f, "\"0x%08X\"", seed_virt);
}
runtime/src/overlay_loader.c
+12 -1
View File
@@ -501,6 +501,7 @@ static uint32_t s_rehash_miss = 0; /* hashes that didn't match crc_code */
static uint64_t s_gen_fastpath = 0; /* dispatches that skipped the crc32 */
/* via the unchanged page-generation */
/* fast path (overlay-cache v2 P2) */
static uint64_t s_segment_alias_interp = 0; /* KUSEG/KSEG1 PCs kept off shards */
static uint64_t s_diffgate_interp = 0; /* CPS interior re-entries sent to the */
/* interp because their candidate is */
/* still inside the diff verify budget */
@@ -938,7 +939,7 @@ static ManFn *parse_manifest(const char *path, int *out_n,
arr = na;
}
cur = &arr[n++];
cur->entry = ((uint32_t)parsed_e & 0x1FFFFFFFu) | 0x80000000u;
cur->entry = ((uint32_t)parsed_e & 0x1FFFFFFFu) | PSX_OVERLAY_CODE_SEGMENT;
cur->crc = (uint32_t)parsed_crc;
cur->has_crc = 1;
cur->n = 0;
@@ -3660,6 +3661,15 @@ int overlay_loader_dispatch(CPUState *cpu, uint32_t addr) {
* overlay-off + CPS game hit this and wedged at boot before any game code ran
* (found via Ape Escape, the only overlay-off title). Fail closed here. */
if (!s_active) return 0;
/* A shard's body bakes the KSEG0 PCs it was compiled for; a KUSEG or KSEG1
* alias of its bytes must run at its own PC ($ra, EPC, I-cache tags). The
* CPS continuation switch already rejects such a PC (bad entry); this
* keeps function entries to the same rule. */
if (!psx_overlay_code_segment_pc(addr)) {
s_segment_alias_interp++;
s_disp_interp++;
return 0;
}
if (overlay_cache_window_contains(phys) && lazy_miss_cached(phys)) {
s_disp_interp++;
return 0;
@@ -4096,6 +4106,7 @@ int overlay_loader_lazy_manifest_count(void) { return s_lazy_man_n; }
int overlay_loader_lazy_manifest_overflow(void) { return s_lazy_man_overflow; }
uint64_t overlay_loader_candidate_overflow(void) { return s_cand_overflow; }
uint64_t overlay_loader_pair_aliases(void) { return s_pair_aliases; }
uint64_t overlay_loader_segment_alias_interp(void) { return s_segment_alias_interp; }
/* (sljit removed 2026-07-15: overlay_loader_sljit_obsoleted and the
* overlay_loader_sljit_probe one-shot JIT helper lived here.) */
runtime/tests/overlay_pair_dedup_harness.c
+38
View File
@@ -23,6 +23,10 @@ _Static_assert(PSX_OVERLAY_TEST_CANDIDATE_CAP == 4,
static uint8_t s_ram[RAM_SIZE];
static uint8_t s_scratch[1024];
/* Live RAM geometry (psx_memory.h): retail 2 MiB. */
uint32_t g_psx_ram_size = RAM_SIZE;
uint32_t g_psx_ram_mask = RAM_SIZE - 1u;
uint32_t g_debug_current_func_addr;
uint32_t g_debug_last_store_pc;
uint32_t g_overlay_region_floor;
@@ -302,6 +306,39 @@ static int reveal_second_pair(const char *second) {
return 1;
}
/* A shard compiled for KSEG0 runs only for KSEG0 PCs. The same bytes reached
* through KUSEG or KSEG1 are a different architectural PC ($ra, EPC, I-cache
* tag), so those dispatches must fall to the interpreter without touching the
* shard (overlay_loader.h, PSX_OVERLAY_CODE_SEGMENT). */
static int segment_alias(const char *first) {
CPUState cpu;
int ok = 1;
ok &= expect_int("registered", overlay_loader_registered_count(), 2);
static const uint32_t aliases[] = { 0x00010000u, 0xA0010000u };
for (unsigned i = 0; i < sizeof(aliases) / sizeof(aliases[0]); i++) {
memset(&cpu, 0, sizeof(cpu));
ok &= expect_int("alias dispatch",
overlay_loader_dispatch(&cpu, aliases[i]), 0);
ok &= expect_int("alias left registers", cpu.gpr[2], 0);
}
ok &= expect_int("alias calls", counter_value(first, "test_call_count"), 0);
ok &= expect_int("alias counter",
(long long)overlay_loader_segment_alias_interp(), 2);
memset(&cpu, 0, sizeof(cpu));
ok &= expect_int("kseg0 dispatch",
overlay_loader_dispatch(&cpu, 0x80010000u), 1);
ok &= expect_int("kseg0 marker", cpu.gpr[2], TEST_MARKER);
ok &= expect_int("kseg0 calls", counter_value(first, "test_call_count"), 1);
ok &= expect_int("kseg0 counter",
(long long)overlay_loader_segment_alias_interp(), 2);
if (!ok) {
fprintf(stderr, "loader: %s\n", overlay_loader_last_msg());
return 1;
}
printf("PASS segment-alias\n");
return 0;
}
int main(int argc, char **argv) {
if (argc != 5) {
fprintf(stderr, "usage: %s <cache-root> <scenario> <first> <second>\n",
@@ -313,6 +350,7 @@ int main(int argc, char **argv) {
const char *second = argv[4];
memset(s_ram, 0, sizeof(s_ram));
overlay_loader_init(argv[1], "PAIR-TEST", 0);
if (strcmp(scenario, "segment-alias") == 0) return segment_alias(first);
int alias = strcmp(scenario, "alias-at-cap") == 0;
int partial = strcmp(scenario, "partial-first") == 0;
runtime/tests/test_overlay_segment_gate.py
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
"""Build the real overlay loader and prove shards run only for their segment.
Overlay shards are compiled for the KSEG0 address of their bytes and bake that
segment into every PC they produce: jal/jalr link values, jump and exception
resume PCs, I-cache fetch tags. OpenBIOS's exception path enters its RAM patch
slots at KUSEG (0x0000281C), and a KSEG0 shard run there wrote $ra=0x8000282C
instead of 0x0000282C and missed I-cache lines the kernel had just filled with
KUSEG tags: R4 warm (native) and cold (interpreted) overlay-cache runs took the
next VBlank IRQ at different instructions and split at guest frame 194.
This drives the loader harness (overlay_pair_dedup_harness.c) with a fixture
shard at 0x80010000: KUSEG and KSEG1 aliases must fall to the interpreter
without calling the shard, and the KSEG0 PC must still run native.
Usage: python runtime/tests/test_overlay_segment_gate.py
Exit 0 = PASS.
"""
from __future__ import annotations
import argparse
import os
import pathlib
import platform
import shutil
import sys
import tempfile
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
import test_overlay_pair_dedup_runtime as pair # noqa: E402
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--gcc", default=shutil.which("gcc") or shutil.which("cc"))
args = parser.parse_args()
if not args.gcc:
raise SystemExit("gcc/cc is required")
ext = ".dll" if platform.system() == "Windows" else ".so"
exe = ".exe" if platform.system() == "Windows" else ""
with tempfile.TemporaryDirectory(
prefix="psx-segment-gate-", ignore_cleanup_errors=True) as raw:
tmp = pathlib.Path(raw)
shard = tmp / f"shard{ext}"
harness = tmp / f"segment-harness{exe}"
pair.compile_fixture(args.gcc, shard, instance=1)
pair.compile_harness(args.gcc, harness)
cache = tmp / "cache"
first = pair.publish(cache, "gcc", f"00010000_11111111{ext}", shard,
pair.manifest([(0x80010000, 4), (0x80010004, 4)]))
os.environ.pop("PSX_PAIR_TEST_TRACE", None)
pair.run([str(harness), str(cache), "segment-alias", str(first),
str(tmp / f"unused{ext}")])
print("PASS: overlay shards run only for the segment they were compiled for")
return 0
if __name__ == "__main__":
raise SystemExit(main())
tools/compile_overlays.py
+13 -1
View File
@@ -2532,6 +2532,10 @@ def generate_overlay_dispatch(variants: list) -> str:
unique = []
seen = set()
for variant in variants:
# The lookup keys on the exact PC; every body is compiled for KSEG0.
if variant['addr'] & 0xE0000000 != 0x80000000:
raise ValueError(f"static overlay entry 0x{variant['addr']:08X} "
"is not a KSEG0 PC")
ranges = tuple((lo & 0x1FFFFFFF, length)
for lo, length in variant['ranges'])
key = (variant['addr'], variant['crc'], ranges)
@@ -2712,7 +2716,15 @@ def generate_overlay_dispatch(variants: list) -> str:
'static uint32_t psx_ov_found_entry = 0;',
'',
'static const PsxOvVariant *psx_overlay_static_find_variant(uint32_t addr) {',
' const uint32_t key = (addr & 0x1FFFFFFFu) | 0x80000000u;',
' /* Entries are the KSEG0 PCs their bodies were compiled for',
' * (PSX_OVERLAY_CODE_SEGMENT, overlay_loader.h). A KUSEG or KSEG1',
' * alias of the same bytes is a different architectural PC and',
' * stays in the interpreter. */',
' if ((addr & 0xE0000000u) != 0x80000000u) {',
' psx_ov_static_address_misses++;',
' return 0;',
' }',
' const uint32_t key = addr;',
]
if not flat_variants:
tools/test_compile_overlays_static_split.py
+65
View File
@@ -225,6 +225,71 @@ class StaticDispatchApiTests(unittest.TestCase):
subprocess.run([exe], check=True)
def test_segment_aliases_stay_interpreted(self):
# Bodies bake the KSEG0 PCs they were compiled for ($ra, EPC, I-cache
# tags); a KUSEG/KSEG1 alias of the same bytes must not reach them.
cc = shutil.which('gcc') or shutil.which('clang')
if cc is None:
self.skipTest('no C compiler available')
variants = [{'addr': 0x8000281C, 'symbol': 'ov_func_k', 'crc': 0x11111111,
'ranges': ((0x0000281C, 16),)}]
dispatch = compile_overlays.generate_overlay_dispatch(variants)
with tempfile.TemporaryDirectory() as tmp:
Path(tmp, 'psx_runtime.h').write_text(
'#include <stdint.h>\n'
'typedef struct CPUState { uint32_t pc; } CPUState;\n',
encoding='utf-8')
exe = os.path.join(tmp, 'segment.exe' if os.name == 'nt'
else 'segment')
src = os.path.join(tmp, 'segment.c')
Path(src).write_text(textwrap.dedent(f'''
#include "psx_runtime.h"
void ov_func_k(CPUState *cpu);
{dispatch}
static int calls;
int psx_overlay_static_code_matches(
const uint32_t *lo_len_pairs, uint32_t count,
uint32_t expected_crc)
{{
(void)lo_len_pairs;
(void)count;
return expected_crc == 0x11111111u;
}}
void ov_func_k(CPUState *cpu) {{ (void)cpu; calls++; }}
int main(void)
{{
CPUState cpu = {{0}};
uint64_t checks = 0, hits = 0, vm = 0, am = 0;
if (psx_overlay_static_can_dispatch(0x0000281Cu)) return 1;
if (psx_overlay_static_can_dispatch(0xA000281Cu)) return 2;
if (psx_overlay_dispatch(&cpu, 0x0000281Cu)) return 3;
if (psx_overlay_dispatch(&cpu, 0xA000281Cu)) return 4;
if (calls != 0) return 5;
psx_overlay_static_get_stats(&checks, &hits, &vm, &am);
if (checks != 0 || hits != 0 || am != 4) return 6;
if (!psx_overlay_static_can_dispatch(0x8000281Cu)) return 7;
if (!psx_overlay_dispatch(&cpu, 0x8000281Cu)) return 8;
if (calls != 1) return 9;
return 0;
}}
'''), encoding='utf-8')
subprocess.run([cc, src, '-I', tmp, '-std=c99', '-Wall', '-Werror',
'-o', exe], check=True)
subprocess.run([exe], check=True)
def test_non_kseg0_entry_is_rejected(self):
with self.assertRaises(ValueError):
compile_overlays.generate_overlay_dispatch(
[{'addr': 0x0000281C, 'symbol': 'ov_func_k', 'crc': 1,
'ranges': ((0x0000281C, 16),)}])
class StaticWorkerContractTests(unittest.TestCase):
def test_worker_is_picklable_for_a_process_pool(self):
# ProcessPoolExecutor pickles the callable by qualified name; a nested