runtime: restore native Tomba FMV overlays

recompiler/CMakeLists.txt

@@ -291,6 +291,10 @@ if(BUILD_TESTING)
             COMMAND ${Python3_EXECUTABLE}
                     ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_fmv_cycle_batch_codegen.py
                     --recompiler $<TARGET_FILE:psxrecomp-game>)
+        add_test(
+            NAME overlay_shim_compile_contract
+            COMMAND ${Python3_EXECUTABLE}
+                    ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_overlay_shim_compile.py)
         add_test(
             NAME dirty_text_continuation_guards
             COMMAND ${Python3_EXECUTABLE}

recompiler/tests/test_overlay_shim_compile.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+"""Compile the overlay callback shim against the public runtime headers."""
+
+import os
+import pathlib
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+ROOT = pathlib.Path(__file__).resolve().parents[2]
+INCLUDE = ROOT / "runtime" / "include"
+
+
+def find_gcc() -> str:
+    candidates = [
+        os.environ.get("CC"),
+        r"C:\msys64\mingw64\bin\gcc.exe" if os.name == "nt" else None,
+        shutil.which("gcc"),
+        shutil.which("cc"),
+    ]
+    for candidate in candidates:
+        if candidate and pathlib.Path(candidate).is_file():
+            return candidate
+    raise SystemExit("gcc/cc is required for the overlay shim compile test")
+
+
+def main() -> int:
+    source = r'''#include "cpu_state.h"
+#include "overlay_dispatch_preamble.c.inc"
+
+void func_80010000(CPUState *cpu) {
+    psx_cyc_bb_defer_begin();
+    psx_cyc_charge(1u);
+    psx_advance_cycles(1u);
+    cpu->gpr[2] = psx_cyc_load_word(cpu, cpu->gpr[4], 2u, 0u);
+    cpu->gpr[3] = psx_cyc_load_half(cpu, cpu->gpr[5], 3u, 0u);
+    psx_cyc_bb_defer_flush();
+    psx_cyc_bb_defer_end();
+    if (psx_slice_block(cpu, 0x80010000u, 1u, 0)) return;
+    debug_server_log_call_entry(0x80010000u);
+}
+'''
+    gcc = find_gcc()
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp = pathlib.Path(temp_dir)
+        c_path = temp / "overlay_shim_contract.c"
+        out_path = temp / ("overlay_shim_contract.dll" if os.name == "nt"
+                           else "overlay_shim_contract.so")
+        c_path.write_text(source, encoding="ascii")
+        command = [
+            gcc, "-shared", "-O2",
+            "-DPSX_OVERLAY_DLL_BUILD",
+            "-DPSX_NO_DEBUG_TOOLS",
+            "-DPSX_ENABLE_BLOCK_CYCLES=1",
+            "-DPSX_OVERLAY_FLAVOR=0",
+        ]
+        if os.name != "nt":
+            command.append("-fPIC")
+        command.extend([
+            str(c_path), "-o", str(out_path), f"-I{INCLUDE}", "-lm",
+        ])
+        result = subprocess.run(command, capture_output=True, text=True)
+        if result.returncode != 0:
+            sys.stderr.write(result.stdout)
+            sys.stderr.write(result.stderr)
+            return result.returncode
+        if not out_path.is_file():
+            raise AssertionError("overlay compiler reported success without output")
+    print("PASS: overlay callback shim compiles against runtime headers")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

runtime/codegen_hash_sources.cmake

@@ -40,8 +40,10 @@ set(PSXRECOMP_CODEGEN_HASH_SRCS
     # loading, and every NEW shard silently failed to compile against the moved-on
     # runtime (the recurring "header change broke the shards" class). Folding the
     # compile surface in makes such a change reshard SAME-TREE instead of breaking
-    # silently. overlay_api.h transitively pulls stdint; cpu_state.h is its only
-    # project include; the .inc is the shim every shard links against.
+    # silently. overlay_api.h pulls cpu_state.h and its cycle-helper headers; the
+    # .inc is the shim every shard links against.
     ${PSXRECOMP_CODEGEN_HASH_ROOT}/runtime/include/overlay_api.h
     ${PSXRECOMP_CODEGEN_HASH_ROOT}/runtime/include/cpu_state.h
+    ${PSXRECOMP_CODEGEN_HASH_ROOT}/runtime/include/psx_cyc.h
+    ${PSXRECOMP_CODEGEN_HASH_ROOT}/runtime/include/psx_cycles.h
     ${PSXRECOMP_CODEGEN_HASH_ROOT}/runtime/include/overlay_dispatch_preamble.c.inc)

runtime/include/cpu_state.h

@@ -168,10 +168,14 @@ extern void psx_dispatch_call(CPUState* cpu, uint32_t target_addr, uint32_t retu
 extern int g_psx_precise_slice;
 extern int psx_slice_block_impl(CPUState* cpu, uint32_t block_addr, uint32_t bcyc, int side_effects);
 void psx_precise_slice_init_from_env(void);
+#ifdef PSX_OVERLAY_DLL_BUILD
+int psx_slice_block(CPUState* cpu, uint32_t block_addr, uint32_t bcyc, int side_effects);
+#else
 static inline int psx_slice_block(CPUState* cpu, uint32_t block_addr, uint32_t bcyc, int side_effects) {
     if (!g_psx_precise_slice) return 0;
     return psx_slice_block_impl(cpu, block_addr, bcyc, side_effects);
 }
+#endif
 
 /* Unknown dispatch — defined in runtime/src/traps.c */
 extern void psx_unknown_dispatch(CPUState* cpu, uint32_t addr, uint32_t phys);
@@ -305,9 +309,13 @@ static inline int psx_call_contract(CPUState* cpu, uint32_t site_ra,
 extern "C" {
 #endif
 extern volatile uint32_t g_psx_last_fn_entry;
+#ifdef PSX_OVERLAY_DLL_BUILD
+void debug_server_log_call_entry(uint32_t func_addr);
+#else
 static inline void debug_server_log_call_entry(uint32_t func_addr) {
     g_psx_last_fn_entry = func_addr;
 }
+#endif
 #ifdef __cplusplus
 }
 #endif

runtime/include/psx_cyc.h

@@ -45,12 +45,19 @@ extern "C" {
  * state still updates per instruction, and guest totals at barriers match. */
 enum { PSX_CYC_BATCH_SOFT = 64u };
 
+#if defined(PSX_OVERLAY_DLL_BUILD)
+/* The overlay shim already batches cycle charges until a host callback edge. */
+static inline void psx_cyc_bb_defer_begin(void) { }
+static inline void psx_cyc_bb_defer_end(void) { }
+static inline void psx_cyc_bb_defer_flush(void) { overlay_flush_cycles(); }
+#else
 static inline void psx_cyc_bb_defer_begin(void) { g_psx_cyc_bb_defer++; }
 static inline void psx_cyc_bb_defer_end(void) {
     if (g_psx_cyc_bb_defer > 0) g_psx_cyc_bb_defer--;
     if (g_psx_cyc_bb_defer == 0) psx_cyc_batch_flush();
 }
 static inline void psx_cyc_bb_defer_flush(void) { psx_cyc_batch_flush(); }
+#endif
 
 #if defined(__GNUC__) || defined(__clang__)
 static inline void psx_cyc_bb_defer_cleanup(int *guard) {
@@ -61,6 +68,9 @@ static inline void psx_cyc_bb_defer_cleanup(int *guard) {
 
 static inline void psx_cyc_charge(uint32_t cycles) {
     if (cycles == 0u) return;
+#if defined(PSX_OVERLAY_DLL_BUILD)
+    psx_advance_cycles(cycles);
+#else
 #if defined(__GNUC__) || defined(__clang__)
     if (__builtin_expect(g_ls_replay_active | g_event_step_conservative, 0)) {
 #else
@@ -98,6 +108,7 @@ static inline void psx_cyc_charge(uint32_t cycles) {
     }
 #endif
     psx_advance_cycles(cycles); /* publishes any pending batch first */
+#endif
 }
 
 /* §1 base (Beetle cpu.cpp:795-798). */
@@ -169,6 +180,15 @@ int psx_load_delay_enabled(void);
 uint32_t psx_cyc_load_word_slow(CPUState* cpu, uint32_t addr, uint32_t rt, uint32_t reg_mask);
 uint16_t psx_cyc_load_half_slow(CPUState* cpu, uint32_t addr, uint32_t rt, uint32_t reg_mask);
 
+/* Overlay DLLs route timed loads through the host callback shim so they share
+ * runtime timing and memory state. Normal generated code keeps the main-RAM
+ * fast path inline. */
+#if defined(PSX_OVERLAY_DLL_BUILD)
+uint32_t psx_cyc_load_word(CPUState* cpu, uint32_t addr,
+                           uint32_t rt, uint32_t reg_mask);
+uint16_t psx_cyc_load_half(CPUState* cpu, uint32_t addr,
+                           uint32_t rt, uint32_t reg_mask);
+#else
 /* CPU data load value+timing. Full Beetle sequence for main RAM is inlined;
  * MMIO / lockstep / data-shard fall through to *_slow in memory.c. */
 static inline uint32_t psx_cyc_load_word(CPUState* cpu, uint32_t addr,
@@ -230,6 +250,7 @@ static inline uint16_t psx_cyc_load_half(CPUState* cpu, uint32_t addr,
     return psx_read_half(addr);
 #endif
 }
+#endif
 
 extern uint8_t psx_cyc_load_byte(CPUState* cpu, uint32_t addr, uint32_t rt, uint32_t reg_mask);
 

runtime/include/psx_cycles.h

@@ -60,7 +60,12 @@ extern uint32_t g_psx_cyc_batch_limit;
  * block. Interrupt/MMIO edges still publish the accumulated guest cycles. */
 extern int g_psx_cyc_bb_defer;
 
-/* Advance guest time. The common production path is inlined: bump the
+/* Advance guest time. Overlay DLLs forward this through their callback shim;
+ * normal runtime/generated code keeps the common production path inlined. */
+#if defined(PSX_OVERLAY_DLL_BUILD)
+void psx_advance_cycles(uint32_t cycles);
+#else
+/* The common production path is inlined: bump the
  * counter and only service devices when the next event deadline is due.
  * Guest-visible timing is unchanged (service_to_now replays exact events).
  *
@@ -110,8 +115,14 @@ static inline void psx_advance_cycles(uint32_t cycles) {
     }
 #endif
 }
+#endif
 
-/* Publish deferred charges (IRQ edge / MMIO / savestate). */
+/* Publish deferred charges (IRQ edge / MMIO / savestate). Overlay DLLs keep
+ * their pending total in the callback shim rather than these host globals. */
+#if defined(PSX_OVERLAY_DLL_BUILD)
+void overlay_flush_cycles(void);
+static inline void psx_cyc_batch_flush(void) { overlay_flush_cycles(); }
+#else
 static inline void psx_cyc_batch_flush(void) {
 #if !defined(PSX_COSIM)
     uint32_t b = g_psx_cyc_batch;
@@ -121,6 +132,7 @@ static inline void psx_cyc_batch_flush(void) {
     psx_advance_cycles(b);
 #endif
 }
+#endif
 
 /* Read accessor for telemetry (includes deferred batch). */
 uint64_t psx_get_cycle_count(void);

runtime/src/overlay_loader.c

@@ -1183,7 +1183,12 @@ static int cache_tier_from_path(const char *path) {
 }
 
 static int cache_name_is_immutable(const char *name) {
-    return name && strlen(name) == 26u + OVERLAY_SHARED_EXT_LEN &&
+    if (!name) return 0;
+    size_t len = strlen(name);
+    /* Current publishers commit a recoverable region/CRC pair. Retain support
+     * for additive artifact-suffixed pairs produced by earlier cache tools. */
+    if (len == 17u + OVERLAY_SHARED_EXT_LEN) return name[8] == '_';
+    return len == 26u + OVERLAY_SHARED_EXT_LEN &&
            name[8] == '_' && name[17] == '_';
 }
 

runtime/tests/overlay_pair_dedup_harness.c

@@ -1,4 +1,6 @@
+#define PSX_OVERLAY_DLL_BUILD 1
 #include "overlay_loader.h"
+#undef PSX_OVERLAY_DLL_BUILD
 
 #include <stdint.h>
 #include <stdio.h>
@@ -82,10 +84,18 @@ uint32_t psx_cyc_load_word(CPUState *cpu, uint32_t addr, uint32_t rt,
                            uint32_t mask) {
     (void)cpu; (void)addr; (void)rt; (void)mask; return 0;
 }
+uint32_t psx_cyc_load_word_slow(CPUState *cpu, uint32_t addr, uint32_t rt,
+                                uint32_t mask) {
+    return psx_cyc_load_word(cpu, addr, rt, mask);
+}
 uint16_t psx_cyc_load_half(CPUState *cpu, uint32_t addr, uint32_t rt,
                            uint32_t mask) {
     (void)cpu; (void)addr; (void)rt; (void)mask; return 0;
 }
+uint16_t psx_cyc_load_half_slow(CPUState *cpu, uint32_t addr, uint32_t rt,
+                                uint32_t mask) {
+    return psx_cyc_load_half(cpu, addr, rt, mask);
+}
 uint8_t psx_cyc_load_byte(CPUState *cpu, uint32_t addr, uint32_t rt,
                           uint32_t mask) {
     (void)cpu; (void)addr; (void)rt; (void)mask; return 0;
@@ -98,6 +108,9 @@ int psx_icache_shadow_replay_begin(void) { return 1; }
 void psx_icache_shadow_replay_end(void) {}
 void psx_icache_shadow_abort(void) {}
 void psx_icache_fetch(CPUState *cpu, uint32_t addr) { (void)cpu; (void)addr; }
+void psx_icache_fetch_fn(CPUState *cpu, uint32_t addr) {
+    psx_icache_fetch(cpu, addr);
+}
 void psx_muldiv_set(CPUState *cpu, uint32_t latency) {
     (void)cpu; (void)latency;
 }
@@ -110,6 +123,10 @@ int psx_slice_block(CPUState *cpu, uint32_t addr, uint32_t cycles,
                     int side_effects) {
     (void)cpu; (void)addr; (void)cycles; (void)side_effects; return 0;
 }
+int psx_slice_block_impl(CPUState *cpu, uint32_t addr, uint32_t cycles,
+                         int side_effects) {
+    return psx_slice_block(cpu, addr, cycles, side_effects);
+}
 
 void gte_execute(CPUState *cpu, uint32_t cmd) { (void)cpu; (void)cmd; }
 uint32_t gte_read_data(CPUState *cpu, uint8_t reg) {
@@ -124,6 +141,8 @@ void gte_write_data(CPUState *cpu, uint8_t reg, uint32_t val) {
 void gte_write_ctrl(CPUState *cpu, uint8_t reg, uint32_t val) {
     (void)cpu; (void)reg; (void)val;
 }
+int32_t psx_ws_plane_nx(int32_t nx) { return nx; }
+uint32_t psx_ws_xclip_bound(uint32_t vanilla) { return vanilla; }
 void gte_precision_store_word(uint32_t addr, uint8_t reg) {
     (void)addr; (void)reg;
 }

runtime/tests/test_overlay_pair_dedup_runtime.py

@@ -97,6 +97,7 @@ def compile_harness(gcc: str, out: pathlib) -> None:
     command = [
         gcc, "-std=c11", "-O0", "-Wall", "-Wextra",
         "-DPSX_NO_DEBUG_TOOLS",
+        "-DPSX_OVERLAY_DLL_BUILD",
         "-DPSX_OVERLAY_TEST_CANDIDATE_CAP=4", f"-I{RUNTIME / 'include'}",
         *platform_defines,
         str(RUNTIME / "src" / "overlay_loader.c"),
@@ -144,7 +145,8 @@ def scenario(tmp: pathlib.Path, harness: pathlib.Path, full_first: pathlib.Path,
     elif name == "provenance-mismatch":
         second_manifest = manifest(two, "hosted-v1")
 
-    first = publish(cache, first_tier, f"00010000_11111111_AAAAAAAA{ext}",
+    # Exercise the current transaction-bound name and the legacy additive name.
+    first = publish(cache, first_tier, f"00010000_11111111{ext}",
                     first_library, first_manifest)
     second = publish(cache, second_tier, f"00011000_22222222_BBBBBBBB{ext}",
                      full_second, second_manifest, pending=True)