mirror/psxrecomp
1
0
Fork 0
mirror of https://github.com/RetroPortingToolKit/psxrecomp synced 2026-10-25 18:05:12 -04:00
Code Issues Packages Projects Releases Wiki Activity

Harden additive overlay shards and speed interpreter fetch

Browse Source
This commit is contained in:
Matthew Stanley
2026-07-19 12:11:56 -07:00
parent 3949854981
commit 7376e6b042
23 changed files with 1684 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files
recompiler/CMakeLists.txt
+12 -3
View File
@@ -242,9 +242,18 @@ if(BUILD_TESTING)
CXX_STANDARD 17
CXX_STANDARD_REQUIRED ON
)
add_test(NAME recompiler_patch_cxx17_header_test
COMMAND recompiler_patch_cxx17_header_test)
endif()
add_test(NAME recompiler_patch_cxx17_header_test
COMMAND recompiler_patch_cxx17_header_test)
find_package(Python3 COMPONENTS Interpreter)
if(Python3_Interpreter_FOUND)
add_test(
NAME overlay_cross_page_delay_codegen
COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tests/test_overlay_cross_page_delay_codegen.py
--recompiler $<TARGET_FILE:psxrecomp-game>)
endif()
endif()
# Reachable main-EXE discovery test: parser-bound validation plus exact-entry
# analysis behavior for direct, indirect, seeded, and out-of-bound targets.
recompiler/src/code_generator.cpp
+44 -1
View File
@@ -11,6 +11,7 @@
#include <algorithm>
#include <cstdlib>
#include <set>
#include <stdexcept>
#include <vector>
namespace PSXRecomp {
@@ -1517,6 +1518,22 @@ std::string CodeGenerator::translate_basic_block(
std::string delay_saved_cond; // branch condition captured before delay
std::string delay_saved_target; // JR/JALR target captured before delay
const uint32_t branch_opcode =
(block.exit_instr.instruction >> 26) & 0x3Fu;
const uint32_t branch_rt =
(block.exit_instr.instruction >> 16) & 0x1Fu;
if (branch_opcode >= 0x14u && branch_opcode <= 0x17u) {
throw std::runtime_error(fmt::format(
"cannot emit reserved R3000A branch-likely opcode at 0x{:08X}",
addr));
}
if (branch_opcode == 0x01u &&
branch_rt != 0x00u && branch_rt != 0x01u &&
branch_rt != 0x10u && branch_rt != 0x11u) {
throw std::runtime_error(fmt::format(
"cannot emit unsupported REGIMM branch at 0x{:08X}", addr));
}
// Per-instruction interlock ORDER (Beetle): the branch's §1+deps+DO_LDS
// runs at the branch PC, THEN the delay slot's at PC+4. Emit the branch
// step FIRST (it is pure timing — does not touch GPR values, so it is
@@ -1566,7 +1583,22 @@ std::string CodeGenerator::translate_basic_block(
uint32_t delay_slot_addr = addr + 4;
auto delay_instr_opt = exe_.read_word(delay_slot_addr);
if (delay_instr_opt.has_value()) {
/* A native control transfer without its architectural delay
* slot is never a legal partial shard. In particular, a
* page-scoped overlay capture can end with the branch at
* ...FFC while the slot lives at ...000 in the next page.
* The old code silently skipped the unavailable word and
* still emitted the transfer, corrupting guest state while
* looking like a successful native call. Fail generation
* closed; the runtime interpreter remains authoritative
* until a capture containing the guard word is available. */
if (!delay_instr_opt.has_value()) {
throw std::runtime_error(fmt::format(
"cannot emit control flow at 0x{:08X}: mandatory "
"delay slot 0x{:08X} is outside the input image",
addr, delay_slot_addr));
}
{
uint32_t delay_instr = *delay_instr_opt;
// For branch-likely variants, delay slot is conditional
@@ -3052,6 +3084,17 @@ std::string CodeGenerator::generate_ranges_manifest(
(void)baddr;
uint32_t lo = blk.start_addr;
uint32_t hi = blk.end_addr + 4u;
/* translate_basic_block clones an out-of-block delay slot when the
* control-flow instruction itself is the block's last word. That
* cloned instruction is compiled code too: include it in the exact
* validity range so its bytes participate in the candidate CRC and
* page-generation watch. Otherwise a later write to the slot could
* leave stale native semantics callable. */
if (blk.exit_instr.has_delay_slot &&
blk.exit_instr.type != ControlFlowType::None &&
blk.exit_instr.address == blk.end_addr && hi <= UINT32_MAX - 4u) {
hi += 4u;
}
if (hi > lo) iv.emplace_back(lo, hi);
}
if (iv.empty()) continue;
recompiler/tests/test_overlay_cross_page_delay_codegen.py
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env python3
"""Regression for an overlay branch whose delay slot crosses a 4 KiB page.
The complete input must emit and hash the delay slot. A page-truncated input
must fail code generation instead of publishing a native transfer that skips it.
"""
import argparse
import os
import struct
import subprocess
import sys
import tempfile
from typing import Optional
LOAD = 0x80010FF0
BRANCH_PC = 0x80010FFC
DELAY_PC = 0x80011000
def make_psxexe(data: bytes) -> bytes:
header = bytearray(2048)
header[0:8] = b"PS-X EXE"
struct.pack_into("<I", header, 0x10, LOAD)
struct.pack_into("<I", header, 0x18, LOAD)
struct.pack_into("<I", header, 0x1C, len(data))
return bytes(header) + data
def run_codegen(recompiler: str, include_delay: bool, root: str,
branch_word: int = 0x10800008,
case_name: Optional[str] = None):
case = case_name or ("complete" if include_delay else "truncated")
case_dir = os.path.join(root, case)
out_dir = os.path.join(case_dir, "out")
os.makedirs(out_dir)
# FF0..FF8: nops; FFC: beq $a0,$zero,0x80011020; 1000: addiu $v0,$zero,1.
words = [0, 0, 0, branch_word]
if include_delay:
words.append(0x24020001)
data = b"".join(struct.pack("<I", word) for word in words)
psx = os.path.join(case_dir, "cross_page.psx")
seeds = os.path.join(case_dir, "seeds.txt")
with open(psx, "wb") as f:
f.write(make_psxexe(data))
with open(seeds, "w", encoding="ascii") as f:
f.write(f"dispatch_root 0x{LOAD:08X}\n")
proc = subprocess.run(
[recompiler, psx, "--seeds", seeds, "--out-dir", out_dir, "--overlay"],
capture_output=True,
text=True,
)
return proc, out_dir
def main() -> int:
here = os.path.dirname(os.path.abspath(__file__))
default_recompiler = os.path.normpath(
os.path.join(here, "..", "build", "psxrecomp-game.exe")
)
parser = argparse.ArgumentParser()
parser.add_argument("--recompiler", default=default_recompiler)
args = parser.parse_args()
if not os.path.isfile(args.recompiler):
raise SystemExit(f"recompiler not found: {args.recompiler}")
failures = []
with tempfile.TemporaryDirectory() as root:
complete, complete_out = run_codegen(args.recompiler, True, root)
if complete.returncode != 0:
failures.append("complete cross-page function failed generation: " +
(complete.stderr or complete.stdout))
else:
full_c = next((name for name in os.listdir(complete_out)
if name.endswith("_full.c")), None)
ranges = next((name for name in os.listdir(complete_out)
if name.endswith("_full.ranges")), None)
if not full_c or not ranges:
failures.append("complete generation omitted C or ranges output")
else:
with open(os.path.join(complete_out, full_c), encoding="utf-8") as f:
source = f.read()
with open(os.path.join(complete_out, ranges), encoding="utf-8") as f:
manifest = f.read()
if "0x80011000" not in source or "0x24020001" not in source:
failures.append("generated C does not execute the cross-page delay slot")
covered = False
for line in manifest.splitlines():
fields = line.split()
if len(fields) == 3 and fields[0] == "R":
lo = int(fields[1], 16)
length = int(fields[2], 16)
if lo <= DELAY_PC and DELAY_PC + 4 <= lo + length:
covered = True
if not covered:
failures.append("range manifest does not hash the cross-page delay slot")
truncated, _ = run_codegen(args.recompiler, False, root)
combined = (truncated.stderr or "") + (truncated.stdout or "")
if truncated.returncode == 0:
failures.append("truncated input incorrectly generated a native shard")
elif "mandatory delay slot" not in combined:
failures.append("truncated generation failed without the delay-slot diagnostic")
likely, _ = run_codegen(
args.recompiler, True, root, branch_word=0x50800008,
case_name="reserved_branch_likely")
likely_output = (likely.stderr or "") + (likely.stdout or "")
if likely.returncode == 0:
failures.append("reserved R3000A branch-likely opcode generated native code")
elif "branch-likely" not in likely_output:
failures.append("branch-likely rejection omitted its diagnostic")
if failures:
for failure in failures:
print("FAIL:", failure)
return 1
print("PASS: cross-page delay slot is emitted, identity-hashed, and truncation fails closed")
return 0
if __name__ == "__main__":
sys.exit(main())
recompiler/tests/test_overlay_guard_codegen.py
+11 -4
View File
@@ -69,11 +69,18 @@ def gen_c(recompiler, overlay, tmp):
if r.returncode != 0:
raise SystemExit(f"recompiler failed ({'overlay' if overlay else 'static'}):\n"
f"{r.stderr or r.stdout}")
full = [f for f in os.listdir(out) if f.endswith("_full.c")]
# Overlay mode emits one *_full.c; static mode may shard it as
# *_full_00.c, *_full_01.c, ... . Inspect the complete generated body in
# either representation.
full = sorted(f for f in os.listdir(out)
if "_full" in f and f.endswith(".c"))
if not full:
raise SystemExit(f"no _full.c emitted in {out}")
with open(os.path.join(out, full[0])) as f:
return f.read()
raise SystemExit(f"no _full*.c emitted in {out}")
chunks = []
for name in full:
with open(os.path.join(out, name)) as f:
chunks.append(f.read())
return "\n".join(chunks)
def main():
runtime/CMakeLists.txt
+12
View File
@@ -1,5 +1,6 @@
cmake_minimum_required(VERSION 3.20)
project(psxrecomp-runtime C CXX)
include(CTest)
set(CMAKE_C_STANDARD 99)
set(CMAKE_CXX_STANDARD 17)
@@ -31,6 +32,17 @@ psxrecomp_add_runtime_target(psx-runtime
DEFAULT_BIOS_PATH "${PSXRECOMP_ROOT}/bios/SCPH1001.BIN"
)
if(BUILD_TESTING)
add_executable(psx_icache_fastpath_test
tests/test_psx_icache_fastpath.c
src/psx_icache.c
)
target_include_directories(psx_icache_fastpath_test PRIVATE include)
target_compile_definitions(psx_icache_fastpath_test PRIVATE
PSX_ENABLE_BLOCK_CYCLES=1)
add_test(NAME psx_icache_fastpath_test COMMAND psx_icache_fastpath_test)
endif()
if(NOT MSVC)
psxrecomp_add_runtime_target(psx-oracle
ORACLE
runtime/include/overlay_loader.h
+6
View File
@@ -90,6 +90,12 @@ void overlay_loader_get_counters(uint32_t *loads, uint32_t *invalidations,
int *regions, uint32_t *revalidations);
void overlay_loader_get_load_timing(uint64_t *total_us, uint64_t *max_us,
uint64_t *last_us);
/* Opt-in PSX_RUNTIME_PERF_DIAG sampler: returns and clears the hottest native
* owner since the preceding call. Disabled runs pay no table update cost. */
void overlay_loader_take_hot_native(uint32_t *pc, uint64_t *calls);
/* Exact shadow-differential summary for opt-in perf diagnostics. */
void overlay_loader_get_shadow_summary(uint64_t *calls, uint64_t *divergences,
uint32_t *first_divergence_pc);
void overlay_loader_get_reload_debug(int *r0_valid, uint32_t *r0_writes,
uint32_t *r0_fn_lo, uint32_t *r0_fn_hi,
runtime/include/overlay_posix.h
+3 -2
View File
@@ -7,11 +7,12 @@
extern "C" {
#endif
/* Parsed <addr8>_<crc8>.{dll,so} cache filename. The parser is shared by the
/* Parsed legacy <addr8>_<logical8> or immutable
* <addr8>_<logical8>_<artifact8>.{dll,so} cache filename. The parser is shared by the
* Windows and POSIX directory walkers so both platforms reject partial hex
* fields and lookalike suffixes identically. */
typedef struct PsxOverlayCacheFile {
char name[22];
char name[40];
char path[768];
uint32_t region_start;
uint32_t content_crc;
runtime/include/psx_icache.h
+38
View File
@@ -0,0 +1,38 @@
#ifndef PSXRECOMP_PSX_ICACHE_H
#define PSXRECOMP_PSX_ICACHE_H
#include "cpu_state.h"
#ifdef __cplusplus
extern "C" {
#endif
extern uint32_t g_psx_icache_tv[1024];
extern int g_psx_icache_active;
extern int g_ls_replay_active;
void psx_icache_fetch_miss(CPUState *cpu, uint32_t addr);
/* Keep the interpreter's steady-state tag hit inside its translation unit.
* Misses use the shared slow path, preserving exact cache evolution/timing. */
static inline void psx_icache_fetch_interp(CPUState *cpu, uint32_t addr) {
#ifdef PSX_ENABLE_BLOCK_CYCLES
if (g_ls_replay_active) return;
if (g_psx_icache_active < 0) {
psx_icache_fetch(cpu, addr);
return;
}
if (!g_psx_icache_active) return;
uint32_t idx = (addr & 0xFFCu) >> 2;
if (g_psx_icache_tv[idx] == addr) return;
psx_icache_fetch_miss(cpu, addr);
#else
(void)cpu;
(void)addr;
#endif
}
#ifdef __cplusplus
}
#endif
#endif
runtime/src/autocompile.c
+49 -22
View File
@@ -71,16 +71,17 @@ static HANDLE s_proc = NULL;
#define AC_PUBLISH_CAP 128
static char s_publish_paths[AC_PUBLISH_CAP][768];
static unsigned s_publish_head = 0, s_publish_count = 0;
static unsigned s_publish_seen_run = 0, s_publish_drops_run = 0;
static unsigned s_publish_drops_run = 0;
static unsigned s_publish_load_fail_run = 0, s_publish_parse_fail_run = 0;
static char s_child_line[1024];
static int s_child_line_len = 0;
static int s_child_line_overflow = 0;
static void publish_line_locked(void) {
static const char marker[] = "PSX_SHARD_PUBLISHED ";
if (strncmp(s_child_line, marker, sizeof(marker) - 1) != 0) return;
const char *path = s_child_line + sizeof(marker) - 1;
if (!path[0]) return;
s_publish_seen_run++;
if (s_publish_count >= AC_PUBLISH_CAP) {
s_publish_drops_run++;
return;
@@ -96,10 +97,16 @@ static void publish_parse_locked(const char *buf, int n) {
if (c == '\r') continue;
if (c == '\n') {
s_child_line[s_child_line_len] = '\0';
publish_line_locked();
if (s_child_line_overflow)
s_publish_parse_fail_run++;
else
publish_line_locked();
s_child_line_len = 0;
s_child_line_overflow = 0;
} else if (s_child_line_len < (int)sizeof(s_child_line) - 1) {
s_child_line[s_child_line_len++] = c;
} else {
s_child_line_overflow = 1;
}
}
}
@@ -155,6 +162,16 @@ static DWORD WINAPI watch_thread(LPVOID arg) {
WaitForSingleObject(ctx->proc, INFINITE);
DWORD code = (DWORD)-1;
GetExitCodeProcess(ctx->proc, &code);
EnterCriticalSection(&s_out_lock);
/* A marker/result line cut off by child termination is not trustworthy.
* Completion will force the idempotent directory rescan instead. */
if (s_child_line_len || s_child_line_overflow) {
s_publish_parse_fail_run++;
s_child_line_len = 0;
s_child_line_overflow = 0;
}
if (s_proc == ctx->proc) s_proc = NULL;
LeaveCriticalSection(&s_out_lock);
CloseHandle(ctx->proc);
HeapFree(GetProcessHeap(), 0, ctx);
InterlockedExchange(&s_exit_code, (LONG)code);
@@ -227,8 +244,14 @@ int autocompile_request(void) {
#ifdef _WIN32
EnterCriticalSection(&s_out_lock);
s_publish_head = s_publish_count = 0;
s_publish_seen_run = s_publish_drops_run = 0;
s_publish_drops_run = 0;
s_publish_load_fail_run = s_publish_parse_fail_run = 0;
s_child_line_len = 0;
s_child_line_overflow = 0;
s_out_len = 0;
s_exit_code = -1;
s_shard_ok = s_shard_fail = s_shard_skipped = 0;
s_shard_result_seen = 0;
LeaveCriticalSection(&s_out_lock);
SECURITY_ATTRIBUTES sa = { sizeof(sa), NULL, TRUE };
HANDLE rd = NULL, wr = NULL;
@@ -273,6 +296,10 @@ int autocompile_request(void) {
WatchCtx *ctx = (WatchCtx *)HeapAlloc(GetProcessHeap(), 0, sizeof(*ctx));
if (!ctx) {
/* The child is already running. Leaving it detached would permit the
* state to remain IDLE and a second writer to enter the same cache. */
TerminateProcess(pi.hProcess, ERROR_NOT_ENOUGH_MEMORY);
WaitForSingleObject(pi.hProcess, INFINITE);
CloseHandle(rd);
CloseHandle(pi.hProcess);
s_fails++;
@@ -290,6 +317,7 @@ int autocompile_request(void) {
/* Without the watcher nobody drains stdout, observes completion, or
* closes the process handle. Cancel the just-created child fail-closed. */
TerminateProcess(pi.hProcess, ERROR_NOT_ENOUGH_MEMORY);
WaitForSingleObject(pi.hProcess, INFINITE);
CloseHandle(rd);
CloseHandle(pi.hProcess);
HeapFree(GetProcessHeap(), 0, ctx);
@@ -329,17 +357,19 @@ static int parse_shard_result(void) {
}
if (!hit) return 0;
unsigned ok = 0, failed = 0, skipped = 0;
/* Tolerant of field order: scan for each key independently within the line. */
const char *ln_end = strchr(hit, '\n');
size_t span = ln_end ? (size_t)(ln_end - hit) : strlen(hit);
char line[256];
size_t cp = span < sizeof(line) - 1 ? span : sizeof(line) - 1;
memcpy(line, hit, cp);
line[cp] = '\0';
const char *f;
if ((f = strstr(line, "ok="))) sscanf(f, "ok=%u", &ok);
if ((f = strstr(line, "failed="))) sscanf(f, "failed=%u", &failed);
if ((f = strstr(line, "skipped="))) sscanf(f, "skipped=%u", &skipped);
int consumed = 0;
if (sscanf(line, "PSX_SHARD_RESULT ok=%u failed=%u skipped=%u %n",
&ok, &failed, &skipped, &consumed) != 3)
return 0;
for (const char *tail = line + consumed; *tail; tail++) {
if (*tail != ' ' && *tail != '\t' && *tail != '\r') return 0;
}
s_shard_ok = ok;
s_shard_fail = failed;
s_shard_skipped = skipped;
@@ -356,7 +386,8 @@ void autocompile_poll_main(void) {
#ifdef _WIN32
char published[768];
if (publish_pop(published, (int)sizeof(published))) {
(void)overlay_loader_load_published(published);
if (overlay_loader_load_published(published) <= 0)
s_publish_load_fail_run++;
s_rescans++;
}
if (ac_state_load() == AC_RUNNING) return;
@@ -366,23 +397,19 @@ void autocompile_poll_main(void) {
if (publish_pending()) return;
#endif
ac_state_store(AC_IDLE);
/* New producers report each DLL directly; a full rescan is only the
* compatibility/overflow fallback. The loader is
* idempotent — rescanning after a failed compile is harmless. */
#ifdef _WIN32
if (s_publish_seen_run == 0 || s_publish_drops_run != 0) {
overlay_loader_rescan();
s_rescans++;
}
#else
s_shard_result_seen = parse_shard_result();
/* Direct markers minimize handoff latency, then one idempotent batch-end
* rescan makes every successfully published artifact visible to additive
* cache queries and the lazy manifest index. This is intentionally once per
* compiler run, never once per DLL or interpreted instruction. */
overlay_loader_rescan();
s_rescans++;
#endif
s_shard_result_seen = parse_shard_result();
/* A run "failed" if the process exited non-zero OR it reported shard
* failures. compile_overlays.py exits 2 when any shard that should have
* built failed, so these usually agree; the parsed count is the detail. */
if (s_exit_code != 0 || (s_shard_result_seen && s_shard_fail > 0))
if (s_exit_code != 0 || !s_shard_result_seen || s_shard_fail > 0 ||
s_publish_drops_run != 0 || s_publish_load_fail_run != 0 ||
s_publish_parse_fail_run != 0)
s_fails++;
}
runtime/src/dirty_ram_interp.c
+2 -1
View File
@@ -28,6 +28,7 @@
#include "debug_server.h"
#include "interrupts.h"
#include "psx_cycles.h"
#include "psx_icache.h"
#include "psx_instr_cost.h" /* psx_instr_base_cycles — single-source cycle cost */
#include "gpu.h" /* psx_ws_is_backdrop_site / psx_ws_backdrop_x (interp hook) */
#include "ws_backdrop_detect.h" /* shared backdrop-window detector (auto_backdrop) */
@@ -1188,7 +1189,7 @@ static int exec_one_fetched(CPUState *cpu, uint32_t pc, uint32_t insn,
/* Instruction FETCH cost (I-cache) — charged FIRST, before the §1 base, exactly
* like Beetle ReadInstruction precedes the per-instruction base (cpu.cpp). HIT=+0,
* KSEG1=+4, cached miss=+3+refill; a miss also clears the load give-back. */
psx_icache_fetch(cpu, pc);
psx_icache_fetch_interp(cpu, pc);
/* Per-instruction R3000A load-delay interlock (single-source: psx_cyc.h, shared
* with both static emitters). §1 base + GPR_DEPRES + DO_LDS run HERE, before the
runtime/src/gte.cpp
+7 -1
View File
@@ -1429,8 +1429,10 @@ extern "C" uint64_t gte_get_exec_count(void) { return s_gte_exec_count; }
extern "C" void gte_execute(CPUState* cpu, uint32_t cmd) {
using namespace PSXRecomp::GTE;
#ifndef PSX_NO_DEBUG_TOOLS
s_gte_exec_count++;
s_gte_caller_ra = cpu->gpr[31]; /* dome-locate probe: game fn that issued this projection */
#endif
GTEState gte;
// Skip reg 15 (SXYP: push-write, would corrupt SXY FIFO) and
@@ -1447,8 +1449,9 @@ extern "C" void gte_execute(CPUState* cpu, uint32_t cmd) {
* (no-op unless the game opts in — early-out on the config flag). */
if (func == 0x01 || func == 0x30)
psx_ws_note_gte_project(func == 0x30 ? 3 : 1);
#ifndef PSX_NO_DEBUG_TOOLS
/* INTPL ring: snapshot inputs before the op mutates IR (outputs recorded
* after the switch). */
* after the switch). Debug tooling only; production does no probe copies. */
int16_t intpl_pre_ir[4] = {0,0,0,0};
int32_t intpl_pre_fc[3] = {0,0,0};
if (func == 0x11) {
@@ -1456,6 +1459,7 @@ extern "C" void gte_execute(CPUState* cpu, uint32_t cmd) {
intpl_pre_ir[2]=gte.IR2; intpl_pre_ir[3]=gte.IR3;
intpl_pre_fc[0]=gte.FC[0]; intpl_pre_fc[1]=gte.FC[1]; intpl_pre_fc[2]=gte.FC[2];
}
#endif
switch (func) {
case 0x01: gte_rtps(&gte, cmd); break;
case 0x06: gte_nclip(&gte, cmd); break;
@@ -1484,8 +1488,10 @@ extern "C" void gte_execute(CPUState* cpu, uint32_t cmd) {
break;
}
#ifndef PSX_NO_DEBUG_TOOLS
if (func == 0x01 || func == 0x30) gte_rtp_record(&gte, cmd);
if (func == 0x11) gte_intpl_record(&gte, intpl_pre_ir, intpl_pre_fc);
#endif
for (int i = 0; i < 32; i++) cpu->gte_data[i] = gte_mfc2(&gte, i);
for (int i = 0; i < 32; i++) cpu->gte_ctrl[i] = gte_cfc2(&gte, i);
runtime/src/main.cpp
+19 -1
View File
@@ -1177,6 +1177,11 @@ struct RuntimePerfSnapshot {
uint64_t overlay_interp = 0;
uint64_t overlay_stale = 0;
uint32_t overlay_revalidations = 0;
uint32_t overlay_hot_native_pc = 0;
uint64_t overlay_hot_native_calls = 0;
uint64_t overlay_shadow_calls = 0;
uint64_t overlay_shadow_divergences = 0;
uint32_t overlay_first_divergence_pc = 0;
uint32_t capture_triggers = 0;
uint64_t capture_last_dispatch_delta = 0;
int capture_overlays = 0;
@@ -1275,6 +1280,11 @@ static RuntimePerfSnapshot runtime_perf_snapshot(uint64_t now) {
&s.overlay_interp, &s.overlay_stale,
nullptr, nullptr, nullptr, nullptr,
&s.overlay_revalidations);
overlay_loader_take_hot_native(&s.overlay_hot_native_pc,
&s.overlay_hot_native_calls);
overlay_loader_get_shadow_summary(&s.overlay_shadow_calls,
&s.overlay_shadow_divergences,
&s.overlay_first_divergence_pc);
int capture_enabled = 0;
overlay_autocapture_get_status(&capture_enabled, &s.capture_triggers,
&s.capture_last_dispatch_delta);
@@ -1410,7 +1420,9 @@ static void runtime_perf_diag_tick() {
"cpu=%.1f tex=%.1f draw=%.1f ms/s; "
"work guest=%.1f pacer=%.1f autocapture=%.1f provider_poll=%.1f ms/s, "
"dirty=%.0f insn/s %.0f dispatch/s; "
"overlay native=+%llu interp=+%llu loads=+%u revalidations=+%u "
"overlay native=+%llu interp=+%llu hot_native=0x%08X/+%llu "
"shadow=+%llu div=+%llu first_div=0x%08X "
"loads=+%u revalidations=+%u "
"load_wall=%.1f ms max=%.1f last=%.1f ms; "
"capture triggers=+%u overlays=+%d last_dispatch_delta=%llu\n",
(double)(current.frame - last.frame) / dt,
@@ -1434,6 +1446,12 @@ static void runtime_perf_diag_tick() {
(double)(current.dirty_dispatches - last.dirty_dispatches) / dt,
(unsigned long long)(current.overlay_native - last.overlay_native),
(unsigned long long)(current.overlay_interp - last.overlay_interp),
current.overlay_hot_native_pc,
(unsigned long long)current.overlay_hot_native_calls,
(unsigned long long)(current.overlay_shadow_calls - last.overlay_shadow_calls),
(unsigned long long)(current.overlay_shadow_divergences -
last.overlay_shadow_divergences),
current.overlay_first_divergence_pc,
current.overlay_loads - last.overlay_loads,
current.overlay_revalidations - last.overlay_revalidations,
(double)(overlay_load_us - last_overlay_load_us) / 1000.0,
runtime/src/overlay_capture.c
+38
View File
@@ -206,6 +206,19 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
} else if (!dirty && in_run) {
uint32_t phys = run_start * page_sz;
uint32_t size = (page - run_start) * page_sz;
/* Carry one coherent guard instruction beyond a dirty-page run.
* A MIPS branch/jump at the final word of the run (...FFC) always
* executes its delay slot in the next page (...000). Supplying
* that word lets overlay codegen emit exact semantics and lets its
* range manifest hash/watch the slot. Keep the established region
* start. A four-byte overlap across an artificial kernel/boot or
* boot/overlay capture-window boundary is required: those windows
* stabilize region keys, but they are not MIPS execution barriers.
* Adjacent dirty pages were already folded into this run. */
const uint32_t ram_size = 2u * 1024u * 1024u;
if (phys <= ram_size && size <= ram_size - phys &&
phys + size <= ram_size - 4u)
size += 4u;
uint32_t virt = 0x80000000u | (phys & 0x1FFFFFu);
in_run = 0;
@@ -717,6 +730,31 @@ static AutocapWriteJob *capture_snapshot_create(uint32_t scope_lo,
if (observed)
job->bitmap[page >> 5] |= 1u << (page & 31u);
}
/* Preserve a one-page EXECUTED halo on both sides of a scoped DMA
* overwrite. The overwritten page can contain either the ...FFC
* control transfer or the ...000 delay slot; the paired instruction
* may live in the neighboring page and must come from this same
* coherent pre-write RAM snapshot. Only executed neighbors qualify,
* so a sector overwrite cannot pull unrelated dirty history into an
* unbounded capture. Evidence clearing remains limited to the pages
* actually overwritten by DMA. */
uint32_t halo_lo = first_page > 0u ? first_page - 1u : first_page;
uint32_t ram_pages = (uint32_t)(ram_size >> 12);
uint32_t halo_hi = last_page + 1u < ram_pages
? last_page + 1u : last_page;
for (uint32_t page = halo_lo; page <= halo_hi; page++) {
if (page >= first_page && page <= last_page) continue;
uint32_t first_exec_word = page * (4096u / 4u);
int observed = 0;
for (uint32_t b = 0; b < 4096u / 4u / 32u; b++) {
if (job->exec_pc_bitmap[(first_exec_word >> 5) + b]) {
observed = 1;
break;
}
}
if (observed)
job->bitmap[page >> 5] |= 1u << (page & 31u);
}
} else {
for (uint32_t i = 0; i < bw; i++)
job->bitmap[i] = dirty_ram_get_bitmap_word(i);
runtime/src/overlay_loader.c
+482 -101
View File
@@ -13,6 +13,7 @@
#include <string.h>
#include <stdio.h>
#include <stdarg.h>
#include <ctype.h>
extern void overlay_watch_set_range(uint32_t phys, uint32_t len);
@@ -74,11 +75,15 @@ typedef struct {
uint32_t val_gen; /* pagegen sum when last (in)validated*/
int state; /* ENTRY_VALID/INVALID/BLACKLIST */
int dll; /* source DLL index */
uint8_t tier; /* gcc=2, tcc=1, unknown=0 */
int next; /* next candidate at same addr, -1 end*/
uint32_t diff_passes; /* clean same-state diffs (verify budget)*/
int device_touch; /* 1 = touches MMIO; never run its shard,
* always interp (shadow diff can't safely
* double-execute device I/O). */
#ifndef PSX_NO_DEBUG_TOOLS
uint32_t native_rank; /* first-seen identity rank for bisection */
#endif
} Candidate;
/* Same-state differential verify budget: diff a candidate this many times with
@@ -111,8 +116,9 @@ static int s_range_index_overflow = 0;
* hundreds of historical, currently-invalid owners in a reused region. */
#define RANGE_PC_CACHE_CAP 16384u
#define RANGE_PC_CACHE_MASK (RANGE_PC_CACHE_CAP - 1u)
typedef struct { uint32_t phys; int cand; } RangePcCache;
typedef struct { uint32_t phys, generation; int cand; } RangePcCache;
static RangePcCache s_range_pc_cache[RANGE_PC_CACHE_CAP];
static uint32_t s_range_candidate_generation = 1u;
static void range_index_add_candidate(int ci) {
Candidate *c = &s_cand[ci];
@@ -364,6 +370,34 @@ static uint64_t s_load_last_us = 0;
static int s_valid_count = 0; /* candidates currently VALID */
static uint64_t s_disp_native = 0;
static uint64_t s_disp_interp = 0;
/* Small diagnostic-only native-owner sampler. A logical softlock can continue
* presenting at 60 Hz while executing a tiny bad native loop, so FPS alone is
* not correctness evidence. This is enabled only with the existing runtime
* perf environment switch and is drained once per telemetry interval. */
#define NATIVE_HOT_CAP 256u
typedef struct { uint32_t pc; uint64_t calls; } NativeHot;
static NativeHot s_native_hot[NATIVE_HOT_CAP];
static int s_native_hot_enabled = 0;
#ifndef PSX_NO_DEBUG_TOOLS
/* Correctness localization without speculative/double guest execution. When
* PSX_NATIVE_RANK_LIMIT is present, each concrete loaded candidate is assigned
* a first-seen rank. Candidates through the limit execute normally; later ones
* fall back to the authoritative interpreter. Binary-searching the limit finds
* the first candidate identity required for a deterministic softlock while
* preserving exactly one guest execution path per run. Debug builds only. */
static uint32_t s_native_rank_limit = UINT32_MAX;
static uint32_t s_native_rank_next = 0;
static uint64_t s_native_rank_blocked = 0;
#endif
static void native_hot_note(uint32_t pc) {
if (!s_native_hot_enabled) return;
uint32_t slot = ((pc >> 2) * 2654435761u) & (NATIVE_HOT_CAP - 1u);
NativeHot *h = &s_native_hot[slot];
if (h->pc != pc) { h->pc = pc; h->calls = 0; }
h->calls++;
}
static uint64_t s_stale_blocked = 0; /* dispatches skipped (candidate !valid)*/
static uint32_t s_invalidations = 0; /* VALID -> INVALID transitions */
static uint32_t s_revalidations = 0; /* INVALID -> VALID (reload-on-return) */
@@ -516,8 +550,8 @@ void overlay_loader_static_match_stats(uint64_t *rehashes,
#endif
/* ---- Per-DLL code-range manifest --------------------------------------- */
/* Minimal line format emitted by tools/compile_overlays.py beside each DLL:
* F <entry_hex> start a function (entry = virtual export addr)
/* Strict v2 line format emitted by tools/compile_overlays.py beside each DLL:
* F <entry_hex> <crc_hex> start a function (entry = virtual export addr)
* R <lo_hex> <len_hex> a code byte-range (virtual addr, byte length)
* The recompiler's per-function instruction walk yields exactly the executed
* code bytes — interleaved jump tables / constant pools are excluded, which is
@@ -531,6 +565,32 @@ typedef struct {
int n;
} ManFn;
static int man_structurally_valid(const ManFn *m) {
const uint32_t ram_size = 2u * 1024u * 1024u;
if (!m || !m->has_crc || m->n < 1 || m->n > MAX_CODE_RANGES)
return 0;
if ((m->entry & 0xFFE00000u) != 0x80000000u) return 0;
uint32_t entry = m->entry & 0x1FFFFFFFu;
if ((entry & 3u) != 0u || entry >= ram_size) return 0;
int entry_covered = 0;
for (int r = 0; r < m->n; r++) {
if ((m->lo[r] & 0xFFE00000u) != 0x80000000u) return 0;
uint32_t lo = m->lo[r] & 0x1FFFFFFFu;
uint32_t len = m->len[r];
if ((lo & 3u) != 0u || (len & 3u) != 0u || len < 4u ||
lo >= ram_size || len > ram_size - lo)
return 0;
if (entry >= lo && entry - lo <= len - 4u) entry_covered = 1;
for (int p = 0; p < r; p++) {
uint32_t prev_lo = m->lo[p] & 0x1FFFFFFFu;
uint32_t prev_hi = prev_lo + m->len[p];
uint32_t hi = lo + len;
if (lo < prev_hi && prev_lo < hi) return 0;
}
}
return entry_covered;
}
static ManFn *parse_manifest(const char *path, int *out_n) {
*out_n = 0;
FILE *f = fopen(path, "r");
@@ -540,34 +600,50 @@ static ManFn *parse_manifest(const char *path, int *out_n) {
if (!arr) { fclose(f); return NULL; }
char line[160];
ManFn *cur = NULL;
int invalid = 0;
while (fgets(line, sizeof(line), f)) {
if (line[0] == 'F') {
if (!strchr(line, '\n') && !feof(f)) { invalid = 1; break; }
char *p = line;
while (*p && isspace((unsigned char)*p)) p++;
if (!*p || *p == '#') continue;
if (*p == 'F') {
if (cur && !man_structurally_valid(cur)) { invalid = 1; break; }
uint32_t e = 0, crc = 0;
int got = sscanf(line + 1, "%x %x", &e, &crc);
if (got >= 1) {
char extra = 0;
int got = sscanf(p + 1, " %x %x %c", &e, &crc, &extra);
if (got == 2) {
if (n >= CAND_CAP) { invalid = 1; break; }
for (int i = 0; i < n; i++) {
if ((arr[i].entry & 0x1FFFFFFFu) ==
(e & 0x1FFFFFFFu)) { invalid = 1; break; }
}
if (invalid) break;
if (n >= cap) {
cap *= 2;
ManFn *na = (ManFn *)realloc(arr, sizeof(ManFn) * cap);
if (!na) break;
if (!na) { invalid = 1; break; }
arr = na;
}
cur = &arr[n++];
cur->entry = e;
cur->crc = crc;
cur->has_crc = (got >= 2) ? 1 : 0;
cur->has_crc = 1;
cur->n = 0;
}
} else if (line[0] == 'R' && cur) {
} else { invalid = 1; break; }
} else if (*p == 'R' && cur) {
uint32_t lo, len;
if (sscanf(line + 1, "%x %x", &lo, &len) == 2 &&
cur->n < MAX_CODE_RANGES && len > 0) {
char extra = 0;
if (sscanf(p + 1, " %x %x %c", &lo, &len, &extra) == 2 &&
cur->n < MAX_CODE_RANGES) {
cur->lo[cur->n] = lo;
cur->len[cur->n] = len;
cur->n++;
}
}
} else { invalid = 1; break; }
} else { invalid = 1; break; }
}
if (!invalid && (n == 0 || !cur || !man_structurally_valid(cur))) invalid = 1;
fclose(f);
if (invalid) { free(arr); return NULL; }
*out_n = n;
return arr;
}
@@ -578,16 +654,97 @@ static ManFn *man_find(ManFn *arr, int n, uint32_t entry) {
return NULL;
}
/* 1 = supported R3000A control transfer with a delay slot, 0 = ordinary
* instruction, -1 = reserved/unsupported branch encoding that native shards
* must never claim. */
static int mips_control_kind(uint32_t instr) {
uint32_t op = instr >> 26;
if (op == 0u) {
uint32_t funct = instr & 0x3Fu;
return funct == 0x08u || funct == 0x09u; /* JR / JALR */
}
if (op == 0x01u) { /* REGIMM */
uint32_t rt = (instr >> 16) & 0x1Fu;
return (rt == 0x00u || rt == 0x01u ||
rt == 0x10u || rt == 0x11u) ? 1 : -1;
}
if (op == 0x02u || op == 0x03u ||
(op >= 0x04u && op <= 0x07u)) /* J/JAL/branches */
return 1;
if (op >= 0x14u && op <= 0x17u) return -1; /* branch-likely: reserved */
if (op >= 0x10u && op <= 0x13u &&
((instr >> 21) & 0x1Fu) == 0x08u) return -1; /* unsupported BCz* */
return 0;
}
static int ranges_contain_word(const uint32_t *lo_list,
const uint32_t *len_list, int n,
uint32_t phys) {
if ((phys & 3u) != 0u || phys > (2u * 1024u * 1024u) - 4u) return 0;
for (int r = 0; r < n; r++) {
uint32_t lo = lo_list[r] & 0x1FFFFFFFu;
uint32_t len = len_list[r];
if (len >= 4u && phys >= lo && phys - lo <= len - 4u) return 1;
}
return 0;
}
/* Defense in depth for caches generated before guarded capture/codegen. Called
* only after the range CRC matches. Scan every hashed instruction so a missing
* slot at any range edge, an unsupported branch form, or a nested transfer in a
* delay slot rejects the whole native identity. */
static int ranges_delay_slots_hashed(const uint32_t *lo_list,
const uint32_t *len_list, int n) {
const uint8_t *ram = memory_get_ram_ptr();
const uint32_t ram_size = 2u * 1024u * 1024u;
if (!ram || n < 1 || n > MAX_CODE_RANGES) return 0;
for (int r = 0; r < n; r++) {
uint32_t lo = lo_list[r] & 0x1FFFFFFFu;
uint32_t len = len_list[r];
if ((lo & 3u) != 0u || (len & 3u) != 0u || len < 4u ||
lo >= ram_size || len > ram_size - lo)
return 0;
for (uint32_t pc = lo; pc < lo + len; pc += 4u) {
uint32_t instr = (uint32_t)ram[pc] |
((uint32_t)ram[pc + 1u] << 8) |
((uint32_t)ram[pc + 2u] << 16) |
((uint32_t)ram[pc + 3u] << 24);
int kind = mips_control_kind(instr);
if (kind < 0) return 0;
if (kind > 0) {
uint32_t slot = pc + 4u;
if (!ranges_contain_word(lo_list, len_list, n, slot)) return 0;
uint32_t slot_instr = (uint32_t)ram[slot] |
((uint32_t)ram[slot + 1u] << 8) |
((uint32_t)ram[slot + 2u] << 16) |
((uint32_t)ram[slot + 3u] << 24);
if (mips_control_kind(slot_instr) != 0) return 0;
}
}
}
return 1;
}
static int man_delay_slots_hashed(const ManFn *m) {
return ranges_delay_slots_hashed(m->lo, m->len, m->n);
}
static int cand_delay_slots_hashed(const Candidate *c) {
return ranges_delay_slots_hashed(c->range_lo, c->range_len, c->nranges);
}
/* ---- Candidate registration -------------------------------------------- */
static void loader_log(const char *fmt, ...); /* defined below */
static void cand_register(uint32_t phys, OverlayFn fn, const ManFn *m, int dll) {
if (s_cand_n >= CAND_CAP) return;
static void cand_register(uint32_t phys, OverlayFn fn, const ManFn *m, int dll,
int tier) {
if (s_cand_n >= CAND_CAP || !man_structurally_valid(m)) return;
int idx = s_cand_n++;
Candidate *c = &s_cand[idx];
c->addr = phys;
c->fn = fn;
c->dll = dll;
c->tier = (uint8_t)tier;
c->state = ENTRY_VALID;
c->nranges = 0;
for (int i = 0; i < m->n && c->nranges < MAX_CODE_RANGES; i++) {
@@ -606,17 +763,29 @@ static void cand_register(uint32_t phys, OverlayFn fn, const ManFn *m, int dll)
* present yet, and other overlays sharing the merged DLL are not present at
* all). A candidate is callable iff live RAM matches this compiled-from hash,
* which makes validity timing-independent and reload-on-return work. */
if (m->has_crc) {
c->crc_code = m->crc;
} else {
c->crc_code = cand_crc(c); /* legacy manifest without hashes */
}
c->crc_code = m->crc;
c->val_gen = cand_gensum(c);
c->state = (cand_crc(c) == c->crc_code) ? ENTRY_VALID : ENTRY_INVALID;
c->next = idx_head(phys);
idx_set_head(phys, idx);
c->state = (cand_crc(c) == c->crc_code && cand_delay_slots_hashed(c))
? ENTRY_VALID : ENTRY_INVALID;
/* Keep higher-priority compiler tiers first without discarding additive
* artifacts. New same-tier repairs precede older ones; lower-tier TCC
* remains available if every GCC candidate fails live-byte validation. */
int head = idx_head(phys);
if (head < 0 || s_cand[head].tier <= c->tier) {
c->next = head;
idx_set_head(phys, idx);
} else {
int prev = head;
while (s_cand[prev].next >= 0 &&
s_cand[s_cand[prev].next].tier > c->tier)
prev = s_cand[prev].next;
c->next = s_cand[prev].next;
s_cand[prev].next = idx;
}
exact_entry_set(phys);
range_index_add_candidate(idx);
s_range_candidate_generation++;
if (s_range_candidate_generation == 0u) s_range_candidate_generation = 1u;
if (c->state == ENTRY_VALID) s_valid_count++;
/* (sljit removed 2026-07-15: the gcc-DLL-obsoletes-older-sljit-shard block
* lived here. With no sljit shards ever registered (dll == -1), there is
@@ -653,16 +822,88 @@ const char *overlay_loader_last_msg(void) { return s_last_msg; }
* (region ran interpreted forever, no diagnostic). Found by the ABI-sweep
* negative test. scan_one_cache_dir now shouts if even 4096 is hit. */
#define CACHE_IDX_CAP 4096
enum { CACHE_TIER_UNKNOWN = 0, CACHE_TIER_TCC = 1, CACHE_TIER_GCC = 2 };
typedef struct {
uint32_t region_start;
uint32_t logical_crc;
uint64_t mtime;
int func_count;
int indexed_func_count;
uint8_t tier;
uint8_t manifest_ok;
uint8_t load_failed;
char path[768];
} CacheEntry;
static CacheEntry s_cache_idx[CACHE_IDX_CAP];
static int s_cache_idx_count = 0;
static int path_component_eq(const char *path, const char *wanted) {
size_t wanted_len = strlen(wanted);
const char *p = path;
while (*p) {
while (*p == '/' || *p == '\\') p++;
const char *start = p;
while (*p && *p != '/' && *p != '\\') p++;
size_t len = (size_t)(p - start);
if (len == wanted_len) {
size_t i = 0;
for (; i < len; i++) {
unsigned char a = (unsigned char)start[i];
unsigned char b = (unsigned char)wanted[i];
if (tolower(a) != tolower(b)) break;
}
if (i == len) return 1;
}
}
return 0;
}
static int cache_tier_from_path(const char *path) {
if (path_component_eq(path, "gcc")) return CACHE_TIER_GCC;
if (path_component_eq(path, "tcc")) return CACHE_TIER_TCC;
return CACHE_TIER_UNKNOWN;
}
static int cache_name_is_immutable(const char *name) {
return name && strlen(name) == 26u + OVERLAY_SHARED_EXT_LEN &&
name[8] == '_' && name[17] == '_';
}
static int cache_path_char_equal(unsigned char a, unsigned char b) {
#ifdef _WIN32
if ((a == '/' || a == '\\') && (b == '/' || b == '\\')) return 1;
return tolower(a) == tolower(b);
#else
return a == b;
#endif
}
static int cache_path_equal(const char *a, const char *b) {
if (!a || !b) return 0;
while (*a && *b) {
if (!cache_path_char_equal((unsigned char)*a, (unsigned char)*b))
return 0;
a++;
b++;
}
return *a == '\0' && *b == '\0';
}
static int cache_path_has_root(const char *path, const char *root) {
if (!path || !root) return 0;
size_t root_len = strlen(root);
while (root_len > 0u &&
(root[root_len - 1u] == '/' || root[root_len - 1u] == '\\'))
root_len--;
for (size_t i = 0; i < root_len; i++) {
if (!path[i] ||
!cache_path_char_equal((unsigned char)path[i],
(unsigned char)root[i]))
return 0;
}
return path[root_len] == '/' || path[root_len] == '\\';
}
#ifdef _WIN32
/* Image-only, bounded mapping for small recovery/fragment DLLs. Windows can
* spend ~100 ms in its loader/security path on the first LoadLibrary, enough to
@@ -876,6 +1117,9 @@ static void rebuild_lazy_manifest_index(void) {
s_lazy_page_tail[i] = -1;
}
for (int ci = 0; ci < s_cache_idx_count; ci++) {
s_cache_idx[ci].func_count = 0;
s_cache_idx[ci].indexed_func_count = 0;
s_cache_idx[ci].manifest_ok = 0;
char path[800];
snprintf(path, sizeof(path), "%s", s_cache_idx[ci].path);
size_t n = strlen(path);
@@ -887,8 +1131,8 @@ static void rebuild_lazy_manifest_index(void) {
int man_n = 0;
ManFn *man = parse_manifest(path, &man_n);
if (!man) continue;
s_cache_idx[ci].manifest_ok = 1;
s_cache_idx[ci].func_count = man_n;
s_cache_idx[ci].indexed_func_count = 0;
for (int mi = 0; mi < man_n; mi++) {
if (!man[mi].has_crc || man[mi].n <= 0) continue;
uint32_t entry = man[mi].entry & 0x1FFFFFFFu;
@@ -954,23 +1198,13 @@ static void rebuild_lazy_manifest_index(void) {
static int cache_idx_has_path(const char *path) {
for (int i = 0; i < s_cache_idx_count; i++)
if (strcmp(s_cache_idx[i].path, path) == 0) return 1;
return 0;
}
/* Dedup by FILENAME (<addr8>_<crc8>.dll = region+crc). Used so a tcc/ shard for a
* region already covered by a gcc/ shard is skipped: scan_cache_dir scans gcc/
* FIRST, so gcc always wins the tie (tier order gcc > tcc). Also subsumes the
* per-path rescan-idempotence check (same dir => same filename). */
static int cache_idx_has_basename(const char *fname) {
for (int i = 0; i < s_cache_idx_count; i++) {
const char *b = strrchr(s_cache_idx[i].path, '/');
b = b ? b + 1 : s_cache_idx[i].path;
if (strcmp(b, fname) == 0) return 1;
}
if (cache_path_equal(s_cache_idx[i].path, path)) return 1;
return 0;
}
/* Cache discovery is idempotent only by full path. Logical identities may have
* multiple immutable artifacts; validation and tier-aware selection happen
* after every artifact is indexed. */
/* Canonical cache arch-abi tag (caches are namespaced per backend AND per
* target so a Windows-x64 gcc DLL and, later, a same-OS arm64 build for the
* same fragment never comingle). compile_overlays.py
@@ -998,19 +1232,28 @@ const char *overlay_loader_arch_abi(void) { return PSX_OVERLAY_ARCH_ABI; }
#ifndef _WIN32
static int add_posix_cache_file(const PsxOverlayCacheFile *file, void *opaque) {
(void)opaque;
int tier = opaque ? *(const int *)opaque : CACHE_TIER_UNKNOWN;
/* Legacy 8_8 pairs were published by a replace protocol that could leave a
* new manifest beside an old DLL. Keep them on disk for Python seed
* migration, but never let them suppress or execute instead of a bound
* immutable 8_8_8 repair. */
if (!cache_name_is_immutable(file->name)) return 0;
if (s_cache_idx_count >= CACHE_IDX_CAP) {
loader_log("*** CACHE INDEX FULL (%d): further DLLs near %s are being "
"IGNORED — their regions will run interpreted. Raise "
"CACHE_IDX_CAP.", CACHE_IDX_CAP, file->path);
return 1;
}
if (cache_idx_has_basename(file->name)) return 0;
if (cache_idx_has_path(file->path)) return 0;
CacheEntry *e = &s_cache_idx[s_cache_idx_count++];
e->region_start = file->region_start;
e->logical_crc = file->content_crc;
e->mtime = file->mtime;
e->func_count = 0;
e->indexed_func_count = 0;
e->tier = (uint8_t)tier;
e->manifest_ok = 0;
e->load_failed = 0;
snprintf(e->path, sizeof(e->path), "%s", file->path);
return 0;
}
@@ -1018,7 +1261,7 @@ static int add_posix_cache_file(const PsxOverlayCacheFile *file, void *opaque) {
/* Scan one directory for <addr8>_<crc8>.dll cache entries into the index.
* `dir` is a full directory path. Idempotent (skips already-indexed paths). */
static void scan_one_cache_dir(const char *dir) {
static void scan_one_cache_dir(const char *dir, int tier) {
#ifdef _WIN32
char pattern[768];
snprintf(pattern, sizeof(pattern), "%s/*_*.dll", dir);
@@ -1028,6 +1271,7 @@ static void scan_one_cache_dir(const char *dir) {
do {
uint32_t addr = 0, crc = 0;
if (!psx_overlay_cache_name_parse(fd.cFileName, &addr, &crc)) continue;
if (!cache_name_is_immutable(fd.cFileName)) continue;
(void)crc;
if (s_cache_idx_count >= CACHE_IDX_CAP) {
/* Never-again (the silent-256 truncation): overflowing the index
@@ -1039,20 +1283,24 @@ static void scan_one_cache_dir(const char *dir) {
}
char full[768];
snprintf(full, sizeof(full), "%s/%s", dir, fd.cFileName);
/* skip if this region+crc is already indexed (rescan idempotence, AND a
* higher-priority namespace already covers it — gcc/ scanned before tcc/). */
if (cache_idx_has_basename(fd.cFileName)) continue;
/* Same full path means a rescan duplicate. Same logical name with a
* different artifact suffix is an additive repair and is retained. */
if (cache_idx_has_path(full)) continue;
CacheEntry *e = &s_cache_idx[s_cache_idx_count++];
e->region_start = addr;
e->logical_crc = crc;
e->mtime = ((uint64_t)fd.ftLastWriteTime.dwHighDateTime << 32) |
(uint64_t)fd.ftLastWriteTime.dwLowDateTime;
e->func_count = 0;
e->indexed_func_count = 0;
e->tier = (uint8_t)tier;
e->manifest_ok = 0;
e->load_failed = 0;
snprintf(e->path, sizeof(e->path), "%s", full);
} while (FindNextFileA(h, &fd));
FindClose(h);
#else
psx_overlay_posix_scan_cache_dir(dir, add_posix_cache_file, NULL);
psx_overlay_posix_scan_cache_dir(dir, add_posix_cache_file, &tier);
#endif
}
@@ -1118,7 +1366,7 @@ static void warn_on_cgtag_mismatch(const char *tier) {
static void cache_idx_remove_path(const char *path) {
for (int i = 0; i < s_cache_idx_count; i++) {
if (strcmp(s_cache_idx[i].path, path) == 0) {
if (cache_path_equal(s_cache_idx[i].path, path)) {
s_cache_idx[i] = s_cache_idx[--s_cache_idx_count];
return;
}
@@ -1149,6 +1397,7 @@ typedef struct PosixAbiSweep {
static int posix_abi_sweep_file(const PsxOverlayCacheFile *file, void *opaque) {
PosixAbiSweep *sweep = (PosixAbiSweep *)opaque;
if (!cache_name_is_immutable(file->name)) return 0;
char error[256] = {0};
void *handle = psx_overlay_posix_library_open(file->path, error, sizeof(error));
int abi = 0;
@@ -1208,6 +1457,7 @@ static void abi_preflight_sweep(const char *dir) {
HANDLE hf = FindFirstFileA(pattern, &fd);
if (hf != INVALID_HANDLE_VALUE) {
do {
if (!cache_name_is_immutable(fd.cFileName)) continue;
char full[900];
snprintf(full, sizeof full, "%s/%s", dir, fd.cFileName);
HMODULE h = LoadLibraryA(full);
@@ -1229,7 +1479,7 @@ static void abi_preflight_sweep(const char *dir) {
}
purged++;
for (int i = 0; i < s_cache_idx_count; i++) {
if (strcmp(s_cache_idx[i].path, full) == 0) {
if (cache_path_equal(s_cache_idx[i].path, full)) {
s_cache_idx[i] = s_cache_idx[--s_cache_idx_count];
break;
}
@@ -1262,19 +1512,18 @@ static void abi_preflight_sweep(const char *dir) {
static void scan_cache_dir(void) {
char dir[768];
/* Tier order: scan gcc/ FIRST (highest native priority — the dev/production
* shards), THEN tcc/ (the toolchain-free fallback). scan_one_cache_dir dedups
* by filename (region+crc), so a tcc shard for a region a gcc shard already
* covers is skipped — gcc always wins the tie. */
/* Index both tiers and every immutable artifact. Runtime selection prefers
* usable GCC over TCC; an invalid GCC artifact cannot suppress a valid TCC
* fallback merely because its filename was enumerated first. */
snprintf(dir, sizeof(dir), "%s/%s/gcc/%s/cg%d_%08x",
s_cache_dir, s_game_id, PSX_OVERLAY_ARCH_ABI, PSX_OVERLAY_CODEGEN_VER,
(unsigned)PSX_OVERLAY_CODEGEN_HASH);
scan_one_cache_dir(dir);
scan_one_cache_dir(dir, CACHE_TIER_GCC);
abi_preflight_sweep(dir);
snprintf(dir, sizeof(dir), "%s/%s/tcc/%s/cg%d_%08x",
s_cache_dir, s_game_id, PSX_OVERLAY_ARCH_ABI, PSX_OVERLAY_CODEGEN_VER,
(unsigned)PSX_OVERLAY_CODEGEN_HASH);
scan_one_cache_dir(dir);
scan_one_cache_dir(dir, CACHE_TIER_TCC);
abi_preflight_sweep(dir);
rebuild_lazy_manifest_index();
@@ -1307,17 +1556,31 @@ static void scan_cache_dir(void) {
* persist/publish writers, on-disk reload scan, and their debug accessors —
* lived here. No shards are produced or reloaded anymore.) */
/* True if the cache holds a DLL for this region compiled from an image with
* this CRC (filename <addr8>_<crc8>.dll). Autocapture's "unseen" test. */
static uint32_t lazy_man_crc(const ManFn *m); /* defined with lazy matching */
/* True if the cache holds a usable DLL for this region/logical image CRC.
* Filename identity alone is insufficient: a structurally valid but stale or
* unsafe manifest must not suppress additive recapture of its own repair. */
int overlay_loader_has_cached_crc(uint32_t region_start, uint32_t crc) {
for (int i = 0; i < s_cache_idx_count; i++) {
if (s_cache_idx[i].region_start != region_start) continue;
const char *fn = strrchr(s_cache_idx[i].path, '/');
fn = fn ? fn + 1 : s_cache_idx[i].path;
if (strlen(fn) == 17u + OVERLAY_SHARED_EXT_LEN &&
strcmp(fn + 17, OVERLAY_SHARED_EXT) == 0 &&
(uint32_t)strtoul(fn + 9, NULL, 16) == crc)
return 1;
if (s_cache_idx[i].manifest_ok &&
!s_cache_idx[i].load_failed &&
s_cache_idx[i].region_start == region_start &&
s_cache_idx[i].logical_crc == crc &&
s_cache_idx[i].func_count > 0 &&
s_cache_idx[i].indexed_func_count == s_cache_idx[i].func_count) {
int seen = 0, usable = 1;
for (int li = s_lazy_bundle_head[i]; li >= 0;
li = s_lazy_man[li].next_bundle) {
seen++;
if (lazy_man_crc(&s_lazy_man[li].fn) != s_lazy_man[li].fn.crc ||
!man_delay_slots_hashed(&s_lazy_man[li].fn)) {
usable = 0;
break;
}
}
if (usable && seen == s_cache_idx[i].func_count) return 1;
}
}
return 0;
}
@@ -1643,6 +1906,7 @@ static void init_callbacks(void) {
#ifdef _WIN32
static int load_overlay_dll(const char *dll_path, ManFn *man, int man_n, int dll) {
int tier = cache_tier_from_path(dll_path);
HMODULE h = LoadLibraryA(dll_path);
if (!h) {
loader_log("LoadLibrary(%s) failed: %lu", dll_path, GetLastError());
@@ -1714,7 +1978,7 @@ static int load_overlay_dll(const char *dll_path, ManFn *man, int man_n, int dll
* the interpreter rather than registered with a guessed extent. */
ManFn *m = man_find(man, man_n, addr);
if (!m || m->n == 0) { s_no_manifest++; continue; }
cand_register(addr & 0x1FFFFFFFu, fn, m, dll);
cand_register(addr & 0x1FFFFFFFu, fn, m, dll, tier);
registered++;
}
loader_log("loaded %s -> %d candidates (%u no-manifest)",
@@ -1723,6 +1987,7 @@ static int load_overlay_dll(const char *dll_path, ManFn *man, int man_n, int dll
}
#else
static int load_overlay_dll(const char *dll_path, ManFn *man, int man_n, int dll) {
int tier = cache_tier_from_path(dll_path);
char error[256] = {0};
void *h = psx_overlay_posix_library_open(dll_path, error, sizeof(error));
if (!h) { loader_log("dlopen(%s) failed: %s", dll_path, error); return 0; }
@@ -1765,7 +2030,7 @@ static int load_overlay_dll(const char *dll_path, ManFn *man, int man_n, int dll
if (m->entry == 0 || m->n == 0) continue;
OverlayFn fn = (OverlayFn)psx_overlay_posix_library_entry(h, m->entry);
if (!fn) continue;
cand_register(m->entry & 0x1FFFFFFFu, fn, m, dll);
cand_register(m->entry & 0x1FFFFFFFu, fn, m, dll, tier);
registered++;
}
loader_log("loaded %s -> %d manifest candidates", dll_path, registered);
@@ -1783,6 +2048,10 @@ static int load_overlay_dll(const char *dll_path, ManFn *man, int man_n, int dll
/* ---- Public API -------------------------------------------------------- */
void overlay_loader_init(const char *cache_dir, const char *game_id) {
{
const char *perf = getenv("PSX_RUNTIME_PERF_DIAG");
s_native_hot_enabled = perf && perf[0] && perf[0] != '0';
}
for (uint32_t p = 0; p < RANGE_PAGE_COUNT; p++) {
s_range_page_head[p] = -1;
s_range_page_tail[p] = -1;
@@ -1821,6 +2090,21 @@ void overlay_loader_init(const char *cache_dir, const char *game_id) {
s_native_block_n, s_native_block_n == 1 ? "y" : "ies", nb);
}
}
#ifndef PSX_NO_DEBUG_TOOLS
{
const char *limit = getenv("PSX_NATIVE_RANK_LIMIT");
if (limit && *limit) {
unsigned long long n = strtoull(limit, NULL, 0);
s_native_rank_limit = n >= UINT32_MAX ? UINT32_MAX - 1u : (uint32_t)n;
s_native_rank_next = 0;
s_native_rank_blocked = 0;
fprintf(stdout,
"psxrecomp: PSX_NATIVE_RANK_LIMIT=%u: "
"first-seen candidate bisection ON\n",
s_native_rank_limit);
}
}
#endif
/* Boot-time full-interp override (diagnostic): PSX_OVERLAY_NATIVE_OFF=1 forces
* native overlay execution off from the first instruction, so a pristine
* interpreter reference can be captured without racing post-boot cmds. Same
@@ -1939,9 +2223,30 @@ void overlay_loader_rescan(void) {
static char s_loaded_paths[MAX_LOADED_DLLS][768];
static int s_nloaded_paths = 0;
#ifndef PSX_NO_DEBUG_TOOLS
static int native_rank_allows(Candidate *c, uint32_t pc) {
if (s_native_rank_limit == UINT32_MAX) return 1;
if (c->native_rank == 0) {
c->native_rank = ++s_native_rank_next;
const char *path = (c->dll >= 0 && c->dll < s_nloaded_paths)
? s_loaded_paths[c->dll] : "<loading>";
fprintf(stdout,
"psxrecomp: native-rank rank=%u candidate=%d owner=0x%08X "
"pc=0x%08X crc=0x%08X dll=%s frame=%llu cycle=%llu allow=%d\n",
c->native_rank, (int)(c - s_cand), c->addr, pc,
c->crc_code, path, (unsigned long long)s_frame_count,
(unsigned long long)psx_get_cycle_count(),
c->native_rank <= s_native_rank_limit);
}
if (c->native_rank <= s_native_rank_limit) return 1;
s_native_rank_blocked++;
return 0;
}
#endif
static int dll_already_loaded(const char *path) {
for (int i = 0; i < s_nloaded_paths; i++)
if (strcmp(s_loaded_paths[i], path) == 0) return 1;
if (cache_path_equal(s_loaded_paths[i], path)) return 1;
return 0;
}
@@ -1995,19 +2300,14 @@ int overlay_loader_load_published(const char *dll_path) {
if (!s_active || !dll_path || !dll_path[0]) return 0;
/* The compiler command is trusted local code, but keep its native-library
* output confined to the cache root the framework injected. */
size_t root_len = strlen(s_cache_dir);
#ifdef _WIN32
if (_strnicmp(dll_path, s_cache_dir, root_len) != 0) return 0;
#else
if (strncmp(dll_path, s_cache_dir, root_len) != 0) return 0;
#endif
if (dll_path[root_len] != '/' && dll_path[root_len] != '\\') return 0;
if (!cache_path_has_root(dll_path, s_cache_dir)) return 0;
const char *base = strrchr(dll_path, '/');
const char *win_base = strrchr(dll_path, '\\');
if (!base || (win_base && win_base > base)) base = win_base;
base = base ? base + 1 : dll_path;
uint32_t addr = 0, crc = 0;
if (!psx_overlay_cache_name_parse(base, &addr, &crc)) return 0;
if (!cache_name_is_immutable(base)) return 0;
(void)addr;
(void)crc;
if (dll_already_loaded(dll_path)) return 0;
@@ -2039,13 +2339,18 @@ static uint32_t lazy_man_gensum(const ManFn *m) {
}
static int lazy_man_matches(LazyMan *lm) {
if (!man_structurally_valid(&lm->fn)) {
lm->state = ENTRY_INVALID;
return 0;
}
uint32_t gen = lazy_man_gensum(&lm->fn);
if (lm->state == ENTRY_VALID && lm->val_gen == gen) return 1;
if (lm->state == ENTRY_INVALID && lm->val_gen == gen) return 0;
uint32_t live = lazy_man_crc(&lm->fn);
lm->val_gen = gen;
s_last_crc = live;
lm->state = (live == lm->fn.crc) ? ENTRY_VALID : ENTRY_INVALID;
lm->state = (live == lm->fn.crc && man_delay_slots_hashed(&lm->fn))
? ENTRY_VALID : ENTRY_INVALID;
return lm->state == ENTRY_VALID;
}
@@ -2055,7 +2360,7 @@ static int lazy_man_matches(LazyMan *lm) {
* coherent snapshot to replace many synchronous incremental LoadLibrary calls
* at a stage transition without substituting a stale CPS owner. */
static int lazy_bundle_matches(int ci) {
if (ci < 0 || ci >= s_cache_idx_count) return 0;
if (ci < 0 || ci >= s_cache_idx_count || ci >= CACHE_IDX_CAP) return 0;
if (s_cache_idx[ci].func_count <= 0 ||
s_cache_idx[ci].indexed_func_count != s_cache_idx[ci].func_count)
return 0;
@@ -2068,6 +2373,27 @@ static int lazy_bundle_matches(int ci) {
return seen == s_cache_idx[ci].func_count;
}
static int lazy_candidate_preferred(int li, int current) {
if (current < 0) return 1;
const CacheEntry *a = &s_cache_idx[s_lazy_man[li].cache_idx];
const CacheEntry *b = &s_cache_idx[s_lazy_man[current].cache_idx];
if (a->tier != b->tier) return a->tier > b->tier;
if (a->mtime != b->mtime) return a->mtime > b->mtime;
return strcmp(a->path, b->path) > 0;
}
/* Prefer GCC over TCC even when the TCC option happens to be a broader bundle.
* Within one tier, a fully coherent bundle remains stronger than a partial
* region-local function match. */
static int lazy_choose_complete_or_fallback(int complete, int fallback) {
if (complete < 0) return fallback;
if (fallback < 0) return complete;
const CacheEntry *a = &s_cache_idx[s_lazy_man[complete].cache_idx];
const CacheEntry *b = &s_cache_idx[s_lazy_man[fallback].cache_idx];
if (a->tier != b->tier) return a->tier > b->tier ? complete : fallback;
return complete;
}
/* A streamed variant is not byte-coherent until immediately before its first
* call, too late to hide Windows' first image-map cost. Map every small bundle
* for this ONE base after boot. Tomba's mature vault has 141 here versus 712 in
@@ -2094,6 +2420,7 @@ static int lazy_is_loadable(int li, uint32_t region_start, uint32_t phys,
if (ci < 0 || ci >= s_cache_idx_count) return 0;
return (!require_region_start ||
s_cache_idx[ci].region_start == region_start) &&
!s_cache_idx[ci].load_failed &&
!dll_already_loaded(s_cache_idx[ci].path) &&
lazy_man_contains(&lm->fn, phys) && lazy_man_matches(lm);
}
@@ -2111,7 +2438,8 @@ static int lazy_better_bundle(int li, int best) {
static int lazy_load_selected(int li) {
if (li < 0 || li >= s_lazy_man_n) return 0;
int ci = s_lazy_man[li].cache_idx;
if (ci < 0 || ci >= s_cache_idx_count ||
if (ci < 0 || ci >= s_cache_idx_count || ci >= CACHE_IDX_CAP ||
s_cache_idx[ci].load_failed ||
dll_already_loaded(s_cache_idx[ci].path) ||
s_cache_idx[ci].func_count <= 0) return 0;
/* If proactive warming has not reached this fragment yet, prefer the
@@ -2120,6 +2448,7 @@ static int lazy_load_selected(int li) {
overlay_image_warm_cancel(ci);
s_last_file_found = 1;
int loaded = load_one_dll(s_cache_idx[ci].path) > 0;
if (!loaded) s_cache_idx[ci].load_failed = 1;
overlay_image_warm_release(ci);
return loaded;
}
@@ -2156,6 +2485,9 @@ static int try_load_region(uint32_t phys) {
if (!already_checked(region_start)) mark_checked(region_start);
int select_attempts = 0;
retry_artifact:
;
/* Prefer the broadest fully coherent snapshot. If none has every function
* live, retain the historical first matching-function fallback. */
int best = -1;
@@ -2177,11 +2509,12 @@ static int try_load_region(uint32_t phys) {
/* Cross-region recovery is safe only for a fully coherent CPS bundle.
* A partial exact-function match can have snapshot-specific internal
* tails, so retain it as fallback only when the heuristic base agrees. */
if (s_cache_idx[ci].region_start == region_start && fallback < 0)
if (s_cache_idx[ci].region_start == region_start &&
lazy_candidate_preferred(li, fallback))
fallback = li;
/* Entry chains are newest-first/semantic. Preserve their established
* order instead of substituting a larger historical bundle. */
if (lazy_bundle_matches(ci) && best < 0)
if (lazy_bundle_matches(ci) && lazy_candidate_preferred(li, best))
best = li;
}
@@ -2198,14 +2531,20 @@ static int try_load_region(uint32_t phys) {
* exact manifested entry, an interior PC alone is not a unique
* identity for a reused-address CPS bundle. */
if (!lazy_is_loadable(li, region_start, phys, 1)) continue;
if (fallback < 0) fallback = li;
if (lazy_candidate_preferred(li, fallback)) fallback = li;
int ci = s_lazy_man[li].cache_idx;
if (lazy_bundle_matches(ci) && best < 0)
if (lazy_bundle_matches(ci) && lazy_candidate_preferred(li, best))
best = li;
}
}
if (best >= 0 && lazy_load_selected(best)) return 1;
return fallback >= 0 && lazy_load_selected(fallback);
int selected = lazy_choose_complete_or_fallback(best, fallback);
if (selected < 0) return 0;
if (lazy_load_selected(selected)) return 1;
/* An unloadable/corrupt GCC artifact must not suppress an older immutable
* repair or the TCC fallback. lazy_load_selected marks the failed cache
* entry, so a bounded re-selection chooses the next live candidate. */
if (++select_attempts < s_cache_idx_count) goto retry_artifact;
return 0;
}
/* O(1)-bucket discriminator between a real manifested function entry and an
@@ -2260,7 +2599,7 @@ static int range_candidate_matches(int i, uint32_t phys) {
s_rehashes++;
s_last_crc = live;
c->val_gen = gen;
if (live == c->crc_code) {
if (live == c->crc_code && cand_delay_slots_hashed(c)) {
if (c->state != ENTRY_VALID) {
c->state = ENTRY_VALID;
s_valid_count++;
@@ -2279,6 +2618,13 @@ static int range_candidate_matches(int i, uint32_t phys) {
return 0;
}
static int range_candidate_preferred(int candidate, int current) {
if (current < 0) return 1;
if (s_cand[candidate].tier != s_cand[current].tier)
return s_cand[candidate].tier > s_cand[current].tier;
return candidate > current; /* newest same-tier repair */
}
static int overlay_find_by_range(uint32_t phys) {
/* Definitive guard for the overlay-off case: the range page index
* (s_range_page_head) is only initialized to -1 by overlay_loader_init,
@@ -2292,29 +2638,30 @@ static int overlay_find_by_range(uint32_t phys) {
RangePcCache *pc = &s_range_pc_cache[
(phys * 2654435761u) & RANGE_PC_CACHE_MASK];
if (pc->cand >= 0 && pc->phys == phys &&
if (pc->generation == s_range_candidate_generation &&
pc->cand >= 0 && pc->phys == phys &&
range_candidate_matches(pc->cand, phys))
return pc->cand;
int best = -1;
if (s_range_index_overflow) {
for (int i = 0; i < s_cand_n; i++)
if (range_candidate_matches(i, phys)) {
pc->phys = phys;
pc->cand = i;
return i;
}
return -1;
}
for (int li = s_range_page_head[page]; li >= 0; li = s_range_links[li].next) {
int i = s_range_links[li].cand;
if (range_candidate_matches(i, phys)) {
pc->phys = phys;
pc->cand = i;
return i;
if (range_candidate_matches(i, phys) &&
range_candidate_preferred(i, best))
best = i;
} else {
for (int li = s_range_page_head[page]; li >= 0;
li = s_range_links[li].next) {
int i = s_range_links[li].cand;
if (range_candidate_matches(i, phys) &&
range_candidate_preferred(i, best))
best = i;
}
}
return -1;
pc->phys = phys;
pc->generation = s_range_candidate_generation;
pc->cand = best;
return best;
}
int overlay_loader_dispatch(CPUState *cpu, uint32_t addr) {
@@ -2400,7 +2747,7 @@ retry_candidates:
s_rehashes++;
s_last_crc = live;
c->val_gen = gen;
matched = (live == c->crc_code);
matched = (live == c->crc_code && cand_delay_slots_hashed(c));
}
if (_probe) s_cps_probe_matched = matched;
if (matched) {
@@ -2432,6 +2779,10 @@ retry_candidates:
}
if (!s_native_exec || overlay_native_blocked(c->addr) || overlay_native_blocked(addr))
{ if (_probe) s_cps_probe_outcome = 4; s_would_run_native++; s_disp_interp++; return 0; }
#ifndef PSX_NO_DEBUG_TOOLS
if (!native_rank_allows(c, addr))
{ if (_probe) s_cps_probe_outcome = 7; s_would_run_native++; s_disp_interp++; return 0; }
#endif
}
if (_probe) s_cps_probe_outcome = 2;
#ifndef PSX_NO_DEBUG_TOOLS
@@ -2445,6 +2796,7 @@ retry_candidates:
s_native_inprogress = c->addr;
#endif
s_native_calls_total++;
native_hot_note(c->addr);
if (s_active_depth < (int)(sizeof(s_active_stack) / sizeof(s_active_stack[0])))
s_active_stack[s_active_depth++] = ci;
s_disp_native++;
@@ -2507,7 +2859,7 @@ retry_candidates:
s_rehashes++;
s_last_crc = live;
c->val_gen = gen;
matched = (live == c->crc_code);
matched = (live == c->crc_code && cand_delay_slots_hashed(c));
}
if (matched) {
if (c->state != ENTRY_VALID) {
@@ -2567,6 +2919,10 @@ retry_candidates:
* routing for one function only (bisection localization). */
if (!s_native_exec || overlay_native_blocked(c->addr))
{ s_would_run_native++; s_disp_interp++; return 0; }
#ifndef PSX_NO_DEBUG_TOOLS
if (!native_rank_allows(c, addr))
{ s_would_run_native++; s_disp_interp++; return 0; }
#endif
/* Record into the always-on ring BEFORE the call; mark in-progress
* so a freeze inside this fn is visible at dump time. */
@@ -2581,6 +2937,7 @@ retry_candidates:
s_native_inprogress = c->addr;
#endif
s_native_calls_total++;
native_hot_note(c->addr);
if (s_active_depth < (int)(sizeof(s_active_stack) / sizeof(s_active_stack[0])))
s_active_stack[s_active_depth++] = i;
@@ -2704,6 +3061,22 @@ void overlay_loader_get_load_timing(uint64_t *total_us, uint64_t *max_us,
if (last_us) *last_us = s_load_last_us;
}
void overlay_loader_take_hot_native(uint32_t *pc, uint64_t *calls) {
uint32_t best_pc = 0;
uint64_t best_calls = 0;
if (s_native_hot_enabled) {
for (uint32_t i = 0; i < NATIVE_HOT_CAP; i++) {
if (s_native_hot[i].calls > best_calls) {
best_pc = s_native_hot[i].pc;
best_calls = s_native_hot[i].calls;
}
s_native_hot[i].calls = 0;
}
}
if (pc) *pc = best_pc;
if (calls) *calls = best_calls;
}
/* Reload diagnostics. Repurposed for the per-entry model:
* r0_valid -> candidates currently VALID
* r0_writes... -> entries blacklisted (self-mod)
@@ -2817,6 +3190,14 @@ static uint32_t s_detail_addr = 0;
static uint32_t s_detail_nat_gpr[32], s_detail_int_gpr[32];
static uint32_t s_detail_nat_hi, s_detail_nat_lo, s_detail_int_hi, s_detail_int_lo;
void overlay_loader_get_shadow_summary(uint64_t *calls, uint64_t *divergences,
uint32_t *first_divergence_pc) {
if (calls) *calls = s_shadow_calls;
if (divergences) *divergences = s_shadow_divs;
if (first_divergence_pc)
*first_divergence_pc = s_detail_captured ? s_detail_addr : 0;
}
void overlay_loader_set_diff_mode(int on) { s_diff_mode = on ? 1 : 0; }
static void run_shadow_diff(CPUState *cpu, Candidate *c, uint32_t addr) {
runtime/src/overlay_posix.c
+13 -4
View File
@@ -23,18 +23,27 @@ static int parse_hex8(const char *text, uint32_t *value) {
int psx_overlay_cache_name_parse(const char *name, uint32_t *region_start,
uint32_t *content_crc) {
uint32_t addr = 0, crc = 0;
uint32_t addr = 0, crc = 0, artifact = 0;
#ifdef _WIN32
static const char extension[] = ".dll";
#else
static const char extension[] = ".so";
#endif
if (!name || strlen(name) != 17 + sizeof(extension) - 1 || name[8] != '_' ||
strcmp(name + 17, extension) != 0 ||
!parse_hex8(name, &addr) || !parse_hex8(name + 9, &crc))
if (!name) return 0;
size_t name_len = strlen(name);
size_t ext_len = sizeof(extension) - 1u;
if (name_len < ext_len) return 0;
size_t stem_len = name_len - ext_len;
if (
(stem_len != 17u && stem_len != 26u) || name[8] != '_' ||
strcmp(name + stem_len, extension) != 0 ||
!parse_hex8(name, &addr) || !parse_hex8(name + 9, &crc) ||
(stem_len == 26u &&
(name[17] != '_' || !parse_hex8(name + 18, &artifact))))
return 0;
if (region_start) *region_start = addr;
if (content_crc) *content_crc = crc;
(void)artifact;
return 1;
}
runtime/src/psx_icache.c
+18 -10
View File
@@ -28,6 +28,7 @@
* leader) so the cache evolves identically to Beetle for the same fetch stream.
*/
#include "cpu_state.h"
#include "psx_icache.h"
#include <stdint.h>
#include <stdlib.h>
@@ -51,20 +52,23 @@ int psx_icache_enabled(void) {
/* Per-word tag+validity, mirroring Beetle ICache[idx].TV. Init to a value that can
* never equal an aligned fetch address (aligned addrs have bits 0-1 = 0), so every
* line starts cold (miss). */
static uint32_t s_icache_tv[1024];
uint32_t g_psx_icache_tv[1024];
int g_psx_icache_active = -1;
void psx_icache_reset(void) {
for (int i = 0; i < 1024; i++) s_icache_tv[i] = 0x1u; /* bit0 set => never matches */
g_psx_icache_active = psx_icache_enabled();
for (int i = 0; i < 1024; i++) g_psx_icache_tv[i] = 0x1u;
}
/* Charge the fetch cost for the instruction at `addr` and update the cache tags.
* `addr` is the runtime guest virtual PC (aligned). Gated on PSX_ENABLE_BLOCK_CYCLES. */
void psx_icache_fetch(CPUState* cpu, uint32_t addr) {
void psx_icache_fetch_miss(CPUState* cpu, uint32_t addr) {
{ extern int g_ls_replay_active; if (g_ls_replay_active) return; } /* lockstep replay: no global icache mutation */
#ifdef PSX_ENABLE_BLOCK_CYCLES
if (!psx_icache_enabled()) return;
if (g_psx_icache_active < 0) g_psx_icache_active = psx_icache_enabled();
if (!g_psx_icache_active) return;
uint32_t idx = (addr & 0xFFCu) >> 2;
if (s_icache_tv[idx] == addr) return; /* HIT: +0, no give-back clear */
if (g_psx_icache_tv[idx] == addr) return; /* HIT: +0, no give-back clear */
/* MISS — clear the pending load give-back (Beetle cpu.cpp:542-543). */
cpu->read_absorb[cpu->read_absorb_which] = 0u;
@@ -79,13 +83,13 @@ void psx_icache_fetch(CPUState* cpu, uint32_t addr) {
* + count the words from the missing one to the line end. */
uint32_t line = addr & 0xFFFFFFF0u;
uint32_t bidx = (addr & 0xFF0u) >> 2; /* first word index of the line */
s_icache_tv[bidx + 0] = line | 0x0u | 0x2u;
s_icache_tv[bidx + 1] = line | 0x4u | 0x2u;
s_icache_tv[bidx + 2] = line | 0x8u | 0x2u;
s_icache_tv[bidx + 3] = line | 0xCu | 0x2u;
g_psx_icache_tv[bidx + 0] = line | 0x0u | 0x2u;
g_psx_icache_tv[bidx + 1] = line | 0x4u | 0x2u;
g_psx_icache_tv[bidx + 2] = line | 0x8u | 0x2u;
g_psx_icache_tv[bidx + 3] = line | 0xCu | 0x2u;
uint32_t cost = 3u;
for (uint32_t i = (addr & 0xCu) >> 2; i < 4u; i++) {
s_icache_tv[bidx + i] &= ~0x2u; /* valid: TV == that word's addr */
g_psx_icache_tv[bidx + i] &= ~0x2u; /* valid: TV == that word's addr */
cost++;
}
psx_advance_cycles(cost);
@@ -93,3 +97,7 @@ void psx_icache_fetch(CPUState* cpu, uint32_t addr) {
(void)cpu; (void)addr;
#endif
}
void psx_icache_fetch(CPUState* cpu, uint32_t addr) {
psx_icache_fetch_miss(cpu, addr);
}
runtime/tests/run_overlay_posix_test.sh
+1
View File
@@ -15,6 +15,7 @@ cc -std=c99 -Wall -Wextra -Werror -fPIC -shared \
"$root/runtime/tests/overlay_posix_fixture.c" \
-o "$cache/80010000_DEADBEEF.so"
: > "$cache/00000000_00000000.so"
: > "$cache/80010000_DEADBEEF_13579BDF.so"
: > "$cache/80010000_DEADBEG0.so"
: > "$base/$other/80020000_12345678.so"
runtime/tests/test_interpreter_perf_guards.py
+63
View File
@@ -27,7 +27,9 @@ def main():
interp = (ROOT / "runtime/src/dirty_ram_interp.c").read_text(encoding="utf-8")
capture = (ROOT / "runtime/src/overlay_capture.c").read_text(encoding="utf-8")
loader = (ROOT / "runtime/src/overlay_loader.c").read_text(encoding="utf-8")
autocompile = (ROOT / "runtime/src/autocompile.c").read_text(encoding="utf-8")
memory = (ROOT / "runtime/src/memory.c").read_text(encoding="utf-8")
gte = (ROOT / "runtime/src/gte.cpp").read_text(encoding="utf-8")
cycles = (ROOT / "runtime/src/psx_cycles.c").read_text(encoding="utf-8")
cyc_header = (ROOT / "runtime/include/psx_cyc.h").read_text(encoding="utf-8")
starvation = (ROOT / "runtime/include/starvation_ring.h").read_text(encoding="utf-8")
@@ -39,6 +41,12 @@ def main():
raise AssertionError("executed-PC recording regressed to hash probing")
if "sizeof(g_dirty_ram_exec_pc_bitmap)" not in capture:
raise AssertionError("autocapture does not snapshot the 64 KiB bitmap")
write_window = body(capture, "write_json_window")
if ("size += 4u" not in write_window or
"phys + size <= ram_size - 4u" not in write_window or
"window_hi" in write_window):
raise AssertionError(
"capture delay-slot guard is missing or stops at an artificial window boundary")
if "g_dirty_ram_exec_pc_table" in capture:
raise AssertionError("capture still references the removed execution table")
before_dma = body(capture, "overlay_capture_before_dma")
@@ -50,6 +58,9 @@ def main():
raise AssertionError("outgoing variant is not queued before evidence reset")
if "first_page = lo >> page_shift" not in before_dma:
raise AssertionError("pre-DMA capture is not scoped to touched whole pages")
for guard in ("first_page - 1u", "last_page + 1u", "exec_pc_bitmap"):
if guard not in capture:
raise AssertionError(f"pre-DMA capture lost executed-page halo: {guard}")
dispatch_inner = body(interp, "dirty_ram_dispatch_inner")
if "(!current_page_dirty || next_page != current_page)" not in dispatch_inner:
raise AssertionError("page fast path does not preserve clean-miss behavior")
@@ -64,14 +75,45 @@ def main():
raise AssertionError("cache rescan does not invalidate negative misses")
if "lazy_miss_invalidate_loader();" not in body(loader, "load_one_dll"):
raise AssertionError("DLL publication does not invalidate negative misses")
lazy_match = body(loader, "lazy_man_matches")
if "man_delay_slots_hashed(&lm->fn)" not in lazy_match:
raise AssertionError("legacy manifests can expose unhashed delay slots")
rebuild = body(loader, "rebuild_lazy_manifest_index")
if "overlay_watch_set_range(lo, lm->fn.len[r])" not in rebuild:
raise AssertionError("unloaded manifest pages are not generation-watched")
if "manifest_ok = 1" not in rebuild:
raise AssertionError("strictly parsed manifests are not marked usable")
if "cache_idx_has_basename" in loader:
raise AssertionError("immutable repair artifacts are still deduped by logical name")
scan_one = body(loader, "scan_one_cache_dir")
for guard in ("cache_name_is_immutable(fd.cFileName)",
"cache_idx_has_path(full)", "e->logical_crc = crc",
"e->tier = (uint8_t)tier"):
if guard not in scan_one:
raise AssertionError(f"artifact-aware cache indexing lost: {guard}")
has_crc = body(loader, "overlay_loader_has_cached_crc")
for guard in ("manifest_ok", "!s_cache_idx[i].load_failed",
"logical_crc == crc", "lazy_man_crc(",
"man_delay_slots_hashed("):
if guard not in has_crc:
raise AssertionError(f"cached-CRC query accepts unusable artifacts: {guard}")
try_load = body(loader, "try_load_region")
if "lazy_is_loadable(li, region_start, phys, 0)" not in try_load:
raise AssertionError("exact manifest entries are still gated by dirty-run base")
if "lazy_is_loadable(li, region_start, phys, 1)" not in try_load:
raise AssertionError("non-exact range fallback lost region narrowing")
if "lazy_candidate_preferred" not in try_load:
raise AssertionError("lazy artifact selection is still directory-order dependent")
for guard in ("load_failed", "goto retry_artifact"):
if guard not in try_load and guard not in body(loader, "lazy_is_loadable"):
raise AssertionError(f"failed artifact can suppress a valid fallback: {guard}")
range_lookup = body(loader, "overlay_find_by_range")
for guard in ("s_range_candidate_generation", "range_candidate_preferred"):
if guard not in range_lookup:
raise AssertionError(f"loaded range selection lost tier/new-artifact ordering: {guard}")
ac_poll = body(autocompile, "autocompile_poll_main")
if "one idempotent batch-end" not in ac_poll or "overlay_loader_rescan();" not in ac_poll:
raise AssertionError("direct shard handoff is not reconciled into the additive index")
watched_write = body(memory, "overlay_watch_note_write")
if "g_dirty_ram_exec_page_bitmap" not in watched_write:
raise AssertionError("RAM writes do not clear stale per-page capture evidence")
@@ -84,6 +126,17 @@ def main():
candidate = body(loader, "overlay_loader_is_candidate")
if "exact_entry_has(phys)" not in candidate or "idx_head" in candidate:
raise AssertionError("candidate presence regressed from bitmap to hash probes")
if "cache_path_equal(s_loaded_paths[i], path)" not in body(
loader, "dll_already_loaded"):
raise AssertionError("published/scanned path separator aliases can double-load DLLs")
posix_sweep = body(loader, "posix_abi_sweep_file")
if not (posix_sweep.find("cache_name_is_immutable(file->name)") <
posix_sweep.find("psx_overlay_posix_library_open")):
raise AssertionError("ABI preflight can execute a legacy POSIX constructor")
abi_sweep = body(loader, "abi_preflight_sweep")
if not (abi_sweep.find("cache_name_is_immutable(fd.cFileName)") <
abi_sweep.find("LoadLibraryA(full)")):
raise AssertionError("ABI preflight can execute a legacy Windows DllMain")
interp_step = body(interp, "interp_cyc_step")
if "g_psx_cycle_fast_limit" not in interp_step or "next <= g_psx_cycle_fast_limit" not in interp_step:
@@ -137,6 +190,16 @@ def main():
if "defined(PSX_NO_DEBUG_TOOLS) && !defined(PSX_COSIM)" not in memory:
raise AssertionError("main-RAM value fast path leaks into diagnostic/COSIM builds")
gte_exec = body(gte, "gte_execute")
for probe in ("s_gte_exec_count++", "s_gte_caller_ra =",
"int16_t intpl_pre_ir[4]", "gte_rtp_record(&gte, cmd)",
"gte_intpl_record(&gte, intpl_pre_ir, intpl_pre_fc)"):
pos = gte_exec.find(probe)
begin = gte_exec.rfind("#ifndef PSX_NO_DEBUG_TOOLS", 0, pos)
end = gte_exec.find("#endif", pos)
if pos < 0 or begin < 0 or end < 0:
raise AssertionError(f"production GTE path still executes diagnostic probe: {probe}")
print("PASS: capture/cache and interpreter cycle hot-path guards intact")
return 0
runtime/tests/test_overlay_init_guard.py
+1 -1
View File
@@ -61,7 +61,7 @@ def main() -> int:
require_before(
candidate,
r"if\s*\(\s*!s_active\s*\)\s*return\s+0\s*;",
r"idx_head\s*\(",
r"(?:idx_head|exact_entry_has)\s*\(",
"overlay_loader_is_candidate",
)
runtime/tests/test_overlay_posix.c
+11 -2
View File
@@ -27,9 +27,18 @@ static void check_name_parser(void) {
assert(addr == 0 && crc == 0);
assert(psx_overlay_cache_name_parse("89abcdef_ABCDEF01.so", &addr, &crc));
assert(addr == 0x89ABCDEFu && crc == 0xABCDEF01u);
assert(psx_overlay_cache_name_parse(
"89abcdef_ABCDEF01_13579BDF.so", &addr, &crc));
assert(addr == 0x89ABCDEFu && crc == 0xABCDEF01u);
assert(!psx_overlay_cache_name_parse("", &addr, &crc));
assert(!psx_overlay_cache_name_parse(".so", &addr, &crc));
assert(!psx_overlay_cache_name_parse("G0010000_DEADBEEF.so", &addr, &crc));
assert(!psx_overlay_cache_name_parse("80010000_DEADBEG0.so", &addr, &crc));
assert(!psx_overlay_cache_name_parse(
"80010000_DEADBEEF_DEADBEG0.so", &addr, &crc));
assert(!psx_overlay_cache_name_parse(
"80010000_DEADBEEFDEADBEEF.so", &addr, &crc));
assert(!psx_overlay_cache_name_parse("80010000_DEADBEEF.dll", &addr, &crc));
assert(!psx_overlay_cache_name_parse("80010000-DEADBEEF.so", &addr, &crc));
assert(!psx_overlay_cache_name_parse("80010000_DEADBEEF.so.extra", &addr, &crc));
@@ -40,8 +49,8 @@ int main(int argc, char **argv) {
check_name_parser();
ScanResult result = {0};
assert(psx_overlay_posix_scan_cache_dir(argv[1], record_cache_file, &result) == 2);
assert(result.count == 2);
assert(psx_overlay_posix_scan_cache_dir(argv[1], record_cache_file, &result) == 3);
assert(result.count == 3);
assert(result.addr == 0x80010000u);
assert(result.crc == 0xDEADBEEFu);
runtime/tests/test_psx_icache_fastpath.c
+94
View File
@@ -0,0 +1,94 @@
#include "cpu_state.h"
#include "psx_icache.h"
#include <stdio.h>
#include <string.h>
int g_ls_replay_active = 0;
static uint64_t test_cycles = 0;
void psx_advance_cycles(uint32_t cycles) { test_cycles += cycles; }
static int expect(int condition, const char *message) {
if (condition) return 1;
fprintf(stderr, "FAIL: %s\n", message);
return 0;
}
int main(void) {
CPUState cpu;
memset(&cpu, 0, sizeof(cpu));
psx_icache_reset();
g_psx_icache_active = 1;
test_cycles = 0;
cpu.read_absorb_which = 1;
cpu.read_absorb[0] = 77u;
cpu.read_absorb[1] = 99u;
psx_icache_fetch_interp(&cpu, 0x80010000u);
if (!expect(test_cycles == 7u, "word-0 cold refill cost")) return 1;
if (!expect(cpu.read_absorb_which == 0u && cpu.read_absorb[1] == 0u &&
cpu.read_absorb[0] == 77u,
"true miss clears only the selected load give-back")) return 1;
cpu.read_absorb_which = 1;
cpu.read_absorb[1] = 55u;
psx_icache_fetch_interp(&cpu, 0x80010000u);
psx_icache_fetch_interp(&cpu, 0x80010004u);
if (!expect(test_cycles == 7u, "steady tag hits are cycle-free")) return 1;
if (!expect(cpu.read_absorb_which == 1u && cpu.read_absorb[1] == 55u,
"tag hits preserve load give-back")) return 1;
psx_icache_reset();
g_psx_icache_active = 1;
test_cycles = 0;
psx_icache_fetch_interp(&cpu, 0x80010008u);
if (!expect(test_cycles == 5u, "word-2 partial refill cost")) return 1;
psx_icache_fetch_interp(&cpu, 0x80010000u);
if (!expect(test_cycles == 12u, "earlier word remains invalid after partial refill")) return 1;
psx_icache_reset();
g_psx_icache_active = 1;
test_cycles = 0;
psx_icache_fetch_interp(&cpu, 0xA0010000u);
psx_icache_fetch_interp(&cpu, 0xA0010000u);
if (!expect(test_cycles == 8u, "KSEG1 remains uncached")) return 1;
psx_icache_reset();
g_psx_icache_active = 1;
test_cycles = 0;
psx_icache_fetch_interp(&cpu, 0x80010000u);
psx_icache_fetch_interp(&cpu, 0x00010000u);
if (!expect(test_cycles == 14u,
"KUSEG/KSEG0 aliases replace full virtual tags")) return 1;
psx_icache_reset();
if (!expect(g_psx_icache_tv[0] == 1u,
"reset makes a warmed line cold")) return 1;
g_psx_icache_active = 1;
test_cycles = 0;
psx_icache_fetch(&cpu, 0x80010000u);
if (!expect(test_cycles == 7u,
"compiled-code compatibility wrapper preserves refill")) return 1;
psx_icache_reset();
g_psx_icache_active = 1;
test_cycles = 0;
g_ls_replay_active = 1;
cpu.read_absorb_which = 1;
cpu.read_absorb[1] = 44u;
psx_icache_fetch_interp(&cpu, 0x80010000u);
g_ls_replay_active = 0;
if (!expect(test_cycles == 0u, "lockstep replay does not mutate cache")) return 1;
if (!expect(g_psx_icache_tv[0] == 1u, "replay preserved cold tag")) return 1;
if (!expect(cpu.read_absorb_which == 1u && cpu.read_absorb[1] == 44u,
"replay preserves load give-back")) return 1;
g_psx_icache_active = 0;
psx_icache_fetch_interp(&cpu, 0x80010000u);
if (!expect(test_cycles == 0u, "disabled cache model is free")) return 1;
if (!expect(cpu.read_absorb_which == 1u && cpu.read_absorb[1] == 44u,
"disabled cache preserves load give-back")) return 1;
puts("PASS: interpreter I-cache hit path preserves miss/refill semantics");
return 0;
}
tools/compile_overlays.py
+366 -89
View File
@@ -23,6 +23,7 @@ Each DLL exports:
import argparse
import base64
import binascii
import hashlib
from collections import Counter, deque
from concurrent.futures import ThreadPoolExecutor, as_completed
import os
@@ -32,6 +33,7 @@ import subprocess
import sys
import tempfile
import threading
from typing import Optional
class _ThreadLocalStdout:
@@ -245,6 +247,59 @@ def overlay_ext() -> str:
return '.dll' if is_windows() else '.so'
def bundle_path_is_immutable(dll_path: str) -> bool:
"""True for bound <region>_<logical>_<artifact> shared-library names."""
name = os.path.basename(dll_path)
return re.fullmatch(
rf'[0-9A-Fa-f]{{8}}_[0-9A-Fa-f]{{8}}_[0-9A-Fa-f]{{8}}'
rf'{re.escape(overlay_ext())}', name) is not None
def cached_bundle_pairs(logical_dll: str):
"""Yield legacy and immutable pairs for one region/logical cache identity."""
stem, ext = os.path.splitext(logical_dll)
directory = os.path.dirname(logical_dll) or '.'
base = os.path.basename(stem)
legacy_ranges = stem + '.ranges'
if os.path.isfile(logical_dll) and os.path.isfile(legacy_ranges):
yield logical_dll, legacy_ranges
try:
names = os.listdir(directory)
except OSError:
return
pattern = re.compile(rf'^{re.escape(base)}_[0-9A-Fa-f]{{8}}{re.escape(ext)}$')
for name in names:
if not pattern.fullmatch(name):
continue
dll = os.path.join(directory, name)
ranges = os.path.splitext(dll)[0] + '.ranges'
if os.path.isfile(ranges):
yield dll, ranges
def validated_cached_bundle_ids(logical_dll: str, data: bytes,
load_addr: int, size: int):
"""Yield every artifact for a logical identity with strict validation.
The caller deliberately sees all same-logical artifacts: immutable repair
and richer additive captures must not depend on directory enumeration order.
"""
for cached_dll, ranges_path in cached_bundle_pairs(logical_dll):
func_ids = parse_overlay_func_ids(
ranges_path, data, load_addr, size, require_stored_crc=True)
delay_errors = audit_func_id_delay_slots(func_ids, data, load_addr)
yield cached_dll, ranges_path, func_ids, delay_errors
def authoritative_cached_bundle_ids(logical_dll: str, data: bytes,
load_addr: int, size: int):
"""Yield only artifact-bound bundles allowed to suppress a rebuild."""
for bundle in validated_cached_bundle_ids(
logical_dll, data, load_addr, size):
if bundle_path_is_immutable(bundle[0]):
yield bundle
def native_path(p: str) -> str:
"""Absolute form of `p`, in the RUNNING interpreter's own path flavor, for
handing to native tools (gcc, tcc). Two hard-won rules live here
@@ -1384,7 +1439,7 @@ def generate_overlay_dispatch(variants: list) -> str:
# ---------------------------------------------------------------------------
def parse_overlay_func_ids(src_path: str, data: bytes, load_addr: int,
size: int) -> list:
size: int, require_stored_crc: bool = False) -> list:
"""Parse the recompiler's _full.ranges manifest and return the list of
in-overlay function identities as (ev, code_crc, ranges) tuples, where
ev = virtual entry, code_crc = AUTHORITATIVE hash of the captured code-range
@@ -1405,50 +1460,171 @@ def parse_overlay_func_ids(src_path: str, data: bytes, load_addr: int,
return ov_lo <= (a & 0x1FFFFFFF) < ov_hi
# Parse the recompiler manifest into [(entry, [(lo, len), ...]), ...].
funcs: list[tuple[int, list[tuple[int, int]]]] = []
funcs: list[tuple[int, Optional[int], list[tuple[int, int]]]] = []
cur = None
invalid = False
with open(src_path) as f:
for line in f:
s = line.split()
if not s:
if not s or s[0].startswith('#'):
continue
if s[0] == 'F':
try:
if len(s) not in (2, 3):
raise ValueError
addr = int(s[1], 16)
stored_crc = int(s[2], 16) if len(s) == 3 else None
if require_stored_crc and stored_crc is None:
raise ValueError
if (require_stored_crc and
(addr & 0xFFE00000) != 0x80000000):
raise ValueError
except (IndexError, ValueError):
cur = None
continue
invalid = True
break
if in_ov(addr):
cur = (addr, [])
if any((old_addr & 0x1FFFFFFF) ==
(addr & 0x1FFFFFFF) for old_addr, _, _ in funcs):
invalid = True
break
cur = (addr, stored_crc, [])
funcs.append(cur)
else:
if require_stored_crc:
invalid = True
break
cur = None
elif s[0] == 'R' and cur is not None:
try:
if len(s) != 3:
raise ValueError
lo, length = int(s[1], 16), int(s[2], 16)
if (require_stored_crc and
(lo & 0xFFE00000) != 0x80000000):
raise ValueError
except (IndexError, ValueError):
continue
cur[1].append((lo, length))
invalid = True
break
cur[2].append((lo, length))
if len(cur[2]) > 16:
invalid = True
break
elif s[0] == 'R' and cur is None:
if require_stored_crc:
invalid = True
break
else:
invalid = True
break
if invalid:
return []
out = []
for entry, ranges in funcs:
for entry, stored_crc, ranges in funcs:
if not ranges:
continue
return []
crc = 0
ok = True
normalized = []
entry_phys = entry & 0x1FFFFFFF
for lo, length in ranges:
off = (lo & 0x1FFFFFFF) - ov_lo
phys = lo & 0x1FFFFFFF
off = phys - ov_lo
if ((phys | length) & 3) or length < 4:
return []
if off < 0 or off + length > len(data):
ok = False # range outside captured bytes — can't hash reliably
break
return []
crc = binascii.crc32(data[off:off + length], crc)
if not ok:
continue
ev = (entry & 0x1FFFFFFF) | 0x80000000
out.append((ev, crc & 0xFFFFFFFF, ranges))
hi = phys + length
if any(phys < old_hi and old_lo < hi
for old_lo, old_hi in normalized):
return []
normalized.append((phys, hi))
if not any(lo <= entry_phys and entry_phys + 4 <= hi
for lo, hi in normalized):
return []
crc &= 0xFFFFFFFF
if stored_crc is not None and stored_crc != crc:
return []
ev = entry_phys | 0x80000000
out.append((ev, crc, ranges))
return out
def _mips_control_kind(instr: int) -> int:
"""1=supported delayed transfer, 0=ordinary, -1=reserved/unsupported."""
op = (instr >> 26) & 0x3F
if op == 0:
return int((instr & 0x3F) in (0x08, 0x09)) # JR/JALR
if op == 0x01:
return 1 if ((instr >> 16) & 0x1F) in (0x00, 0x01, 0x10, 0x11) else -1
if op in (0x02, 0x03) or 0x04 <= op <= 0x07:
return 1
if 0x14 <= op <= 0x17:
return -1
if 0x10 <= op <= 0x13 and ((instr >> 21) & 0x1F) == 0x08:
return -1
return 0
def audit_func_id_delay_slots(func_ids: list, data: bytes,
load_addr: int) -> list:
"""Return unsafe emitted dependency errors for exact function identities.
Every native delay slot must exist in this coherent capture and inside the
same function's hashed range union. This packaging audit is intentionally
redundant with the recompiler and loader: it prevents an unsafe DLL/ranges
pair from being published or reused even when one layer is stale.
"""
errors = []
base = load_addr & 0x1FFFFFFF
for entry, _crc, ranges in func_ids:
if len(ranges) > 16:
errors.append((entry, None,
f'{len(ranges)} ranges exceed loader limit 16'))
continue
normalized = []
malformed = False
for lo, length in ranges:
phys = lo & 0x1FFFFFFF
off = phys - base
if (phys & 3) or (length & 3) or length < 4 or \
off < 0 or off + length > len(data):
errors.append((entry, phys | 0x80000000,
'malformed/out-of-capture code range'))
malformed = True
break
normalized.append((phys, phys + length))
if malformed:
continue
def contains_word(addr: int) -> bool:
return any(lo <= addr and addr + 4 <= hi
for lo, hi in normalized)
controls = set()
for lo, hi in normalized:
for pc in range(lo, hi, 4):
off = pc - base
instr = int.from_bytes(data[off:off + 4], 'little')
kind = _mips_control_kind(instr)
if kind < 0:
errors.append((entry, pc | 0x80000000,
'reserved/unsupported branch encoding'))
elif kind > 0:
controls.add(pc)
if not contains_word(pc + 4):
errors.append((entry, pc | 0x80000000,
f'delay slot 0x{(pc + 4) | 0x80000000:08X} '
'is not identity-hashed'))
delay_pcs = {pc + 4 for pc in controls}
nested = sorted(controls & delay_pcs)
for pc in nested:
errors.append((entry, pc | 0x80000000,
'control transfer in a delay slot'))
return errors
def write_overlay_ranges_from(func_ids: list, out_path: str) -> int:
"""Write the {phys}_{key}.ranges manifest (v2) from a func-id list produced by
parse_overlay_func_ids. Returns the number of functions written.
@@ -1474,7 +1650,8 @@ def write_overlay_ranges(src_path: str, out_path: str,
parse_overlay_func_ids(src_path, data, load_addr, size), out_path)
def load_region_coverage(cache_dir: str, phys_addr: int) -> set:
def load_region_coverage(cache_dir: str, phys_addr: int, data: bytes,
load_addr: int, size: int) -> set:
"""Set of (ev, code_crc) function identities already provided by built DLLs
for this region_start. The loader content-matches per function across ALL
DLLs sharing a region_start, so a function is "covered" as soon as ANY
@@ -1490,17 +1667,21 @@ def load_region_coverage(cache_dir: str, phys_addr: int) -> set:
for name in names:
if not (name.startswith(prefix) and name.endswith('.ranges')):
continue
try:
with open(os.path.join(cache_dir, name)) as f:
for ln in f:
p = ln.split()
if len(p) >= 3 and p[0] == 'F':
try:
covered.add((int(p[1], 16), int(p[2], 16)))
except ValueError:
pass
except OSError:
pass
# A sidecar is not a committed cache entry by itself. A process can
# stop between the two same-directory renames, or an antivirus/loader
# lock can reject one of them on Windows. Never let an orphan ranges
# file poison coverage and suppress a future rebuild.
dll_name = name[:-len('.ranges')] + overlay_ext()
if not bundle_path_is_immutable(dll_name):
continue
if not os.path.isfile(os.path.join(cache_dir, dll_name)):
continue
manifest = os.path.join(cache_dir, name)
func_ids = parse_overlay_func_ids(
manifest, data, load_addr, size, require_stored_crc=True)
if not func_ids or audit_func_id_delay_slots(func_ids, data, load_addr):
continue
covered.update((ev, crc) for ev, crc, _ranges in func_ids)
return covered
@@ -1515,7 +1696,8 @@ def _addr_in_func_ids(addr: int, func_ids: list) -> bool:
return False
def load_region_entry_set(cache_dir: str, phys_addr: int) -> set:
def load_region_entry_set(cache_dir: str, phys_addr: int, data: bytes,
load_addr: int, size: int) -> set:
"""Set of phys-normalized F-line ENTRY addresses provided by ALL built DLLs
(region + fragment) for this region_start. This — not range coverage — is
the dispatchability test: native code is enterable ONLY at F entries, so a
@@ -1533,17 +1715,17 @@ def load_region_entry_set(cache_dir: str, phys_addr: int) -> set:
for name in names:
if not (name.startswith(prefix) and name.endswith('.ranges')):
continue
try:
with open(os.path.join(cache_dir, name)) as f:
for ln in f:
p = ln.split()
if len(p) >= 2 and p[0] == 'F':
try:
out.add(int(p[1], 16) & 0x1FFFFFFF)
except ValueError:
pass
except OSError:
pass
dll_name = name[:-len('.ranges')] + overlay_ext()
if not bundle_path_is_immutable(dll_name):
continue
if not os.path.isfile(os.path.join(cache_dir, dll_name)):
continue
manifest = os.path.join(cache_dir, name)
func_ids = parse_overlay_func_ids(
manifest, data, load_addr, size, require_stored_crc=True)
if not func_ids or audit_func_id_delay_slots(func_ids, data, load_addr):
continue
out.update(ev & 0x1FFFFFFF for ev, _crc, _ranges in func_ids)
return out
@@ -1634,6 +1816,8 @@ def generate_interior_fragment_static(interior: int, data: bytes,
if audit['unknown_bad'] or audit['unsupported_todo_addrs']:
return None
func_ids = parse_overlay_func_ids(ranges_src, data, load_addr, size)
if audit_func_id_delay_slots(func_ids, data, load_addr):
return None
ids_by_addr = {}
for ev, code_crc, ranges in func_ids:
ids_by_addr.setdefault(ev, []).append((code_crc, ranges))
@@ -1748,6 +1932,12 @@ def compile_interior_fragment(interior: int, data: bytes, load_addr: int,
frag_ids = parse_overlay_func_ids(ranges_src, data, load_addr, size)
if not frag_ids:
return None, 'no-func-ids (empty ranges manifest)'
delay_errors = audit_func_id_delay_slots(frag_ids, data, load_addr)
if delay_errors:
entry, pc, reason = delay_errors[0]
where = f' at 0x{pc:08X}' if pc is not None else ''
return None, (f'delay-slot-identity-audit: func 0x{entry:08X}'
f'{where}: {reason}')
# Key the fragment DLL by its func-identity SET (dedup like a region
# bundle); the loader keys DLLs by the region_start filename prefix and
# content-matches each function, so a fragment is just another DLL for
@@ -1756,8 +1946,12 @@ def compile_interior_fragment(interior: int, data: bytes, load_addr: int,
struct.pack('<II', ev, crc)
for ev, crc, _ in sorted(frag_ids))) & 0xFFFFFFFF
dll_path = os.path.join(cache_dir, f'{phys_addr:08X}_{key:08X}{overlay_ext()}')
if os.path.exists(dll_path) and not args.force:
return frag_ids, 'cached' # already built
if not args.force:
for (_cached_dll, _ranges_path, cached_ids,
delay_errors) in authoritative_cached_bundle_ids(
dll_path, data, load_addr, len(data)):
if cached_ids and not delay_errors:
return cached_ids, 'cached'
patched_c = os.path.join(tmp, 'frag_patched.c')
with open(patched_c, 'w') as f:
f.write(src)
@@ -1944,16 +2138,16 @@ def compile_dll(c_path: str, out_dll: str, include_dirs: list[str],
def compile_and_publish_dll(c_path: str, out_dll: str, include_dirs: list[str],
func_ids, gcc: str = 'gcc', flavor: int = 0,
compiler: str = 'gcc', tcc: str = 'tcc'):
"""Compile privately, then atomically publish ranges followed by the DLL.
"""Compile privately, then publish an immutable DLL/ranges bundle.
Live runtime rescans may occur while a batch is still running. The DLL name
is therefore the commit marker: a scanner can see neither artifact, or a
complete ranges file plus a complete DLL, but never a half-linked library.
The final stem is content-addressed from both artifacts. Ranges are linked
into place first; the uniquely named DLL is the commit marker and is linked
last. A scanner therefore sees either no DLL or the exact sidecar that was
hashed with it, never a new DLL paired with an older mutable manifest.
"""
stem, ext = os.path.splitext(out_dll)
token = f'{os.getpid()}.{threading.get_ident()}'
tmp_dll = f'{stem}.{token}.tmp{ext}'
out_ranges = stem + '.ranges'
tmp_ranges = f'{stem}.{token}.tmp.ranges'
for path in (tmp_dll, tmp_ranges):
try:
@@ -1967,12 +2161,58 @@ def compile_and_publish_dll(c_path: str, out_dll: str, include_dirs: list[str],
nfunc = write_overlay_ranges_from(func_ids, tmp_ranges)
if not nfunc:
return False, 0
os.replace(tmp_ranges, out_ranges)
os.replace(tmp_dll, out_dll) # publication/commit marker, always last
with open(tmp_dll, 'rb') as f:
dll_bytes = f.read()
with open(tmp_ranges, 'rb') as f:
ranges_bytes = f.read()
digest = hashlib.sha256(
hashlib.sha256(dll_bytes).digest() + ranges_bytes).digest()
directory = os.path.dirname(out_dll)
logical = os.path.basename(stem)
published_dll = published_ranges = None
def file_equals(path: str, expected: bytes) -> bool:
try:
with open(path, 'rb') as f:
return f.read() == expected
except OSError:
return False
for attempt in range(256):
key_bytes = (digest[attempt * 4:attempt * 4 + 4]
if attempt < 8 else
hashlib.sha256(digest + attempt.to_bytes(4, 'little')).digest()[:4])
key = int.from_bytes(key_bytes, 'big')
final_stem = os.path.join(directory, f'{logical}_{key:08X}')
candidate_dll = final_stem + ext
candidate_ranges = final_stem + '.ranges'
if os.path.exists(candidate_dll) or os.path.exists(candidate_ranges):
if (file_equals(candidate_dll, dll_bytes) and
file_equals(candidate_ranges, ranges_bytes)):
published_dll = candidate_dll
published_ranges = candidate_ranges
break
continue
try:
os.link(tmp_ranges, candidate_ranges)
os.link(tmp_dll, candidate_dll) # immutable commit marker, always last
except FileExistsError:
if (file_equals(candidate_dll, dll_bytes) and
file_equals(candidate_ranges, ranges_bytes)):
published_dll = candidate_dll
published_ranges = candidate_ranges
break
continue
published_dll, published_ranges = candidate_dll, candidate_ranges
break
if not published_dll or not published_ranges:
raise OSError('could not allocate an immutable shard bundle name')
# Machine-readable live handoff. flush=True is essential because stdout
# is a pipe under the runtime and a completed shard must not wait in
# Python's block buffer until the whole batch exits.
print(f'PSX_SHARD_PUBLISHED {out_dll}', flush=True)
print(f'PSX_SHARD_PUBLISHED {published_dll}', flush=True)
return True, nfunc
except OSError as exc:
print(f' PUBLISH ERROR: {exc}')
@@ -2185,6 +2425,10 @@ def main():
static_entry_sources[entry] = (
data, load_addr, size, phys_addr)
if not args.static:
dll_path = os.path.join(
cache_dir, f'{phys_addr:08X}_{crc32:08X}{overlay_ext()}')
# Merge evidence from a prior build of the SAME bytes: every F entry
# in an existing ranges manifest for this exact image was proven
# compilable before, so a fresh (poorer) capture can't regress
@@ -2194,31 +2438,26 @@ def main():
# re-derives their interior/alias disposition — feeding them back as
# roots would truncate their hosts.
if not args.static:
ranges_name = f'{phys_addr:08X}_{crc32:08X}.ranges'
prior_ranges = os.path.join(cache_dir, ranges_name)
if os.path.exists(prior_ranges):
prior_entries = []
with open(prior_ranges) as pf:
for ln in pf:
parts = ln.split()
if parts and parts[0] == 'F':
try:
prior_entries.append(int(parts[1], 16))
except (IndexError, ValueError):
pass
if prior_entries:
fe = _parse_addr_list(cap.get('function_entry_pcs', []))
de = _parse_addr_list(cap.get('dispatch_entry_pcs', []))
for a in prior_entries:
if _callable_legacy_seed(data, load_addr, a):
fe.add(a)
else:
de.add(a)
cap['function_entry_pcs'] = sorted(fe)
cap['dispatch_entry_pcs'] = sorted(de)
cap.setdefault('schema', 'merged')
print(f' merged {len(prior_entries)} prior-manifest entries '
f'from {prior_ranges}')
prior_entries = set()
for (_prior_dll, prior_ranges, prior_ids,
prior_errors) in validated_cached_bundle_ids(
dll_path, data, load_addr, size):
if not prior_ids or prior_errors:
continue
prior_entries.update(ev for ev, _crc, _ranges in prior_ids)
if prior_entries:
fe = _parse_addr_list(cap.get('function_entry_pcs', []))
de = _parse_addr_list(cap.get('dispatch_entry_pcs', []))
for a in prior_entries:
if _callable_legacy_seed(data, load_addr, a):
fe.add(a)
else:
de.add(a)
cap['function_entry_pcs'] = sorted(fe)
cap['dispatch_entry_pcs'] = sorted(de)
cap.setdefault('schema', 'merged')
print(f' merged {len(prior_entries)} validated prior-manifest '
'entries from additive bundles')
seeds, seed_audit = classify_overlay_seeds(cap, data, load_addr, size,
crc32, toml)
@@ -2242,9 +2481,6 @@ def main():
interior_frag_jobs.append((phys_addr, load_addr, size, data,
_interiors | _disp_roots, _executed))
if not args.static:
dll_path = os.path.join(cache_dir, f'{phys_addr:08X}_{crc32:08X}{overlay_ext()}')
print(f'Overlay load=0x{load_addr:08X} size={size} crc32=0x{crc32:08X}')
if args.static:
print(f' seeds: {len(seeds)} mode: static -> {static_out}')
@@ -2258,10 +2494,32 @@ def main():
stats.add_skip()
return
if not args.static and os.path.exists(dll_path) and not args.force:
print(' SKIP: DLL already exists (use --force to recompile)\n')
stats.add_skip()
return
if not args.static and not args.force:
required_entries = {
int(seed.split()[-1], 16) & 0xFFFFFFFF
for seed in root_seeds
}
cached_entries = set()
for (_cached_dll, ranges_path, existing_ids,
existing_delay_errors) in authoritative_cached_bundle_ids(
dll_path, data, load_addr, size):
if existing_ids and not existing_delay_errors:
cached_entries.update(
ev for ev, _crc, _ranges in existing_ids)
continue
reason = (existing_delay_errors[0][2]
if existing_delay_errors else 'empty/unverifiable manifest')
print(f' STALE/UNSAFE existing pair: {reason}; rebuilding fail-closed')
if required_entries and required_entries <= cached_entries:
print(' SKIP: additive shard bundles cover every requested entry '
'(use --force to recompile)\n')
stats.add_skip()
return
if cached_entries:
missing = sorted(required_entries - cached_entries)
print(' INCOMPLETE existing bundles: missing requested entries '
+ ', '.join(f'0x{entry:08X}' for entry in missing)
+ '; rebuilding additively')
with tempfile.TemporaryDirectory() as tmp:
# Write fake PS-EXE. The header entry PC becomes a walk root in the
@@ -2348,6 +2606,15 @@ def main():
func_ids = parse_overlay_func_ids(ranges_src, data,
load_addr, size)
delay_errors = audit_func_id_delay_slots(
func_ids, data, load_addr)
if delay_errors:
entry, pc, reason = delay_errors[0]
where = f' at 0x{pc:08X}' if pc is not None else ''
print(f' STATIC DELAY-SLOT IDENTITY AUDIT FAILED: '
f'func 0x{entry:08X}{where}: {reason}\n')
stats.add_fail(_label, 'delay_slot_identity', reason)
return
ids_by_addr = {}
for ev, code_crc, ranges in func_ids:
ids_by_addr.setdefault(ev, []).append((code_crc, ranges))
@@ -2421,12 +2688,22 @@ def main():
# an endless pile of redundant DLLs.
this_ids = (parse_overlay_func_ids(ranges_src, data, load_addr, size)
if ranges_src else [])
delay_errors = audit_func_id_delay_slots(
this_ids, data, load_addr)
if delay_errors:
entry, pc, reason = delay_errors[0]
where = f' at 0x{pc:08X}' if pc is not None else ''
print(f' DELAY-SLOT IDENTITY AUDIT FAILED: '
f'func 0x{entry:08X}{where}: {reason}\n')
stats.add_fail(_label, 'delay_slot_identity', reason)
return
this_set = {(ev, crc) for ev, crc, _ in this_ids}
with cov_lock:
covered = region_coverage_cache.get(phys_addr)
if covered is None:
covered = load_region_coverage(cache_dir, phys_addr)
covered = load_region_coverage(
cache_dir, phys_addr, data, load_addr, size)
region_coverage_cache[phys_addr] = covered
fully_covered = (bool(this_set) and this_set <= covered
and not args.force)
@@ -2451,8 +2728,7 @@ def main():
compiler=args.compiler, tcc=args.tcc)
if success:
if this_ids:
print(f' ranges: {nfn} functions -> '
f'{os.path.splitext(dll_path)[0]}.ranges')
print(f' ranges: {nfn} functions in immutable shard bundle')
# New identities are now available for this region_start;
# keep the warm coverage set current so later captures in
# this same run dedup against them. (Parallel note: the
@@ -2463,7 +2739,7 @@ def main():
# DLLs at a region.)
with cov_lock:
covered |= this_set
print(f' OK -> {dll_path}\n')
print(' OK -> immutable shard bundle published\n')
stats.add_ok()
else:
print(' WARNING: recompiler emitted no _full.ranges — '
@@ -2511,7 +2787,8 @@ def main():
# range" does NOT make a dispatch target servable — a range-
# covered PC with no F entry anywhere still interps its whole
# chain on every dispatch. Demand an entry at exactly this PC.
covered_entries = load_region_entry_set(cache_dir, phys_addr)
covered_entries = load_region_entry_set(
cache_dir, phys_addr, data, load_addr, size)
orphans = sorted(a for a in interior_pcs
if (a in executed or a in forced_interiors)
and (a & 0x1FFFFFFF) not in covered_entries)
tools/test_compile_overlays_additive.py
+269 -9
View File
@@ -27,7 +27,149 @@ def region(load_addr, payload, executed=(), dispatch=(), functions=(), seeds=Non
class AdditiveCaptureTests(unittest.TestCase):
def test_live_dll_publication_uses_private_output_and_commits_dll_last(self):
def test_cached_bundle_pairs_preserves_logical_key_for_immutable_names(self):
with tempfile.TemporaryDirectory() as td:
logical = Path(td) / "00010000_DEADBEEF.dll"
legacy_ranges = logical.with_suffix(".ranges")
logical.write_bytes(b"legacy")
legacy_ranges.write_text("legacy", encoding="ascii")
immutable = Path(td) / "00010000_DEADBEEF_13579BDF.dll"
immutable_ranges = immutable.with_suffix(".ranges")
immutable.write_bytes(b"immutable")
immutable_ranges.write_text("immutable", encoding="ascii")
# A different logical key must not be treated as the same cache
# identity merely because it shares the region prefix.
other = Path(td) / "00010000_FEEDFACE_2468ACE0.dll"
other.write_bytes(b"other")
other.with_suffix(".ranges").write_text("other", encoding="ascii")
pairs = set(compile_overlays.cached_bundle_pairs(str(logical)))
self.assertEqual(pairs, {
(str(logical), str(legacy_ranges)),
(str(immutable), str(immutable_ranges)),
})
def test_validated_same_logical_artifacts_union_additive_entries(self):
with tempfile.TemporaryDirectory() as td:
load = 0x80010000
payload = b"\0" * 8
crc = compile_overlays.binascii.crc32(b"\0" * 4) & 0xFFFFFFFF
logical = Path(td) / "00010000_DEADBEEF.dll"
entries = set()
for artifact, entry in (("11111111", load),
("22222222", load + 4)):
dll = Path(td) / f"00010000_DEADBEEF_{artifact}.dll"
dll.write_bytes(b"dll")
dll.with_suffix(".ranges").write_text(
"# psxrecomp overlay code-range manifest v2\n"
f"F {entry:08X} {crc:08X}\nR {entry:08X} 4\n",
encoding="ascii")
bundles = list(compile_overlays.validated_cached_bundle_ids(
str(logical), payload, load, len(payload)))
self.assertEqual(len(bundles), 2)
for _dll, _ranges, func_ids, errors in bundles:
self.assertFalse(errors)
entries.update(ev for ev, _crc, _func_ranges in func_ids)
self.assertEqual(entries, {load, load + 4})
def test_legacy_pair_is_seed_evidence_not_authoritative_coverage(self):
with tempfile.TemporaryDirectory() as td:
load = 0x80010000
payload = b"\0" * 4
crc = compile_overlays.binascii.crc32(payload) & 0xFFFFFFFF
logical = Path(td) / "00010000_DEADBEEF.dll"
logical.write_bytes(b"old-unbound-dll")
logical.with_suffix(".ranges").write_text(
"# psxrecomp overlay code-range manifest v2\n"
f"F {load:08X} {crc:08X}\nR {load:08X} 4\n",
encoding="ascii")
seed_evidence = list(
compile_overlays.validated_cached_bundle_ids(
str(logical), payload, load, len(payload)))
authoritative = list(
compile_overlays.authoritative_cached_bundle_ids(
str(logical), payload, load, len(payload)))
self.assertEqual(len(seed_evidence), 1)
self.assertTrue(seed_evidence[0][2])
self.assertEqual(authoritative, [])
def test_delay_slot_identity_audit_requires_cross_page_slot(self):
load = 0x80010FF0
# NOPs through FF8, BEQ at FFC, ADDIU delay slot at 1000.
payload = (b"\0" * 12 +
(0x10800008).to_bytes(4, "little") +
(0x24020001).to_bytes(4, "little"))
safe = [(load, 0, [(load, len(payload))])]
unsafe = [(load, 0, [(load, len(payload) - 4)])]
self.assertEqual(
compile_overlays.audit_func_id_delay_slots(safe, payload, load), [])
errors = compile_overlays.audit_func_id_delay_slots(
unsafe, payload[:-4], load)
self.assertEqual(len(errors), 1)
self.assertEqual(errors[0][1], 0x80010FFC)
self.assertIn("0x80011000", errors[0][2])
def test_delay_slot_identity_audit_rejects_nested_control_flow(self):
load = 0x80020000
# BEQ followed by JR in its delay slot; both words are present, but the
# interpreter treats control flow in a delay slot as unsupported.
payload = ((0x10000001).to_bytes(4, "little") +
(0x03E00008).to_bytes(4, "little") +
(0x00000000).to_bytes(4, "little"))
ids = [(load, 0, [(load, len(payload))])]
errors = compile_overlays.audit_func_id_delay_slots(ids, payload, load)
self.assertTrue(any("control transfer in a delay slot" in e[2]
for e in errors))
def test_delay_slot_identity_audit_rejects_reserved_branch_likely(self):
load = 0x80020000
payload = ((0x50800001).to_bytes(4, "little") +
(0x00000000).to_bytes(4, "little"))
ids = [(load, 0, [(load, len(payload))])]
errors = compile_overlays.audit_func_id_delay_slots(ids, payload, load)
self.assertTrue(any("reserved/unsupported" in e[2] for e in errors))
def test_published_manifest_parse_is_all_or_nothing(self):
with tempfile.TemporaryDirectory() as td:
load = 0x80010000
payload = b"\0" * 8
crc = compile_overlays.binascii.crc32(payload) & 0xFFFFFFFF
manifest = Path(td) / "bundle.ranges"
manifest.write_text(
f"F {load:08X} {crc:08X}\nR {load:08X} 8\n",
encoding="ascii")
parsed = compile_overlays.parse_overlay_func_ids(
str(manifest), payload, load, len(payload),
require_stored_crc=True)
self.assertEqual(len(parsed), 1)
manifest.write_text(
f"F {load + 0x20000000:08X} {crc:08X}\n"
f"R {load + 0x20000000:08X} 8\n",
encoding="ascii")
self.assertEqual(compile_overlays.parse_overlay_func_ids(
str(manifest), payload, load, len(payload),
require_stored_crc=True), [])
manifest.write_text(
f"F {load:08X}\nR {load:08X} 8\n", encoding="ascii")
self.assertEqual(compile_overlays.parse_overlay_func_ids(
str(manifest), payload, load, len(payload),
require_stored_crc=True), [])
manifest.write_text(
f"F {load:08X} {crc:08X}\nR {load:08X} 8\n"
f"F {load + 4:08X} 00000000\nR {load + 8:08X} 4\n",
encoding="ascii")
self.assertEqual(compile_overlays.parse_overlay_func_ids(
str(manifest), payload, load, len(payload),
require_stored_crc=True), [])
def test_live_dll_publication_is_immutable_and_dll_commits_last(self):
with tempfile.TemporaryDirectory() as td:
out_dll = Path(td) / "00010000_DEADBEEF.dll"
source = Path(td) / "source.c"
@@ -43,26 +185,144 @@ class AdditiveCaptureTests(unittest.TestCase):
observed.append(private)
return True
real_replace = compile_overlays.os.replace
real_link = compile_overlays.os.link
def observe_replace(source_path, dest_path):
def observe_link(source_path, dest_path):
observed.append(Path(dest_path))
real_replace(source_path, dest_path)
real_link(source_path, dest_path)
with mock.patch.object(compile_overlays, "compile_dll",
side_effect=fake_compile), \
mock.patch.object(compile_overlays.os, "replace",
side_effect=observe_replace):
mock.patch.object(compile_overlays.os, "link",
side_effect=observe_link):
ok, count = compile_overlays.compile_and_publish_dll(
str(source), str(out_dll), [], func_ids)
self.assertTrue(ok)
self.assertEqual(count, 1)
self.assertEqual(out_dll.read_bytes(), b"complete-dll")
self.assertTrue(out_dll.with_suffix(".ranges").exists())
self.assertEqual(observed[-1], out_dll)
published = list(Path(td).glob("00010000_*.dll"))
self.assertEqual(len(published), 1)
self.assertNotEqual(published[0], out_dll)
self.assertRegex(
published[0].name,
r"^00010000_DEADBEEF_[0-9A-F]{8}\.dll$")
self.assertEqual(published[0].read_bytes(), b"complete-dll")
self.assertTrue(published[0].with_suffix(".ranges").exists())
self.assertEqual(observed[-2:],
[published[0].with_suffix(".ranges"), published[0]])
self.assertFalse(list(Path(td).glob("*.tmp.*")))
def test_live_publication_first_link_failure_leaves_no_pair(self):
with tempfile.TemporaryDirectory() as td:
out_dll = Path(td) / "00010000_DEADBEEF.dll"
source = Path(td) / "source.c"
source.write_text("/* test */", encoding="utf-8")
func_ids = [(0x80010000, 0x12345678, [(0x80010000, 4)])]
def fake_compile(_source, private_dll, _includes, **_kwargs):
Path(private_dll).write_bytes(b"complete-dll")
return True
with mock.patch.object(compile_overlays, "compile_dll",
side_effect=fake_compile), \
mock.patch.object(compile_overlays.os, "link",
side_effect=OSError("simulated DLL lock")):
ok, count = compile_overlays.compile_and_publish_dll(
str(source), str(out_dll), [], func_ids)
self.assertFalse(ok)
self.assertEqual(count, 0)
self.assertFalse(list(Path(td).glob("00010000_*.dll")))
self.assertFalse(list(Path(td).glob("00010000_*.ranges")))
def test_live_publication_second_link_failure_leaves_only_ignored_ranges(self):
with tempfile.TemporaryDirectory() as td:
out_dll = Path(td) / "00010000_DEADBEEF.dll"
source = Path(td) / "source.c"
source.write_text("/* test */", encoding="utf-8")
func_ids = [(0x80010000, 0x12345678, [(0x80010000, 4)])]
real_link = compile_overlays.os.link
calls = 0
def fake_compile(_source, private_dll, _includes, **_kwargs):
Path(private_dll).write_bytes(b"complete-dll")
return True
def fail_second_link(source_path, dest_path):
nonlocal calls
calls += 1
if calls == 2:
raise OSError("simulated ranges lock")
real_link(source_path, dest_path)
with mock.patch.object(compile_overlays, "compile_dll",
side_effect=fake_compile), \
mock.patch.object(compile_overlays.os, "link",
side_effect=fail_second_link):
ok, count = compile_overlays.compile_and_publish_dll(
str(source), str(out_dll), [], func_ids)
self.assertFalse(ok)
self.assertEqual(count, 0)
self.assertFalse(list(Path(td).glob("00010000_*.dll")))
self.assertEqual(len(list(Path(td).glob("00010000_*.ranges"))), 1)
self.assertEqual(compile_overlays.load_region_coverage(
td, 0x00010000, b"\0" * 4, 0x80010000, 4), set())
def test_coverage_and_entries_ignore_orphan_ranges(self):
with tempfile.TemporaryDirectory() as td:
load = 0x80010000
payload = b"\0" * 4
crc = compile_overlays.binascii.crc32(payload) & 0xFFFFFFFF
ranges = Path(td) / "00010000_DEADBEEF_13579BDF.ranges"
ranges.write_text(
"# psxrecomp overlay code-range manifest v2\n"
f"F 80010000 {crc:08X}\nR 80010000 4\n",
encoding="utf-8")
self.assertEqual(compile_overlays.load_region_coverage(
td, 0x00010000, payload, load, len(payload)), set())
self.assertEqual(compile_overlays.load_region_entry_set(
td, 0x00010000, payload, load, len(payload)), set())
# Legacy DLL/ranges pairs predate bound publication and may be a
# strict new manifest beside an old DLL after a crash. They remain
# seed evidence, but cannot suppress the first immutable rebuild.
legacy = Path(td) / "00010000_DEADBEEF.ranges"
legacy.write_text(
"# psxrecomp overlay code-range manifest v2\n"
f"F 80010000 {crc:08X}\nR 80010000 4\n",
encoding="utf-8")
legacy.with_suffix(compile_overlays.overlay_ext()).write_bytes(b"old-dll")
self.assertFalse(compile_overlays.bundle_path_is_immutable(
str(legacy.with_suffix(compile_overlays.overlay_ext()))))
self.assertEqual(compile_overlays.load_region_coverage(
td, 0x00010000, payload, load, len(payload)), set())
self.assertEqual(compile_overlays.load_region_entry_set(
td, 0x00010000, payload, load, len(payload)), set())
ranges.with_suffix(compile_overlays.overlay_ext()).write_bytes(b"dll")
self.assertEqual(
compile_overlays.load_region_coverage(
td, 0x00010000, payload, load, len(payload)),
{(0x80010000, crc)})
self.assertEqual(
compile_overlays.load_region_entry_set(
td, 0x00010000, payload, load, len(payload)),
{0x00010000})
# A paired manifest with a valid-looking prefix and malformed tail
# must contribute nothing. Otherwise it can poison additive
# coverage and suppress publication of its own valid repair.
ranges.write_text(
"# psxrecomp overlay code-range manifest v2\n"
f"F 80010000 {crc:08X}\nR 80010000 4\n"
"TRAILING GARBAGE\n",
encoding="utf-8")
self.assertEqual(compile_overlays.load_region_coverage(
td, 0x00010000, payload, load, len(payload)), set())
self.assertEqual(compile_overlays.load_region_entry_set(
td, 0x00010000, payload, load, len(payload)), set())
def test_unions_evidence_but_preserves_reused_address_variants(self):
with tempfile.TemporaryDirectory() as td:
base = Path(td) / "overlay_captures.json"