diff --git a/.gitignore b/.gitignore index b67a0585..cdd34f56 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ sdl2-msvc/ # Diagnostic dumps & smoke logs (transient) *.log *.jsonl +/.aot_capture_history/ # Build-generated codegen-hash header (hash_codegen.cmake): folds an emitter # content hash into the overlay cache path; regenerated each build, never committed. diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index e5182101..3eb87c84 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -116,6 +116,24 @@ executed_pcs[], dispatch_entry_pcs[], function_entry_pcs[], seeds[] (`g_dirty_ram_pc_table`, `g_dirty_ram_exec_pc_table`) — **execution-verified**, not disassembly guesses. +#### Durable capture history (opt-in) + +`overlay_captures.json` remains the canonical latest snapshot consumed by the live +compiler, but it is published through a same-directory temporary file plus atomic +replace so readers never observe a half-written manifest. Games may additionally +set `overlay_capture_history = true` to append each changed coherent snapshot as +an independent JSONL record to `overlay_captures.addendum.jsonl` beside the +executable. A hard kill can damage only the last line; the next append quarantines +that tail, and `tools/coverage_vault.py merge --addendum ...` ignores malformed +lines while additively merging every valid record. + +Development configs may also set a project-relative +`overlay_capture_persist_dir`. This creates one atomically published, immutable +JSON snapshot per changed capture. Absolute paths and `..` components are rejected. +Production configs normally omit this key and retain only the naive executable- +local addendum. Both history modes are off by default and contain game-derived +bytes, so their outputs remain private/gitignored artifacts. + ### 1.3 Compile (offline, `tools/compile_overlays.py`) 1. Python does function-boundary discovery from the seeds (`classify_overlay_seeds`, `_walk_overlay_function`) with a strict callable-prologue gate; interior/ diff --git a/recompiler/src/config_loader.cpp b/recompiler/src/config_loader.cpp index bfa105a9..77493ed0 100644 --- a/recompiler/src/config_loader.cpp +++ b/recompiler/src/config_loader.cpp @@ -234,6 +234,25 @@ static RuntimeConfig parse_runtime_block(const toml::value& cfg, const fs::path& if (runtime.contains("overlay_cache")) { rt.overlay_cache = toml::find(runtime, "overlay_cache"); } + if (runtime.contains("overlay_capture_history")) { + rt.overlay_capture_history = + toml::find(runtime, "overlay_capture_history"); + } + if (runtime.contains("overlay_capture_persist_dir")) { + rt.overlay_capture_persist_dir = + toml::find(runtime, "overlay_capture_persist_dir"); + const std::filesystem::path persist(rt.overlay_capture_persist_dir); + if (persist.is_absolute()) { + throw std::runtime_error( + "runtime.overlay_capture_persist_dir must be project-relative"); + } + for (const auto& component : persist) { + if (component == "..") { + throw std::runtime_error( + "runtime.overlay_capture_persist_dir must stay inside the project"); + } + } + } if (runtime.contains("turbo_loads")) { rt.turbo_loads = toml::find(runtime, "turbo_loads"); } diff --git a/recompiler/src/config_loader.h b/recompiler/src/config_loader.h index d90e1017..cdb4e114 100644 --- a/recompiler/src/config_loader.h +++ b/recompiler/src/config_loader.h @@ -138,6 +138,19 @@ struct RuntimeConfig { // overlay bytes to overlay_captures.json for offline compilation. bool overlay_cache = false; + // overlay_capture_history: opt-in durable capture history. The runtime + // keeps overlay_captures.json as an atomic latest snapshot for the live + // compiler and additionally appends every changed coherent snapshot to + // overlay_captures.addendum.jsonl beside the executable. A malformed tail + // from a hard kill cannot destroy earlier records. + bool overlay_capture_history = false; + + // overlay_capture_persist_dir: optional DEV-only, project-relative safe + // directory for one immutable JSON file per changed snapshot. Absolute + // paths and `..` components are rejected. Production normally leaves this + // unset and retains only the addendum beside the executable. + std::string overlay_capture_persist_dir; + // turbo_loads: OPT-IN per game. While the game is loading (CD data // stream active, XA/FMV excluded, post-BIOS-handoff only) the frontend // skips wall-clock pacing so the guest runs at host speed — compressing diff --git a/recompiler/tests/recompiler_patch_test.cpp b/recompiler/tests/recompiler_patch_test.cpp index e1546e89..acfe0f64 100644 --- a/recompiler/tests/recompiler_patch_test.cpp +++ b/recompiler/tests/recompiler_patch_test.cpp @@ -166,6 +166,37 @@ replacement = "0x24020001" "instruction-aligned", "parser rejects unaligned sites"); } +void capture_history_config_tests(const fs::path& root) { + const auto valid = write_config(root, "capture-history", R"toml( +[runtime] +overlay_cache = true +overlay_capture_history = true +overlay_capture_persist_dir = ".aot_capture_history/TEST-00000" +)toml"); + const auto cfg = PSXRecompV4::load_game_config(valid); + check(cfg.runtime.overlay_capture_history, + "parser enables durable overlay capture history"); + check(cfg.runtime.overlay_capture_persist_dir == + ".aot_capture_history/TEST-00000", + "parser preserves project-relative capture history directory"); + + const auto escaping = write_config(root, "capture-history-escape", R"toml( +[runtime] +overlay_capture_persist_dir = "../outside" +)toml"); + check_throws([&] { (void)PSXRecompV4::load_game_config(escaping); }, + "must stay inside the project", + "parser rejects escaping capture history directory"); + + const auto absolute = write_config(root, "capture-history-absolute", R"toml( +[runtime] +overlay_capture_persist_dir = "C:/outside" +)toml"); + check_throws([&] { (void)PSXRecompV4::load_game_config(absolute); }, + "must be project-relative", + "parser rejects absolute capture history directory"); +} + void codegen_tests() { constexpr uint32_t original = 0x24020002u; // addiu v0, zero, 2 constexpr uint32_t replacement = 0x24020001u; // addiu v0, zero, 1 @@ -224,6 +255,7 @@ int main() { try { parser_tests(root); + capture_history_config_tests(root); codegen_tests(); } catch (const std::exception& e) { fmt::print(stderr, "FAIL unexpected exception: {}\n", e.what()); diff --git a/runtime/include/overlay_capture.h b/runtime/include/overlay_capture.h index 240184c7..f2fa580c 100644 --- a/runtime/include/overlay_capture.h +++ b/runtime/include/overlay_capture.h @@ -32,6 +32,13 @@ void overlay_capture_init(const char *out_dir); * overlay_capture_on_dma() and overlay_capture_write_json() are no-ops. */ void overlay_capture_set_enabled(int enabled); +/* Opt-in durable history. When enabled, every changed coherent latest snapshot + * is appended to overlay_captures.addendum.jsonl beside the executable. If + * persist_dir is non-empty, the same snapshot is also copied atomically to one + * new immutable JSON file there. game_id is used only in safe filenames. */ +void overlay_capture_configure_history(int enabled, const char *persist_dir, + const char *game_id); + /* Call from dma.c execute_ch3_cdrom() after every forward CH3 transfer with * load_start < 0x1C0000 and fntrace_is_game_started(). * load_addr is the physical RAM destination, size is byte count, @@ -54,6 +61,8 @@ int overlay_capture_count(void); * when [runtime] overlay_autocompile_cmd is configured. */ void overlay_autocapture_set_enabled(int on); void overlay_autocapture_tick(void); +/* Join any in-flight atomic writer before the final shutdown snapshot. */ +void overlay_autocapture_shutdown(void); void overlay_autocapture_get_status(int *enabled, uint32_t *triggers, uint64_t *last_delta); diff --git a/runtime/src/main.cpp b/runtime/src/main.cpp index 68642a3e..3e1eb0bb 100644 --- a/runtime/src/main.cpp +++ b/runtime/src/main.cpp @@ -978,6 +978,7 @@ static void shutdown_runtime(void) { /* (sljit removed 2026-07-15: overlay_compile_worker_stop joined the * off-thread JIT worker here; the worker no longer exists.) */ memcard_flush_all(); + overlay_autocapture_shutdown(); overlay_capture_write_json(); if (sdl_audio_device) { SDL_ClearQueuedAudio(sdl_audio_device); @@ -3241,6 +3242,26 @@ int main(int argc, char** argv) { std::string cache_dir = (exe_dir / "cache").string(); overlay_capture_set_out_dir(exe_dir.string().c_str()); overlay_capture_set_enabled(1); + std::string capture_persist_dir; + if (gc.runtime.overlay_capture_history && + !gc.runtime.overlay_capture_persist_dir.empty()) { + std::filesystem::path persist = + gc.project_root / gc.runtime.overlay_capture_persist_dir; + std::error_code persist_ec; + std::filesystem::create_directories(persist, persist_ec); + if (persist_ec) { + std::fprintf(stderr, + "psxrecomp: cannot create overlay capture history %s: %s\n", + persist.string().c_str(), persist_ec.message().c_str()); + } else { + capture_persist_dir = persist.string(); + } + } + overlay_capture_configure_history( + gc.runtime.overlay_capture_history ? 1 : 0, + capture_persist_dir.empty() ? nullptr : + capture_persist_dir.c_str(), + game_id.c_str()); overlay_loader_init(cache_dir.c_str(), game_id.c_str()); for (uint32_t addr : gc.runtime.overlay_native_block) { overlay_loader_native_block_add(addr); diff --git a/runtime/src/overlay_capture.c b/runtime/src/overlay_capture.c index 1cb51d10..516be488 100644 --- a/runtime/src/overlay_capture.c +++ b/runtime/src/overlay_capture.c @@ -6,7 +6,16 @@ #include #include #include +#include +#include #include +#ifdef _WIN32 +#include +#include +#include +#else +#include +#endif /* ---- Capture set -------------------------------------------------------- * Tracks which physical addresses have received CD DMA loads since game @@ -34,6 +43,68 @@ static int s_count = 0; static char s_out_dir[512]; static int s_active = 0; static int s_enabled = 0; /* config gate; off unless overlay cache enabled */ +static int s_history_enabled = 0; +static char s_history_addendum[600]; +static char s_history_persist_dir[600]; +static char s_history_game_id[64]; +static char s_history_session[64]; +static uint32_t s_history_sequence; +static uint64_t s_history_last_sig; +static int s_history_written; +static SDL_atomic_t s_autocap_write_state; /* 0 idle, 1 writing, 2 complete */ + +static int capture_process_id(void) +{ +#ifdef _WIN32 + return _getpid(); +#else + return (int)getpid(); +#endif +} + +static void sanitize_filename(char *dst, size_t dst_size, const char *src) +{ + size_t out = 0; + if (!dst_size) return; + while (src && *src && out + 1 < dst_size) { + unsigned char c = (unsigned char)*src++; + dst[out++] = (char)(isalnum(c) || c == '-' || c == '_' ? c : '_'); + } + dst[out] = '\0'; +} + +static int sync_file(FILE *f) +{ + if (fflush(f) != 0) return 0; +#ifdef _WIN32 + return _commit(_fileno(f)) == 0; +#else + return fsync(fileno(f)) == 0; +#endif +} + +static int atomic_replace_file(const char *tmp_path, const char *path) +{ +#ifdef _WIN32 + return MoveFileExA(tmp_path, path, + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) != 0; +#else + return rename(tmp_path, path) == 0; +#endif +} + +static uint64_t capture_file_sig(const char *path) +{ + FILE *f = fopen(path, "rb"); + uint64_t h = 1469598103934665603ull; + unsigned char buf[65536]; + size_t n, i; + if (!f) return 0; + while ((n = fread(buf, 1, sizeof(buf), f)) > 0) + for (i = 0; i < n; i++) { h ^= buf[i]; h *= 1099511628211ull; } + fclose(f); + return h; +} /* ---- Base64 encoder ----------------------------------------------------- */ @@ -72,6 +143,44 @@ void overlay_capture_set_enabled(int enabled) s_enabled = enabled ? 1 : 0; } +void overlay_capture_configure_history(int enabled, const char *persist_dir, + const char *game_id) +{ + time_t now; + struct tm tm_now; + char stamp[32]; + s_history_enabled = enabled ? 1 : 0; + s_history_persist_dir[0] = '\0'; + s_history_sequence = 0; + s_history_last_sig = 0; + s_history_written = 0; + sanitize_filename(s_history_game_id, sizeof(s_history_game_id), + game_id && game_id[0] ? game_id : "UNKNOWN"); + if (persist_dir && persist_dir[0]) { + snprintf(s_history_persist_dir, sizeof(s_history_persist_dir), "%s", + persist_dir); + } + snprintf(s_history_addendum, sizeof(s_history_addendum), + "%s/overlay_captures.addendum.jsonl", s_out_dir); + + now = time(NULL); +#ifdef _WIN32 + gmtime_s(&tm_now, &now); +#else + gmtime_r(&now, &tm_now); +#endif + strftime(stamp, sizeof(stamp), "%Y%m%dT%H%M%SZ", &tm_now); + snprintf(s_history_session, sizeof(s_history_session), "%s-p%d", + stamp, capture_process_id()); + if (s_history_enabled) { + fprintf(stdout, + "psxrecomp: durable overlay capture history enabled (%s%s%s)\n", + s_history_addendum, + s_history_persist_dir[0] ? "; immutable dir=" : "", + s_history_persist_dir[0] ? s_history_persist_dir : ""); + } +} + void overlay_capture_on_dma(uint32_t load_addr, uint32_t size, const uint8_t *bytes) { @@ -237,17 +346,17 @@ static void write_json_window(FILE *f, uint32_t win_lo_page, } } -static int write_json_snapshot(const char *path, uint32_t bw, - const uint32_t *bitmap, - const DirtyRamPcEntry *pc_table, - const DirtyRamPcEntry *exec_pc_table, - const uint8_t *ram_base) +static int write_json_snapshot_direct(const char *path, uint32_t bw, + const uint32_t *bitmap, + const DirtyRamPcEntry *pc_table, + const DirtyRamPcEntry *exec_pc_table, + const uint8_t *ram_base) { FILE *f; uint32_t page_sz; int first_region; page_sz = 4096u; - f = fopen(path, "w"); + f = fopen(path, "wb"); if (!f) return 0; fprintf(f, "[\n"); @@ -271,10 +380,152 @@ static int write_json_snapshot(const char *path, uint32_t bw, &first_region, bitmap, pc_table, exec_pc_table, ram_base); fprintf(f, "\n]\n"); - fclose(f); + { + int ok = !ferror(f) && sync_file(f); + if (fclose(f) != 0) ok = 0; + return ok; + } +} + +/* The live compiler needs one canonical latest snapshot, but it must never see + * a half-written JSON file. Format and fsync a same-directory temporary, then + * atomically replace the old snapshot. A hard kill leaves either the old valid + * file or the new valid file (plus a harmless .tmp), never a truncated latest. */ +static int write_json_snapshot_atomic(const char *path, uint32_t bw, + const uint32_t *bitmap, + const DirtyRamPcEntry *pc_table, + const DirtyRamPcEntry *exec_pc_table, + const uint8_t *ram_base) +{ + char tmp_path[700]; + snprintf(tmp_path, sizeof(tmp_path), "%s.tmp-p%d", path, + capture_process_id()); + if (!write_json_snapshot_direct(tmp_path, bw, bitmap, pc_table, + exec_pc_table, ram_base)) { + remove(tmp_path); + return 0; + } + if (!atomic_replace_file(tmp_path, path)) { + remove(tmp_path); + return 0; + } return 1; } +static int copy_file_atomic(const char *src_path, const char *dst_path) +{ + char tmp_path[700]; + unsigned char buf[65536]; + FILE *src, *dst; + size_t n; + int ok = 1; + snprintf(tmp_path, sizeof(tmp_path), "%s.tmp-p%d", dst_path, + capture_process_id()); + src = fopen(src_path, "rb"); + if (!src) return 0; + dst = fopen(tmp_path, "wb"); + if (!dst) { fclose(src); return 0; } + while ((n = fread(buf, 1, sizeof(buf), src)) > 0) { + if (fwrite(buf, 1, n, dst) != n) { ok = 0; break; } + } + if (ferror(src)) ok = 0; + fclose(src); + if (ok) ok = sync_file(dst); + if (fclose(dst) != 0) ok = 0; + if (!ok || !atomic_replace_file(tmp_path, dst_path)) { + remove(tmp_path); + return 0; + } + return 1; +} + +/* Append the canonical JSON array as one minified JSONL record. Whitespace is + * stripped only outside strings, so bytes_b64 remains byte-for-byte intact. + * A crash can damage only the final line; every earlier launch remains a valid + * independently parseable record. */ +static int append_history_record(const char *snapshot_path, const char *reason, + uint64_t sig, uint32_t sequence) +{ + FILE *src = fopen(snapshot_path, "rb"); + FILE *dst, *tail; + int c, in_string = 0, escape = 0, ok = 1; + int needs_separator = 0; + if (!src) return 0; + tail = fopen(s_history_addendum, "rb"); + if (tail) { + if (fseek(tail, -1, SEEK_END) == 0) { + c = fgetc(tail); + needs_separator = c != '\n'; + } + fclose(tail); + } + dst = fopen(s_history_addendum, "ab"); + if (!dst) { fclose(src); return 0; } + /* Quarantine an incomplete final record from a previous hard kill. */ + if (needs_separator && fputc('\n', dst) == EOF) ok = 0; + if (ok && fprintf(dst, + "{\"schema\":\"psxrecomp overlay capture addendum v1\"," + "\"game\":\"%s\",\"session\":\"%s\",\"sequence\":%u," + "\"reason\":\"%s\",\"fnv64\":\"%016llX\",\"captures\":", + s_history_game_id, s_history_session, sequence, + reason ? reason : "snapshot", (unsigned long long)sig) < 0) ok = 0; + while (ok && (c = fgetc(src)) != EOF) { + if (in_string) { + if (fputc(c, dst) == EOF) { ok = 0; break; } + if (escape) escape = 0; + else if (c == '\\') escape = 1; + else if (c == '"') in_string = 0; + } else { + if (c == '"') { + in_string = 1; + if (fputc(c, dst) == EOF) { ok = 0; break; } + } else if (!isspace((unsigned char)c)) { + if (fputc(c, dst) == EOF) { ok = 0; break; } + } + } + } + if (ferror(src) || in_string) ok = 0; + fclose(src); + if (ok && fprintf(dst, "}\n") < 0) ok = 0; + if (ok) ok = sync_file(dst); + if (fclose(dst) != 0) ok = 0; + return ok; +} + +static void persist_history_snapshot(const char *snapshot_path, + const char *reason, uint64_t sig) +{ + char persist_path[800]; + uint32_t sequence; + int addendum_saved; + int persist_saved = s_history_persist_dir[0] ? 0 : 1; + if (!s_history_enabled || !sig) return; + if (s_history_written && sig == s_history_last_sig) return; + sequence = ++s_history_sequence; + + if (s_history_persist_dir[0]) { + snprintf(persist_path, sizeof(persist_path), + "%s/%s_%s_%04u_%016llX.json", + s_history_persist_dir, s_history_game_id, s_history_session, + sequence, (unsigned long long)sig); + if (copy_file_atomic(snapshot_path, persist_path)) persist_saved = 1; + else fprintf(stderr, + "psxrecomp: failed to persist overlay capture snapshot %s\n", + persist_path); + } + addendum_saved = append_history_record(snapshot_path, reason, sig, sequence); + if (!addendum_saved) + fprintf(stderr, "psxrecomp: failed to append overlay capture history %s\n", + s_history_addendum); + /* Retry an unchanged snapshot if either configured durable sink failed. + * Repeated addendum records or immutable files are harmless: vault merges + * are content-keyed and idempotent. */ + if (addendum_saved && persist_saved) { + s_history_last_sig = sig; + s_history_written = 1; + } +} + void overlay_capture_write_json(void) { extern uint8_t *memory_get_ram_ptr(void); @@ -282,12 +533,18 @@ void overlay_capture_write_json(void) uint32_t bw = dirty_ram_get_bitmap_word_count(); uint32_t *bitmap; if (!s_active) return; + /* Manual dumps do not race the periodic worker. Shutdown joins it first. */ + if (SDL_AtomicGet(&s_autocap_write_state) != 0) return; bitmap = (uint32_t *)malloc((size_t)bw * sizeof(uint32_t)); if (!bitmap) return; for (uint32_t i = 0; i < bw; i++) bitmap[i] = dirty_ram_get_bitmap_word(i); snprintf(path, sizeof(path), "%s/overlay_captures.json", s_out_dir); - write_json_snapshot(path, bw, bitmap, g_dirty_ram_pc_table, - g_dirty_ram_exec_pc_table, memory_get_ram_ptr()); + if (write_json_snapshot_atomic(path, bw, bitmap, g_dirty_ram_pc_table, + g_dirty_ram_exec_pc_table, + memory_get_ram_ptr())) { + uint64_t sig = capture_file_sig(path); + persist_history_snapshot(path, "shutdown-or-manual", sig); + } free(bitmap); } @@ -355,9 +612,9 @@ typedef struct { uint32_t *bitmap; uint32_t bitmap_words; uint64_t manifest_sig; + int write_ok; char path[600]; } AutocapWriteJob; -static SDL_atomic_t s_autocap_write_state; /* 0 idle, 1 writing, 2 complete */ static SDL_Thread *s_autocap_write_thread; static AutocapWriteJob *s_autocap_write_job; static int s_autocap_seeded_existing; @@ -401,17 +658,8 @@ void overlay_autocapture_get_futility(uint32_t *backoff, uint32_t *futile) { static uint64_t autocap_manifest_sig(void) { char path[600]; - FILE *f; - uint64_t h = 1469598103934665603ull; - unsigned char buf[65536]; - size_t n, i; snprintf(path, sizeof(path), "%s/overlay_captures.json", s_out_dir); - f = fopen(path, "rb"); - if (!f) return 0; - while ((n = fread(buf, 1, sizeof(buf), f)) > 0) - for (i = 0; i < n; i++) { h ^= buf[i]; h *= 1099511628211ull; } - fclose(f); - return h; + return capture_file_sig(path); } static void autocap_write_job_free(AutocapWriteJob *job) @@ -425,9 +673,14 @@ static int autocap_write_thread_main(void *opaque) { AutocapWriteJob *job = (AutocapWriteJob *)opaque; SDL_SetThreadPriority(SDL_THREAD_PRIORITY_LOW); - if (write_json_snapshot(job->path, job->bitmap_words, job->bitmap, - job->pc_table, job->exec_pc_table, job->ram)) - job->manifest_sig = autocap_manifest_sig(); + job->write_ok = write_json_snapshot_atomic( + job->path, job->bitmap_words, job->bitmap, + job->pc_table, job->exec_pc_table, job->ram); + if (job->write_ok) { + job->manifest_sig = capture_file_sig(job->path); + persist_history_snapshot(job->path, "autocap", + job->manifest_sig); + } SDL_AtomicSet(&s_autocap_write_state, 2); return 0; } @@ -531,6 +784,22 @@ void overlay_autocapture_tick(void) autocap_write_start(); } +void overlay_autocapture_shutdown(void) +{ + int state = SDL_AtomicGet(&s_autocap_write_state); + if (state == 0) return; + if (s_autocap_write_thread) { + SDL_WaitThread(s_autocap_write_thread, NULL); + s_autocap_write_thread = NULL; + } + /* The worker atomically published and durably archived the coherent + * snapshot before setting state=2. No compiler request is needed while + * shutting down; just release its copied inputs before the final snapshot. */ + autocap_write_job_free(s_autocap_write_job); + s_autocap_write_job = NULL; + SDL_AtomicSet(&s_autocap_write_state, 0); +} + uint32_t overlay_capture_get_region_crc(uint32_t region_start, uint32_t region_size) { diff --git a/runtime/tests/test_capture_history.py b/runtime/tests/test_capture_history.py new file mode 100644 index 00000000..4d43578e --- /dev/null +++ b/runtime/tests/test_capture_history.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Regression checks for append-only overlay capture history ingestion.""" +import importlib.util +import json +import pathlib +import tempfile + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "coverage_vault", ROOT / "tools" / "coverage_vault.py") +MOD = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MOD) + + +def region(addr, payload, pcs): + return { + "schema": "psxrecomp overlay capture v2", + "load_addr": addr, + "size": 4, + "bytes_b64": payload, + "executed_pcs": pcs, + "dispatch_entry_pcs": pcs, + "function_entry_pcs": [], + "seeds": pcs, + } + + +def main(): + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + addendum = root / "overlay_captures.addendum.jsonl" + vault = root / "vault" / "overlay_captures.json" + first = region("0x80100000", "AAAAAA==", ["0x80100000"]) + second = region("0x80100000", "AQAAAA==", ["0x80100004"]) + records = [ + {"schema": "psxrecomp overlay capture addendum v1", + "captures": [first]}, + {"schema": "psxrecomp overlay capture addendum v1", + "captures": [first, second]}, + ] + with addendum.open("w", encoding="utf-8", newline="\n") as out: + out.write(json.dumps(records[0], separators=(",", ":")) + "\n") + # A later launch first quarantines a hard-killed partial record. + out.write('{"schema":"torn-tail"\n') + out.write(json.dumps({"schema": "unrelated", "captures": [second]}, + separators=(",", ":")) + "\n") + out.write(json.dumps(records[1], separators=(",", ":")) + "\n") + + new_variants, new_pcs = MOD.merge_addendum(str(vault), str(addendum)) + assert new_variants == 2 + assert new_pcs == 2 + merged = json.loads(vault.read_text(encoding="utf-8")) + assert len(merged) == 2 + assert not list(vault.parent.glob("*.tmp-*")) + + # Re-ingesting the same durable history is additive and idempotent. + assert MOD.merge_addendum(str(vault), str(addendum)) == (0, 0) + print("ALL PASS") + + +if __name__ == "__main__": + main() diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 296aded4..65aa210c 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -43,6 +43,24 @@ A mis-positioned / data-as-code shard either fails audit at compile time or neve fires at runtime → coverage loss, never incorrect execution. Whatever static misses, production autocompile (tcc/gcc) self-heals on first visit. +## Durable runtime capture history + +The canonical `overlay_captures.json` is atomically replaced for live consumers. +With `[runtime] overlay_capture_history = true`, each changed coherent snapshot is +also appended as one independent record to +`overlay_captures.addendum.jsonl` beside the executable. A torn final line does +not invalidate earlier or later records. Merge it into the private additive vault +with: + +```sh +python tools/coverage_vault.py merge --vault \ + --addendum /overlay_captures.addendum.jsonl +``` + +Dev configs can additionally set the project-relative +`overlay_capture_persist_dir` to retain immutable per-snapshot JSON files. Both +settings are opt-in; capture artifacts contain game bytes and must stay ignored. + ## Multi-game sweep findings (2026-07-17) `extract_generic.py` runs the improved extraction from a game.toml. Results: diff --git a/tools/coverage_vault.py b/tools/coverage_vault.py index 929a8c04..22c9bda6 100644 --- a/tools/coverage_vault.py +++ b/tools/coverage_vault.py @@ -19,7 +19,8 @@ It contains game-derived bytes, so the vault dir must stay gitignored / private and is committed. Usage: - coverage_vault.py merge --vault DIR [--captures captures.json] [--cache CACHE_DIR] + coverage_vault.py merge --vault DIR [--captures captures.json] + [--addendum captures.addendum.jsonl] [--cache CACHE_DIR] coverage_vault.py stats --vault DIR """ import argparse, json, os, shutil, hashlib, sys @@ -35,16 +36,58 @@ def _load_list(path): if not os.path.exists(path): return [] try: - v = json.load(open(path, encoding="utf-8")) - return v if isinstance(v, list) else [] + with open(path, encoding="utf-8") as source: + value = json.load(source) + return value if isinstance(value, list) else [] except Exception as e: print(" warn: could not read %s (%s); starting fresh" % (path, e)) return [] -def merge_captures(vault_json, src_json): - if not src_json or not os.path.exists(src_json): - return 0, 0 - src = _load_list(src_json) +def _load_addendum(path): + """Load every valid append-only history record, ignoring a torn tail. + + Each line is an independent snapshot wrapper. A hard kill may truncate only + the final line; earlier launches remain usable and a later runtime append + quarantines the bad tail with a newline. + """ + regions = [] + if not path or not os.path.exists(path): + return regions + with open(path, encoding="utf-8", errors="replace") as history: + for lineno, line in enumerate(history, 1): + if not line.strip(): + continue + try: + record = json.loads(line) + except Exception as e: + print(" warn: ignoring invalid addendum line %d in %s (%s)" % + (lineno, path, e)) + continue + if not isinstance(record, dict) or record.get("schema") != ( + "psxrecomp overlay capture addendum v1"): + print(" warn: ignoring unknown addendum record on line %d in %s" % + (lineno, path)) + continue + captures = record.get("captures", []) + if isinstance(captures, list): + regions.extend(r for r in captures if isinstance(r, dict)) + return regions + +def _write_list_atomic(path, value): + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + tmp = "%s.tmp-%d" % (path, os.getpid()) + try: + with open(tmp, "w", encoding="utf-8", newline="\n") as out: + json.dump(value, out, indent=1) + out.write("\n") + out.flush() + os.fsync(out.fileno()) + os.replace(tmp, path) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + +def merge_capture_regions(vault_json, src): if not src: return 0, 0 index = { _variant_key(r): r for r in _load_list(vault_json) } @@ -63,10 +106,17 @@ def merge_captures(vault_json, src_json): if fld == "executed_pcs": new_pcs += len(add - cur) tgt[fld] = sorted(cur | add) - os.makedirs(os.path.dirname(vault_json) or ".", exist_ok=True) - json.dump(list(index.values()), open(vault_json, "w", encoding="utf-8"), indent=1) + _write_list_atomic(vault_json, list(index.values())) return new_variants, new_pcs +def merge_captures(vault_json, src_json): + if not src_json or not os.path.exists(src_json): + return 0, 0 + return merge_capture_regions(vault_json, _load_list(src_json)) + +def merge_addendum(vault_json, addendum): + return merge_capture_regions(vault_json, _load_addendum(addendum)) + def merge_cache(vault_cache, src_cache): """Mirror compiled DLLs/.ranges into the vault, PRESERVING the relative layout (//cg_/file). The layout is load- @@ -111,6 +161,7 @@ def main(): ap.add_argument("cmd", choices=["merge", "stats"]) ap.add_argument("--vault", required=True, help="vault directory (kept gitignored/private)") ap.add_argument("--captures", help="source overlay_captures.json to merge in") + ap.add_argument("--addendum", help="append-only overlay capture history (.jsonl)") ap.add_argument("--cache", help="source cache dir (e.g. build/cache/) to merge in") a = ap.parse_args() if a.cmd == "stats": @@ -118,7 +169,13 @@ def main(): return 0 vj = os.path.join(a.vault, CAP_NAME) vc = os.path.join(a.vault, CACHE_SUB) - nv, np_ = merge_captures(vj, a.captures) if a.captures else (0, 0) + nv = np_ = 0 + if a.captures: + add_v, add_p = merge_captures(vj, a.captures) + nv += add_v; np_ += add_p + if a.addendum: + add_v, add_p = merge_addendum(vj, a.addendum) + nv += add_v; np_ += add_p nd = merge_cache(vc, a.cache) if a.cache else 0 print("coverage_vault: +%d new variant(s), +%d new PC(s), +%d new DLL(s) -> %s" % (nv, np_, nd, a.vault)) return 0