From 65d89a06d7eaf385693e91d9bd2566dbcb852e9f Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 18 Jul 2026 02:56:53 -0700 Subject: [PATCH] runtime: compact persisted capture addenda --- docs/AOT_OVERLAY_PLAN.md | 15 ++- runtime/src/overlay_capture.c | 56 +++++++++- tools/aot_overlay_spike/README.md | 22 +++- tools/coverage_vault.py | 170 ++++++++++++++++++++++++++++-- tools/test_coverage_vault.py | 98 +++++++++++++++++ 5 files changed, 346 insertions(+), 15 deletions(-) create mode 100644 tools/test_coverage_vault.py diff --git a/docs/AOT_OVERLAY_PLAN.md b/docs/AOT_OVERLAY_PLAN.md index 027b5b64..5bd6b002 100644 --- a/docs/AOT_OVERLAY_PLAN.md +++ b/docs/AOT_OVERLAY_PLAN.md @@ -131,9 +131,13 @@ compiler, but it is published through a same-directory temporary file plus atomi replace so readers never observe a half-written manifest. Games may additionally set `overlay_capture_history = true` to append each changed coherent snapshot as an independent JSONL record to `overlay_captures.addendum.jsonl` beside the -executable. A hard kill can damage only the last line; the next append quarantines -that tail, and `tools/coverage_vault.py merge --addendum ...` ignores malformed -lines while additively merging every valid record. +executable. Without a persist directory the v1 record embeds the full snapshot. +With dev immutable persistence enabled, a compact v2 record references the +atomically published JSON and vault merge verifies its FNV signature before use; +this avoids repeatedly embedding multi-megabyte base64 snapshots. A hard kill can +damage only the last line; the next append quarantines that tail, and +`tools/coverage_vault.py merge --addendum ...` ignores malformed records while +additively merging every valid snapshot. Development configs may also set a project-relative `overlay_capture_persist_dir`. This creates one atomically published, immutable @@ -141,6 +145,11 @@ JSON snapshot per changed capture. Absolute paths and `..` components are reject Production configs normally omit this key and retain only the naive executable- local addendum. Both history modes are off by default and contain game-derived bytes, so their outputs remain private/gitignored artifacts. +Previously embedded development histories can be converted to verified compact +references with `coverage_vault.py compact-addendum --addendum ... +--persist-dir ...`. The tool verifies every valid record against its exact +immutable snapshot before atomically replacing the addendum, and leaves the +source untouched if any snapshot is missing or mismatched. The recall scoreboard can roll a persisted live gap set forward after a verified monotonic static expansion with diff --git a/runtime/src/overlay_capture.c b/runtime/src/overlay_capture.c index 516be488..fbf17b95 100644 --- a/runtime/src/overlay_capture.c +++ b/runtime/src/overlay_capture.c @@ -492,6 +492,55 @@ static int append_history_record(const char *snapshot_path, const char *reason, return ok; } +static int write_json_string(FILE *dst, const char *value) +{ + const unsigned char *p = (const unsigned char *)(value ? value : ""); + if (fputc('"', dst) == EOF) return 0; + for (; *p; p++) { + if (*p == '"' || *p == '\\') { + if (fputc('\\', dst) == EOF || fputc(*p, dst) == EOF) return 0; + } else if (*p < 0x20) { + if (fprintf(dst, "\\u%04X", (unsigned)*p) < 0) return 0; + } else if (fputc(*p, dst) == EOF) { + return 0; + } + } + return fputc('"', dst) != EOF; +} + +/* Dev persistence already owns an atomically published immutable full JSON + * snapshot. Append only a durable reference to it instead of embedding the + * same multi-megabyte capture array again every autocap interval. Production + * configs without persist_dir continue to use the self-contained v1 record. */ +static int append_history_reference(const char *persist_path, const char *reason, + uint64_t sig, uint32_t sequence) +{ + FILE *dst, *tail; + int c, needs_separator = 0, ok = 1; + tail = fopen(s_history_addendum, "rb"); + if (tail) { + if (fseek(tail, -1, SEEK_END) == 0) { + c = fgetc(tail); + needs_separator = c != '\n'; + } + fclose(tail); + } + dst = fopen(s_history_addendum, "ab"); + if (!dst) return 0; + if (needs_separator && fputc('\n', dst) == EOF) ok = 0; + if (ok && fprintf(dst, + "{\"schema\":\"psxrecomp overlay capture addendum v2\"," + "\"game\":\"%s\",\"session\":\"%s\",\"sequence\":%u," + "\"reason\":\"%s\",\"fnv64\":\"%016llX\",\"snapshot\":", + s_history_game_id, s_history_session, sequence, + reason ? reason : "snapshot", (unsigned long long)sig) < 0) ok = 0; + if (ok) ok = write_json_string(dst, persist_path); + if (ok && fputs("}\n", dst) == EOF) ok = 0; + if (ok) ok = sync_file(dst); + if (fclose(dst) != 0) ok = 0; + return ok; +} + static void persist_history_snapshot(const char *snapshot_path, const char *reason, uint64_t sig) { @@ -513,7 +562,12 @@ static void persist_history_snapshot(const char *snapshot_path, "psxrecomp: failed to persist overlay capture snapshot %s\n", persist_path); } - addendum_saved = append_history_record(snapshot_path, reason, sig, sequence); + if (s_history_persist_dir[0] && persist_saved) + addendum_saved = append_history_reference( + persist_path, reason, sig, sequence); + else + addendum_saved = append_history_record( + snapshot_path, reason, sig, sequence); if (!addendum_saved) fprintf(stderr, "psxrecomp: failed to append overlay capture history %s\n", s_history_addendum); diff --git a/tools/aot_overlay_spike/README.md b/tools/aot_overlay_spike/README.md index 40973158..06dfb77b 100644 --- a/tools/aot_overlay_spike/README.md +++ b/tools/aot_overlay_spike/README.md @@ -49,10 +49,13 @@ misses, production autocompile (tcc/gcc) self-heals on first visit. The canonical `overlay_captures.json` is atomically replaced for live consumers. With `[runtime] overlay_capture_history = true`, each changed coherent snapshot is -also appended as one independent record to -`overlay_captures.addendum.jsonl` beside the executable. A torn final line does -not invalidate earlier or later records. Merge it into the private additive vault -with: +also represented by an independent record in `overlay_captures.addendum.jsonl` +beside the executable. Production's simple mode embeds the full snapshot (v1). +When an immutable persist directory is configured, dev mode writes a small v2 +record referencing the already-atomically-published snapshot instead of duplicating +megabytes of base64 every autocap interval. Vault merge verifies the referenced +file's FNV signature before ingesting it. A torn final line does not invalidate +earlier or later records. Merge it into the private additive vault with: ```sh python tools/coverage_vault.py merge --vault \ @@ -62,6 +65,17 @@ python tools/coverage_vault.py merge --vault \ Dev configs can additionally set the project-relative `overlay_capture_persist_dir` to retain immutable per-snapshot JSON files. Both settings are opt-in; capture artifacts contain game bytes and must stay ignored. +Legacy dev addenda that embedded full snapshots before v2 can be compacted only +after every immutable copy passes its signature check: + +```sh +python tools/coverage_vault.py compact-addendum \ + --addendum /overlay_captures.addendum.jsonl \ + --persist-dir +``` + +The replacement is atomic and aborts without changing the addendum if any valid +record lacks its exact immutable snapshot. ## Multi-game sweep findings (2026-07-17) diff --git a/tools/coverage_vault.py b/tools/coverage_vault.py index 22c9bda6..0df9f4fb 100644 --- a/tools/coverage_vault.py +++ b/tools/coverage_vault.py @@ -22,6 +22,8 @@ Usage: coverage_vault.py merge --vault DIR [--captures captures.json] [--addendum captures.addendum.jsonl] [--cache CACHE_DIR] coverage_vault.py stats --vault DIR + coverage_vault.py compact-addendum --addendum captures.addendum.jsonl + --persist-dir IMMUTABLE_DIR """ import argparse, json, os, shutil, hashlib, sys @@ -51,6 +53,7 @@ def _load_addendum(path): quarantines the bad tail with a newline. """ regions = [] + seen_refs = set() if not path or not os.path.exists(path): return regions with open(path, encoding="utf-8", errors="replace") as history: @@ -63,16 +66,56 @@ def _load_addendum(path): print(" warn: ignoring invalid addendum line %d in %s (%s)" % (lineno, path, e)) continue - if not isinstance(record, dict) or record.get("schema") != ( - "psxrecomp overlay capture addendum v1"): + if not isinstance(record, dict): print(" warn: ignoring unknown addendum record on line %d in %s" % (lineno, path)) continue - captures = record.get("captures", []) - if isinstance(captures, list): - regions.extend(r for r in captures if isinstance(r, dict)) + schema = record.get("schema") + if schema == "psxrecomp overlay capture addendum v1": + captures = record.get("captures", []) + if isinstance(captures, list): + regions.extend(r for r in captures if isinstance(r, dict)) + continue + if schema == "psxrecomp overlay capture addendum v2": + snapshot = record.get("snapshot") + expected = str(record.get("fnv64", "")).upper() + if not isinstance(snapshot, str) or not snapshot: + print(" warn: v2 addendum line %d has no snapshot" % lineno) + continue + if not os.path.isabs(snapshot): + snapshot = os.path.join(os.path.dirname(path), snapshot) + ref_key = (os.path.normcase(os.path.abspath(snapshot)), expected) + if ref_key in seen_refs: + continue + seen_refs.add(ref_key) + if not os.path.exists(snapshot): + print(" warn: v2 snapshot missing on line %d: %s" % + (lineno, snapshot)) + continue + actual = _fnv64_file(snapshot) + if expected and expected != "%016X" % actual: + print(" warn: v2 snapshot signature mismatch on line %d: %s" % + (lineno, snapshot)) + continue + regions.extend(r for r in _load_list(snapshot) + if isinstance(r, dict)) + continue + print(" warn: ignoring unknown addendum record on line %d in %s" % + (lineno, path)) return regions +def _fnv64_file(path): + value = 1469598103934665603 + with open(path, "rb") as source: + while True: + chunk = source.read(65536) + if not chunk: + break + for byte in chunk: + value ^= byte + value = (value * 1099511628211) & 0xFFFFFFFFFFFFFFFF + return value + def _write_list_atomic(path, value): os.makedirs(os.path.dirname(path) or ".", exist_ok=True) tmp = "%s.tmp-%d" % (path, os.getpid()) @@ -87,6 +130,111 @@ def _write_list_atomic(path, value): if os.path.exists(tmp): os.unlink(tmp) +def _verified_v2_record(record, addendum, lineno): + snapshot = record.get("snapshot") + expected = str(record.get("fnv64", "")).upper() + if not isinstance(snapshot, str) or not snapshot: + raise ValueError("v2 addendum line %d has no snapshot" % lineno) + if not os.path.isabs(snapshot): + snapshot = os.path.join(os.path.dirname(addendum), snapshot) + snapshot = os.path.abspath(snapshot) + if not os.path.isfile(snapshot): + raise ValueError("v2 snapshot missing on line %d: %s" % + (lineno, snapshot)) + actual = "%016X" % _fnv64_file(snapshot) + if not expected or expected != actual: + raise ValueError("v2 snapshot signature mismatch on line %d: %s" % + (lineno, snapshot)) + result = dict(record) + result["fnv64"] = actual + result["snapshot"] = snapshot + return result + +def compact_addendum(addendum, persist_dir): + """Atomically replace embedded v1 snapshots with verified v2 references. + + Immutable snapshots are the authority during this operation. Every valid + v1 record must have its exact runtime-named file and matching FNV signature; + otherwise the original addendum is left byte-for-byte untouched. + """ + if not addendum or not os.path.isfile(addendum): + raise ValueError("addendum does not exist: %s" % addendum) + if not persist_dir or not os.path.isdir(persist_dir): + raise ValueError("persist directory does not exist: %s" % persist_dir) + addendum = os.path.abspath(addendum) + persist_dir = os.path.abspath(persist_dir) + old_bytes = os.path.getsize(addendum) + records = [] + seen_refs = set() + v1_count = v2_count = invalid_count = duplicate_count = 0 + with open(addendum, encoding="utf-8", errors="replace") as history: + for lineno, line in enumerate(history, 1): + if not line.strip(): + continue + try: + record = json.loads(line) + except Exception as e: + print(" warn: dropping invalid addendum line %d (%s)" % + (lineno, e)) + invalid_count += 1 + continue + if not isinstance(record, dict): + raise ValueError("unknown addendum record on line %d" % lineno) + schema = record.get("schema") + if schema == "psxrecomp overlay capture addendum v1": + game = record.get("game") + session = record.get("session") + sequence = record.get("sequence") + expected = str(record.get("fnv64", "")).upper() + if (not isinstance(game, str) or not game or + not isinstance(session, str) or not session or + not isinstance(sequence, int) or sequence < 0 or + len(expected) != 16 or + any(c not in "0123456789ABCDEF" for c in expected)): + raise ValueError("malformed v1 metadata on line %d" % lineno) + basename = "%s_%s_%04d_%s.json" % ( + game, session, sequence, expected) + snapshot = os.path.join(persist_dir, basename) + converted = dict(record) + converted.pop("captures", None) + converted["schema"] = "psxrecomp overlay capture addendum v2" + converted["snapshot"] = snapshot + converted = _verified_v2_record(converted, addendum, lineno) + v1_count += 1 + elif schema == "psxrecomp overlay capture addendum v2": + converted = _verified_v2_record(record, addendum, lineno) + v2_count += 1 + else: + raise ValueError("unknown addendum schema on line %d: %r" % + (lineno, schema)) + ref_key = (os.path.normcase(converted["snapshot"]), + converted["fnv64"]) + if ref_key in seen_refs: + duplicate_count += 1 + continue + seen_refs.add(ref_key) + records.append(converted) + + tmp = "%s.tmp-%d" % (addendum, os.getpid()) + try: + with open(tmp, "w", encoding="utf-8", newline="\n") as out: + for record in records: + out.write(json.dumps(record, separators=(",", ":"))) + out.write("\n") + out.flush() + os.fsync(out.fileno()) + os.replace(tmp, addendum) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + new_bytes = os.path.getsize(addendum) + print("coverage_vault: compacted %s" % addendum) + print(" records: %d v1 converted, %d v2 retained, %d duplicates and %d invalid lines dropped" % + (v1_count, v2_count, duplicate_count, invalid_count)) + print(" bytes: %d -> %d (saved %d)" % + (old_bytes, new_bytes, old_bytes - new_bytes)) + return v1_count, v2_count, duplicate_count, invalid_count + def merge_capture_regions(vault_json, src): if not src: return 0, 0 @@ -158,12 +306,20 @@ def cmd_stats(vault): def main(): ap = argparse.ArgumentParser() - ap.add_argument("cmd", choices=["merge", "stats"]) - ap.add_argument("--vault", required=True, help="vault directory (kept gitignored/private)") + ap.add_argument("cmd", choices=["merge", "stats", "compact-addendum"]) + ap.add_argument("--vault", help="vault directory (kept gitignored/private)") ap.add_argument("--captures", help="source overlay_captures.json to merge in") ap.add_argument("--addendum", help="append-only overlay capture history (.jsonl)") ap.add_argument("--cache", help="source cache dir (e.g. build/cache/) to merge in") + ap.add_argument("--persist-dir", help="immutable capture snapshot directory") a = ap.parse_args() + if a.cmd == "compact-addendum": + if not a.addendum or not a.persist_dir: + ap.error("compact-addendum requires --addendum and --persist-dir") + compact_addendum(a.addendum, a.persist_dir) + return 0 + if not a.vault: + ap.error("%s requires --vault" % a.cmd) if a.cmd == "stats": cmd_stats(a.vault) return 0 diff --git a/tools/test_coverage_vault.py b/tools/test_coverage_vault.py new file mode 100644 index 00000000..a689cc28 --- /dev/null +++ b/tools/test_coverage_vault.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +import importlib.util +import json +import os +import pathlib +import tempfile +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "coverage_vault", ROOT / "tools" / "coverage_vault.py") +MOD = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MOD) + + +class CoverageVaultHistoryTests(unittest.TestCase): + def test_v1_and_verified_v2_records_survive_a_torn_tail(self): + with tempfile.TemporaryDirectory() as tmp: + snapshot = os.path.join(tmp, "immutable.json") + addendum = os.path.join(tmp, "history.jsonl") + v1_region = {"load_addr": "0x80010000", "bytes_b64": "AA=="} + v2_region = {"load_addr": "0x80100000", "bytes_b64": "AQ=="} + with open(snapshot, "w", encoding="utf-8", newline="\n") as out: + json.dump([v2_region], out) + signature = "%016X" % MOD._fnv64_file(snapshot) + records = [ + {"schema": "psxrecomp overlay capture addendum v1", + "captures": [v1_region]}, + {"schema": "psxrecomp overlay capture addendum v2", + "snapshot": snapshot, "fnv64": signature}, + # A repeated reference is idempotent and should not inflate RAM. + {"schema": "psxrecomp overlay capture addendum v2", + "snapshot": snapshot, "fnv64": signature}, + # A bad signature must fail closed instead of ingesting corruption. + {"schema": "psxrecomp overlay capture addendum v2", + "snapshot": snapshot, "fnv64": "0000000000000000"}, + ] + with open(addendum, "w", encoding="utf-8", newline="\n") as out: + for record in records: + out.write(json.dumps(record) + "\n") + out.write('{"schema":"torn') + self.assertEqual(MOD._load_addendum(addendum), + [v1_region, v2_region]) + + def test_compact_addendum_converts_v1_and_is_idempotent(self): + with tempfile.TemporaryDirectory() as tmp: + persist = os.path.join(tmp, "immutable") + os.mkdir(persist) + addendum = os.path.join(tmp, "history.jsonl") + snapshot = os.path.join( + persist, "GAME_session_0001_0000000000000000.json") + region = {"load_addr": "0x80010000", "bytes_b64": "AA=="} + with open(snapshot, "w", encoding="utf-8", newline="\n") as out: + json.dump([region], out) + signature = "%016X" % MOD._fnv64_file(snapshot) + corrected = os.path.join( + persist, "GAME_session_0001_%s.json" % signature) + os.replace(snapshot, corrected) + v1 = {"schema": "psxrecomp overlay capture addendum v1", + "game": "GAME", "session": "session", "sequence": 1, + "reason": "autocap", "fnv64": signature, + "captures": [region]} + with open(addendum, "w", encoding="utf-8", newline="\n") as out: + out.write(json.dumps(v1) + "\n") + out.write('{"schema":"torn') + self.assertEqual(MOD.compact_addendum(addendum, persist), + (1, 0, 0, 1)) + with open(addendum, encoding="utf-8") as source: + compacted = json.loads(source.readline()) + self.assertNotIn("captures", compacted) + self.assertEqual(compacted["snapshot"], corrected) + self.assertEqual(MOD._load_addendum(addendum), [region]) + self.assertEqual(MOD.compact_addendum(addendum, persist), + (0, 1, 0, 0)) + self.assertEqual(MOD._load_addendum(addendum), [region]) + + def test_compact_addendum_failure_preserves_source(self): + with tempfile.TemporaryDirectory() as tmp: + persist = os.path.join(tmp, "immutable") + os.mkdir(persist) + addendum = os.path.join(tmp, "history.jsonl") + v1 = {"schema": "psxrecomp overlay capture addendum v1", + "game": "GAME", "session": "missing", "sequence": 1, + "reason": "autocap", "fnv64": "1234567890ABCDEF", + "captures": []} + original = (json.dumps(v1) + "\n").encode() + with open(addendum, "wb") as out: + out.write(original) + with self.assertRaisesRegex(ValueError, "snapshot missing"): + MOD.compact_addendum(addendum, persist) + with open(addendum, "rb") as source: + self.assertEqual(source.read(), original) + + +if __name__ == "__main__": + unittest.main()