runtime: compact persisted capture addenda

docs/AOT_OVERLAY_PLAN.md

@@ -131,9 +131,13 @@ compiler, but it is published through a same-directory temporary file plus atomi
 replace so readers never observe a half-written manifest. Games may additionally
 set `overlay_capture_history = true` to append each changed coherent snapshot as
 an independent JSONL record to `overlay_captures.addendum.jsonl` beside the
-executable. A hard kill can damage only the last line; the next append quarantines
-that tail, and `tools/coverage_vault.py merge --addendum ...` ignores malformed
-lines while additively merging every valid record.
+executable. Without a persist directory the v1 record embeds the full snapshot.
+With dev immutable persistence enabled, a compact v2 record references the
+atomically published JSON and vault merge verifies its FNV signature before use;
+this avoids repeatedly embedding multi-megabyte base64 snapshots. A hard kill can
+damage only the last line; the next append quarantines that tail, and
+`tools/coverage_vault.py merge --addendum ...` ignores malformed records while
+additively merging every valid snapshot.
 
 Development configs may also set a project-relative
 `overlay_capture_persist_dir`. This creates one atomically published, immutable
@@ -141,6 +145,11 @@ JSON snapshot per changed capture. Absolute paths and `..` components are reject
 Production configs normally omit this key and retain only the naive executable-
 local addendum. Both history modes are off by default and contain game-derived
 bytes, so their outputs remain private/gitignored artifacts.
+Previously embedded development histories can be converted to verified compact
+references with `coverage_vault.py compact-addendum --addendum ...
+--persist-dir ...`. The tool verifies every valid record against its exact
+immutable snapshot before atomically replacing the addendum, and leaves the
+source untouched if any snapshot is missing or mismatched.
 
 The recall scoreboard can roll a persisted live gap set forward after a verified
 monotonic static expansion with

runtime/src/overlay_capture.c

@@ -492,6 +492,55 @@ static int append_history_record(const char *snapshot_path, const char *reason,
     return ok;
 }
 
+static int write_json_string(FILE *dst, const char *value)
+{
+    const unsigned char *p = (const unsigned char *)(value ? value : "");
+    if (fputc('"', dst) == EOF) return 0;
+    for (; *p; p++) {
+        if (*p == '"' || *p == '\\') {
+            if (fputc('\\', dst) == EOF || fputc(*p, dst) == EOF) return 0;
+        } else if (*p < 0x20) {
+            if (fprintf(dst, "\\u%04X", (unsigned)*p) < 0) return 0;
+        } else if (fputc(*p, dst) == EOF) {
+            return 0;
+        }
+    }
+    return fputc('"', dst) != EOF;
+}
+
+/* Dev persistence already owns an atomically published immutable full JSON
+ * snapshot. Append only a durable reference to it instead of embedding the
+ * same multi-megabyte capture array again every autocap interval. Production
+ * configs without persist_dir continue to use the self-contained v1 record. */
+static int append_history_reference(const char *persist_path, const char *reason,
+                                    uint64_t sig, uint32_t sequence)
+{
+    FILE *dst, *tail;
+    int c, needs_separator = 0, ok = 1;
+    tail = fopen(s_history_addendum, "rb");
+    if (tail) {
+        if (fseek(tail, -1, SEEK_END) == 0) {
+            c = fgetc(tail);
+            needs_separator = c != '\n';
+        }
+        fclose(tail);
+    }
+    dst = fopen(s_history_addendum, "ab");
+    if (!dst) return 0;
+    if (needs_separator && fputc('\n', dst) == EOF) ok = 0;
+    if (ok && fprintf(dst,
+            "{\"schema\":\"psxrecomp overlay capture addendum v2\","
+            "\"game\":\"%s\",\"session\":\"%s\",\"sequence\":%u,"
+            "\"reason\":\"%s\",\"fnv64\":\"%016llX\",\"snapshot\":",
+            s_history_game_id, s_history_session, sequence,
+            reason ? reason : "snapshot", (unsigned long long)sig) < 0) ok = 0;
+    if (ok) ok = write_json_string(dst, persist_path);
+    if (ok && fputs("}\n", dst) == EOF) ok = 0;
+    if (ok) ok = sync_file(dst);
+    if (fclose(dst) != 0) ok = 0;
+    return ok;
+}
+
 static void persist_history_snapshot(const char *snapshot_path,
                                      const char *reason, uint64_t sig)
 {
@@ -513,7 +562,12 @@ static void persist_history_snapshot(const char *snapshot_path,
             "psxrecomp: failed to persist overlay capture snapshot %s\n",
             persist_path);
     }
-    addendum_saved = append_history_record(snapshot_path, reason, sig, sequence);
+    if (s_history_persist_dir[0] && persist_saved)
+        addendum_saved = append_history_reference(
+            persist_path, reason, sig, sequence);
+    else
+        addendum_saved = append_history_record(
+            snapshot_path, reason, sig, sequence);
     if (!addendum_saved)
         fprintf(stderr, "psxrecomp: failed to append overlay capture history %s\n",
                 s_history_addendum);

tools/aot_overlay_spike/README.md

@@ -49,10 +49,13 @@ misses, production autocompile (tcc/gcc) self-heals on first visit.
 
 The canonical `overlay_captures.json` is atomically replaced for live consumers.
 With `[runtime] overlay_capture_history = true`, each changed coherent snapshot is
-also appended as one independent record to
-`overlay_captures.addendum.jsonl` beside the executable. A torn final line does
-not invalidate earlier or later records. Merge it into the private additive vault
-with:
+also represented by an independent record in `overlay_captures.addendum.jsonl`
+beside the executable. Production's simple mode embeds the full snapshot (v1).
+When an immutable persist directory is configured, dev mode writes a small v2
+record referencing the already-atomically-published snapshot instead of duplicating
+megabytes of base64 every autocap interval. Vault merge verifies the referenced
+file's FNV signature before ingesting it. A torn final line does not invalidate
+earlier or later records. Merge it into the private additive vault with:
 
 ```sh
 python tools/coverage_vault.py merge --vault <vault-dir> \
@@ -62,6 +65,17 @@ python tools/coverage_vault.py merge --vault <vault-dir> \
 Dev configs can additionally set the project-relative
 `overlay_capture_persist_dir` to retain immutable per-snapshot JSON files. Both
 settings are opt-in; capture artifacts contain game bytes and must stay ignored.
+Legacy dev addenda that embedded full snapshots before v2 can be compacted only
+after every immutable copy passes its signature check:
+
+```sh
+python tools/coverage_vault.py compact-addendum \
+  --addendum <exe-dir>/overlay_captures.addendum.jsonl \
+  --persist-dir <immutable-snapshot-dir>
+```
+
+The replacement is atomic and aborts without changing the addendum if any valid
+record lacks its exact immutable snapshot.
 
 ## Multi-game sweep findings (2026-07-17)
 

tools/coverage_vault.py

@@ -22,6 +22,8 @@ Usage:
   coverage_vault.py merge --vault DIR [--captures captures.json]
                           [--addendum captures.addendum.jsonl] [--cache CACHE_DIR]
   coverage_vault.py stats --vault DIR
+  coverage_vault.py compact-addendum --addendum captures.addendum.jsonl
+                                      --persist-dir IMMUTABLE_DIR
 """
 import argparse, json, os, shutil, hashlib, sys
 
@@ -51,6 +53,7 @@ def _load_addendum(path):
     quarantines the bad tail with a newline.
     """
     regions = []
+    seen_refs = set()
     if not path or not os.path.exists(path):
         return regions
     with open(path, encoding="utf-8", errors="replace") as history:
@@ -63,16 +66,56 @@ def _load_addendum(path):
                 print("  warn: ignoring invalid addendum line %d in %s (%s)" %
                       (lineno, path, e))
                 continue
-            if not isinstance(record, dict) or record.get("schema") != (
-                    "psxrecomp overlay capture addendum v1"):
+            if not isinstance(record, dict):
                 print("  warn: ignoring unknown addendum record on line %d in %s" %
                       (lineno, path))
                 continue
-            captures = record.get("captures", [])
-            if isinstance(captures, list):
-                regions.extend(r for r in captures if isinstance(r, dict))
+            schema = record.get("schema")
+            if schema == "psxrecomp overlay capture addendum v1":
+                captures = record.get("captures", [])
+                if isinstance(captures, list):
+                    regions.extend(r for r in captures if isinstance(r, dict))
+                continue
+            if schema == "psxrecomp overlay capture addendum v2":
+                snapshot = record.get("snapshot")
+                expected = str(record.get("fnv64", "")).upper()
+                if not isinstance(snapshot, str) or not snapshot:
+                    print("  warn: v2 addendum line %d has no snapshot" % lineno)
+                    continue
+                if not os.path.isabs(snapshot):
+                    snapshot = os.path.join(os.path.dirname(path), snapshot)
+                ref_key = (os.path.normcase(os.path.abspath(snapshot)), expected)
+                if ref_key in seen_refs:
+                    continue
+                seen_refs.add(ref_key)
+                if not os.path.exists(snapshot):
+                    print("  warn: v2 snapshot missing on line %d: %s" %
+                          (lineno, snapshot))
+                    continue
+                actual = _fnv64_file(snapshot)
+                if expected and expected != "%016X" % actual:
+                    print("  warn: v2 snapshot signature mismatch on line %d: %s" %
+                          (lineno, snapshot))
+                    continue
+                regions.extend(r for r in _load_list(snapshot)
+                               if isinstance(r, dict))
+                continue
+            print("  warn: ignoring unknown addendum record on line %d in %s" %
+                  (lineno, path))
     return regions
 
+def _fnv64_file(path):
+    value = 1469598103934665603
+    with open(path, "rb") as source:
+        while True:
+            chunk = source.read(65536)
+            if not chunk:
+                break
+            for byte in chunk:
+                value ^= byte
+                value = (value * 1099511628211) & 0xFFFFFFFFFFFFFFFF
+    return value
+
 def _write_list_atomic(path, value):
     os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
     tmp = "%s.tmp-%d" % (path, os.getpid())
@@ -87,6 +130,111 @@ def _write_list_atomic(path, value):
         if os.path.exists(tmp):
             os.unlink(tmp)
 
+def _verified_v2_record(record, addendum, lineno):
+    snapshot = record.get("snapshot")
+    expected = str(record.get("fnv64", "")).upper()
+    if not isinstance(snapshot, str) or not snapshot:
+        raise ValueError("v2 addendum line %d has no snapshot" % lineno)
+    if not os.path.isabs(snapshot):
+        snapshot = os.path.join(os.path.dirname(addendum), snapshot)
+    snapshot = os.path.abspath(snapshot)
+    if not os.path.isfile(snapshot):
+        raise ValueError("v2 snapshot missing on line %d: %s" %
+                         (lineno, snapshot))
+    actual = "%016X" % _fnv64_file(snapshot)
+    if not expected or expected != actual:
+        raise ValueError("v2 snapshot signature mismatch on line %d: %s" %
+                         (lineno, snapshot))
+    result = dict(record)
+    result["fnv64"] = actual
+    result["snapshot"] = snapshot
+    return result
+
+def compact_addendum(addendum, persist_dir):
+    """Atomically replace embedded v1 snapshots with verified v2 references.
+
+    Immutable snapshots are the authority during this operation. Every valid
+    v1 record must have its exact runtime-named file and matching FNV signature;
+    otherwise the original addendum is left byte-for-byte untouched.
+    """
+    if not addendum or not os.path.isfile(addendum):
+        raise ValueError("addendum does not exist: %s" % addendum)
+    if not persist_dir or not os.path.isdir(persist_dir):
+        raise ValueError("persist directory does not exist: %s" % persist_dir)
+    addendum = os.path.abspath(addendum)
+    persist_dir = os.path.abspath(persist_dir)
+    old_bytes = os.path.getsize(addendum)
+    records = []
+    seen_refs = set()
+    v1_count = v2_count = invalid_count = duplicate_count = 0
+    with open(addendum, encoding="utf-8", errors="replace") as history:
+        for lineno, line in enumerate(history, 1):
+            if not line.strip():
+                continue
+            try:
+                record = json.loads(line)
+            except Exception as e:
+                print("  warn: dropping invalid addendum line %d (%s)" %
+                      (lineno, e))
+                invalid_count += 1
+                continue
+            if not isinstance(record, dict):
+                raise ValueError("unknown addendum record on line %d" % lineno)
+            schema = record.get("schema")
+            if schema == "psxrecomp overlay capture addendum v1":
+                game = record.get("game")
+                session = record.get("session")
+                sequence = record.get("sequence")
+                expected = str(record.get("fnv64", "")).upper()
+                if (not isinstance(game, str) or not game or
+                        not isinstance(session, str) or not session or
+                        not isinstance(sequence, int) or sequence < 0 or
+                        len(expected) != 16 or
+                        any(c not in "0123456789ABCDEF" for c in expected)):
+                    raise ValueError("malformed v1 metadata on line %d" % lineno)
+                basename = "%s_%s_%04d_%s.json" % (
+                    game, session, sequence, expected)
+                snapshot = os.path.join(persist_dir, basename)
+                converted = dict(record)
+                converted.pop("captures", None)
+                converted["schema"] = "psxrecomp overlay capture addendum v2"
+                converted["snapshot"] = snapshot
+                converted = _verified_v2_record(converted, addendum, lineno)
+                v1_count += 1
+            elif schema == "psxrecomp overlay capture addendum v2":
+                converted = _verified_v2_record(record, addendum, lineno)
+                v2_count += 1
+            else:
+                raise ValueError("unknown addendum schema on line %d: %r" %
+                                 (lineno, schema))
+            ref_key = (os.path.normcase(converted["snapshot"]),
+                       converted["fnv64"])
+            if ref_key in seen_refs:
+                duplicate_count += 1
+                continue
+            seen_refs.add(ref_key)
+            records.append(converted)
+
+    tmp = "%s.tmp-%d" % (addendum, os.getpid())
+    try:
+        with open(tmp, "w", encoding="utf-8", newline="\n") as out:
+            for record in records:
+                out.write(json.dumps(record, separators=(",", ":")))
+                out.write("\n")
+            out.flush()
+            os.fsync(out.fileno())
+        os.replace(tmp, addendum)
+    finally:
+        if os.path.exists(tmp):
+            os.unlink(tmp)
+    new_bytes = os.path.getsize(addendum)
+    print("coverage_vault: compacted %s" % addendum)
+    print("  records: %d v1 converted, %d v2 retained, %d duplicates and %d invalid lines dropped" %
+          (v1_count, v2_count, duplicate_count, invalid_count))
+    print("  bytes:   %d -> %d (saved %d)" %
+          (old_bytes, new_bytes, old_bytes - new_bytes))
+    return v1_count, v2_count, duplicate_count, invalid_count
+
 def merge_capture_regions(vault_json, src):
     if not src:
         return 0, 0
@@ -158,12 +306,20 @@ def cmd_stats(vault):
 
 def main():
     ap = argparse.ArgumentParser()
-    ap.add_argument("cmd", choices=["merge", "stats"])
-    ap.add_argument("--vault", required=True, help="vault directory (kept gitignored/private)")
+    ap.add_argument("cmd", choices=["merge", "stats", "compact-addendum"])
+    ap.add_argument("--vault", help="vault directory (kept gitignored/private)")
     ap.add_argument("--captures", help="source overlay_captures.json to merge in")
     ap.add_argument("--addendum", help="append-only overlay capture history (.jsonl)")
     ap.add_argument("--cache", help="source cache dir (e.g. build/cache/<game_id>) to merge in")
+    ap.add_argument("--persist-dir", help="immutable capture snapshot directory")
     a = ap.parse_args()
+    if a.cmd == "compact-addendum":
+        if not a.addendum or not a.persist_dir:
+            ap.error("compact-addendum requires --addendum and --persist-dir")
+        compact_addendum(a.addendum, a.persist_dir)
+        return 0
+    if not a.vault:
+        ap.error("%s requires --vault" % a.cmd)
     if a.cmd == "stats":
         cmd_stats(a.vault)
         return 0

tools/test_coverage_vault.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+import importlib.util
+import json
+import os
+import pathlib
+import tempfile
+import unittest
+
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+SPEC = importlib.util.spec_from_file_location(
+    "coverage_vault", ROOT / "tools" / "coverage_vault.py")
+MOD = importlib.util.module_from_spec(SPEC)
+assert SPEC.loader is not None
+SPEC.loader.exec_module(MOD)
+
+
+class CoverageVaultHistoryTests(unittest.TestCase):
+    def test_v1_and_verified_v2_records_survive_a_torn_tail(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            snapshot = os.path.join(tmp, "immutable.json")
+            addendum = os.path.join(tmp, "history.jsonl")
+            v1_region = {"load_addr": "0x80010000", "bytes_b64": "AA=="}
+            v2_region = {"load_addr": "0x80100000", "bytes_b64": "AQ=="}
+            with open(snapshot, "w", encoding="utf-8", newline="\n") as out:
+                json.dump([v2_region], out)
+            signature = "%016X" % MOD._fnv64_file(snapshot)
+            records = [
+                {"schema": "psxrecomp overlay capture addendum v1",
+                 "captures": [v1_region]},
+                {"schema": "psxrecomp overlay capture addendum v2",
+                 "snapshot": snapshot, "fnv64": signature},
+                # A repeated reference is idempotent and should not inflate RAM.
+                {"schema": "psxrecomp overlay capture addendum v2",
+                 "snapshot": snapshot, "fnv64": signature},
+                # A bad signature must fail closed instead of ingesting corruption.
+                {"schema": "psxrecomp overlay capture addendum v2",
+                 "snapshot": snapshot, "fnv64": "0000000000000000"},
+            ]
+            with open(addendum, "w", encoding="utf-8", newline="\n") as out:
+                for record in records:
+                    out.write(json.dumps(record) + "\n")
+                out.write('{"schema":"torn')
+            self.assertEqual(MOD._load_addendum(addendum),
+                             [v1_region, v2_region])
+
+    def test_compact_addendum_converts_v1_and_is_idempotent(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            persist = os.path.join(tmp, "immutable")
+            os.mkdir(persist)
+            addendum = os.path.join(tmp, "history.jsonl")
+            snapshot = os.path.join(
+                persist, "GAME_session_0001_0000000000000000.json")
+            region = {"load_addr": "0x80010000", "bytes_b64": "AA=="}
+            with open(snapshot, "w", encoding="utf-8", newline="\n") as out:
+                json.dump([region], out)
+            signature = "%016X" % MOD._fnv64_file(snapshot)
+            corrected = os.path.join(
+                persist, "GAME_session_0001_%s.json" % signature)
+            os.replace(snapshot, corrected)
+            v1 = {"schema": "psxrecomp overlay capture addendum v1",
+                  "game": "GAME", "session": "session", "sequence": 1,
+                  "reason": "autocap", "fnv64": signature,
+                  "captures": [region]}
+            with open(addendum, "w", encoding="utf-8", newline="\n") as out:
+                out.write(json.dumps(v1) + "\n")
+                out.write('{"schema":"torn')
+            self.assertEqual(MOD.compact_addendum(addendum, persist),
+                             (1, 0, 0, 1))
+            with open(addendum, encoding="utf-8") as source:
+                compacted = json.loads(source.readline())
+            self.assertNotIn("captures", compacted)
+            self.assertEqual(compacted["snapshot"], corrected)
+            self.assertEqual(MOD._load_addendum(addendum), [region])
+            self.assertEqual(MOD.compact_addendum(addendum, persist),
+                             (0, 1, 0, 0))
+            self.assertEqual(MOD._load_addendum(addendum), [region])
+
+    def test_compact_addendum_failure_preserves_source(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            persist = os.path.join(tmp, "immutable")
+            os.mkdir(persist)
+            addendum = os.path.join(tmp, "history.jsonl")
+            v1 = {"schema": "psxrecomp overlay capture addendum v1",
+                  "game": "GAME", "session": "missing", "sequence": 1,
+                  "reason": "autocap", "fnv64": "1234567890ABCDEF",
+                  "captures": []}
+            original = (json.dumps(v1) + "\n").encode()
+            with open(addendum, "wb") as out:
+                out.write(original)
+            with self.assertRaisesRegex(ValueError, "snapshot missing"):
+                MOD.compact_addendum(addendum, persist)
+            with open(addendum, "rb") as source:
+                self.assertEqual(source.read(), original)
+
+
+if __name__ == "__main__":
+    unittest.main()