From de09f27d5c9c0a86df6cf6ee1af585ce020ca3eb Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 19 Sep 2026 12:15:31 -0700 Subject: [PATCH] tools: enrich runtime overlay captures with static roots --- docs/COMPILING_OVERLAYS.md | 19 +++ recompiler/CMakeLists.txt | 5 + tools/enrich_overlay_captures.py | 128 ++++++++++++++++++++ tools/tests/test_enrich_overlay_captures.py | 62 ++++++++++ 4 files changed, 214 insertions(+) create mode 100644 tools/enrich_overlay_captures.py create mode 100644 tools/tests/test_enrich_overlay_captures.py diff --git a/docs/COMPILING_OVERLAYS.md b/docs/COMPILING_OVERLAYS.md index 81629cac..2a59b95f 100644 --- a/docs/COMPILING_OVERLAYS.md +++ b/docs/COMPILING_OVERLAYS.md @@ -19,6 +19,25 @@ to pre-build coverage for their own machine. > ⚠️ **Privacy:** `overlay_captures.json` contains snapshots of the game's own > code read from *your* disc. Keep it private — do not post it publicly. +### Enriching runtime captures with conservative walk roots + +Runtime capture records PCs it dispatched to, not trusted function boundaries: +continuations and jump-table cases can be valid dispatch PCs in the middle of a +function. For continuation-passing titles, derive portable root evidence from +the captured bytes before compiling rather than placing game addresses in TOML: + +```sh +python psxrecomp/tools/enrich_overlay_captures.py \ + --captures /overlay_captures.json \ + --out /overlay_captures.enriched.json +``` + +The tool writes `static_discovery_entry_pcs`; it never rewrites the +runtime-owned `function_entry_pcs`. It admits only framed entries plus direct +`jal`/dense in-image pointer targets that pass the compiler's bounded CFG +proof. `compile_overlays.py` validates them again, so uncertain data stays +interpreted. Pass the enriched file to `compile_overlays.py --captures`. + §0 frames the production goal; §1 builds the gcc cache you ship; §2 bakes overlays straight into the executable. diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index a82aa26b..ef07dac8 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -576,6 +576,11 @@ if(BUILD_TESTING) COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_coverage_vault.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) + add_test( + NAME enrich_overlay_captures + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/../tools/tests/test_enrich_overlay_captures.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) # The runtime exe name must come from CMake, not be re-derived. add_test( diff --git a/tools/enrich_overlay_captures.py b/tools/enrich_overlay_captures.py new file mode 100644 index 00000000..b752c503 --- /dev/null +++ b/tools/enrich_overlay_captures.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Add conservative static walk-root evidence to runtime overlay captures. + +Runtime capture knows that a dirty-RAM PC was dispatched, but that is not the +same thing as knowing it is a function boundary: CPS continuations and jump +table cases are dispatched too. This tool therefore leaves the runtime-owned +``function_entry_pcs`` untouched and derives the optional, tool-owned +``static_discovery_entry_pcs`` from captured bytes. compile_overlays.py +re-validates every supplied root before compiling it. +""" + +import argparse +import base64 +import json +import struct +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'tools')) +import compile_overlays as CO # noqa: E402 + + +def word_at(data, base, addr): + offset = addr - base + if offset < 0 or offset + 4 > len(data): + return None + return struct.unpack_from('> 26) == 0x03: # jal + target = 0x80000000 | ((word & 0x03ffffff) << 2) + if is_candidate(data, base, target): + roots.add(target) + pointers.append(word if word is not None and not (word & 3) and + base <= word < hi else None) + + # A dense run is a table only when it has three or more in-image pointers. + # Each member must still pass the same root proof; no bare pointer becomes + # authority, and a pointer to the table itself fails the CFG proof. + start = 0 + while start < len(pointers): + if pointers[start] is None: + start += 1 + continue + end = start + 1 + while end < len(pointers) and pointers[end] is not None: + end += 1 + if end - start >= 3: + for target in pointers[start:end]: + if is_candidate(data, base, target): + roots.add(target) + start = end + return sorted(roots) + + +def enrich_record(record): + """Return a copy of *record* with discovered roots unioned into it.""" + out = dict(record) + try: + base = CO._parse_addr(record['load_addr']) + data = base64.b64decode(record['bytes_b64'], validate=True) + size = int(record.get('size', len(data))) + except (KeyError, TypeError, ValueError) as exc: + raise ValueError('capture lacks valid load_addr/bytes_b64/size') from exc + if size != len(data) or base & 3 or len(data) & 3: + raise ValueError('capture bytes must be a word-aligned image matching size') + prior = CO._parse_addr_list(record.get('static_discovery_entry_pcs', [])) + roots = sorted(prior | set(derived_roots(data, base))) + out['static_discovery_entry_pcs'] = [f'0x{addr:08X}' for addr in roots] + return out, len(roots) - len(prior) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--captures', required=True, help='runtime capture JSON') + parser.add_argument('--out', required=True, help='enriched JSON output path') + args = parser.parse_args(argv) + with open(args.captures, encoding='utf-8') as src: + records = json.load(src) + if not isinstance(records, list): + raise SystemExit('captures must contain a JSON array') + enriched, added = [], 0 + for index, record in enumerate(records): + if not isinstance(record, dict): + raise SystemExit(f'capture {index} is not an object') + try: + item, count = enrich_record(record) + except ValueError as exc: + raise SystemExit(f'capture {index}: {exc}') from exc + enriched.append(item) + added += count + with open(args.out, 'w', encoding='utf-8', newline='\n') as dst: + json.dump(enriched, dst, indent=2) + dst.write('\n') + print(f'enriched {len(enriched)} capture(s); added {added} static discovery root(s)') + + +if __name__ == '__main__': + main() diff --git a/tools/tests/test_enrich_overlay_captures.py b/tools/tests/test_enrich_overlay_captures.py new file mode 100644 index 00000000..867b82b3 --- /dev/null +++ b/tools/tests/test_enrich_overlay_captures.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +import base64 +import struct +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'tools')) +import enrich_overlay_captures as MOD # noqa: E402 + +LOAD = 0x80010000 +JR_RA = 0x03E00008 +FRAME = 0x27BDFFE0 +NOP = 0 + + +def image(words): + return b''.join(struct.pack('> 2) & 0x03ffffff) + words = [jal, NOP, JR_RA, NOP, NOP, NOP, NOP, NOP, + FRAME, NOP, JR_RA, NOP] + enriched, added = MOD.enrich_record(self.record(words)) + self.assertEqual(added, 1) + self.assertEqual(enriched['static_discovery_entry_pcs'], + [f'0x{target:08X}']) + self.assertEqual(enriched['function_entry_pcs'], []) + + def test_dense_pointer_table_recovers_proven_targets_not_table_data(self): + targets = [LOAD + 0x30, LOAD + 0x40, LOAD + 0x50] + words = [*targets] + [NOP] * 21 + for target in targets: + off = (target - LOAD) // 4 + words[off:off + 4] = [FRAME, NOP, JR_RA, NOP] + enriched, _ = MOD.enrich_record(self.record(words)) + roots = {int(value, 16) for value in enriched['static_discovery_entry_pcs']} + self.assertTrue(set(targets) <= roots) + self.assertNotIn(LOAD, roots) + + def test_return_adjacent_pointer_data_is_not_promoted(self): + words = [FRAME, NOP, JR_RA, NOP, + LOAD + 0x20, LOAD + 0x24, LOAD + 0x28, + 1, 2, 3, 4, 5] + enriched, added = MOD.enrich_record(self.record(words)) + self.assertEqual(added, 1) # only the real framed entry at LOAD + self.assertEqual(enriched['static_discovery_entry_pcs'], + [f'0x{LOAD:08X}']) + + +if __name__ == '__main__': + unittest.main()