Files
psxrecomp/tools/test_compile_overlays_static_split.py
T
Kevin 7ab698caa5 overlays: parallel static compile, linear resume-wrapper pass, split output
Static mode (--static) was serial end to end and produced one translation
unit, so a whole title compiled on one core twice: once in Python, once in
gcc. Measured on BoF3 (339 captures, 79,688 identities) in a build-dbg loop:
phase "compile all overlay bands" ~12 min, runtime build ~4 min at -O0.

Three changes, each independently verified byte-identical:

1. static_capture_job(): the per-capture static work (recompiler subprocess,
   C audit, range audit, namespacing) is now a module-level, picklable job
   that returns its part + captured log. --jobs runs it in a
   ProcessPoolExecutor; results merge in capture order, so static_parts and
   entry sources are order-identical to the sequential loop. --jobs 1 and
   --jobs 3 on three bands: diff -r of the outputs is empty.

2. add_cps_resume_cases(): the universal CPS resume-wrapper pass called
   add_cps_resume_case() once per entry, and each call re-scanned and
   rebuilt the WHOLE part source. cProfile on three bands: 7,386 entries ->
   22,159 re.search calls, 366 s of 424 s total. The batch form locates
   every signature once, edits each host's segment in isolation, rebuilds
   once. Same three bands: 418 s -> 5 s, outputs byte-identical (diff -r
   against the per-entry output). The per-entry function is kept; a unit
   test asserts the two agree on fresh-switch, existing-switch, missing
   label and missing host cases.

3. write_static_outputs(): overlays_static.c now holds the dispatcher plus an
   extern prototype per compiled entry, and every part becomes its own unit
   overlays_static_NNNN.c (each already carries its #includes and a private
   namespace). runtime.cmake globs the siblings with CONFIGURE_DEPENDS, so a
   changed part count needs no reconfigure. Unchanged units are not
   rewritten (mtime preserved -> not recompiled); stale units are removed.
   --static-single-file restores the monolithic file.

Result, same BoF3 loop on a 12-core box: phase 5a 12 min -> 20 s, runtime
build 4 min -> 69 s (358 units in parallel), whole loop ~16 min -> 90 s.
Shard summary unchanged (332 ok / 7 benign audit fails), identity count
unchanged. Headless boot of the rebuilt runtime: 485,650 static checks,
485,244 hits (99.92%), miss_total 0, 60 emu fps through the LOGO overlay.

Note: tools/test_compile_overlays_additive.py already fails on master (it
references removed functions: compile_and_publish_dll, load_additive_captures,
validated_cached_bundle_ids); untouched here.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-02 09:20:32 -04:00

214 lines
9.1 KiB
Python

"""Static-mode output split + worker contract for compile_overlays.py.
python tools/test_compile_overlays_static_split.py
"""
import os
import pickle
import sys
import tempfile
import time
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import compile_overlays # noqa: E402
def part(ns, body, symbols):
return {
'src': f'#include "psx_runtime.h"\n{body}\n',
'namespace': ns,
'variants': [{'addr': 0x80100000 + i * 4, 'symbol': s, 'crc': 1,
'ranges': ((0x00100000, 8),)} for i, s in enumerate(symbols)],
'func_addrs': set(), 'symbols': {}, 'ids_by_addr': {},
'continuation_owners': {},
}
class StaticSplitTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.out = os.path.join(self.tmp.name, 'overlays_static.c')
def tearDown(self):
self.tmp.cleanup()
def test_split_writes_dispatcher_prototypes_and_parts(self):
parts = [part('ov_a', 'void ov_a_func_80100000(CPUState *cpu) {}',
['ov_a_func_80100000']),
part('ov_b', 'void ov_b_func_80100004(CPUState *cpu) {}',
['ov_b_func_80100004'])]
variants = [v for p in parts for v in p['variants']]
written = compile_overlays.write_static_outputs(
self.out, parts, variants, '/* dispatch */\n')
self.assertEqual([os.path.basename(w) for w in written],
['overlays_static.c', 'overlays_static_0000.c',
'overlays_static_0001.c'])
main = open(self.out, encoding='utf-8').read()
self.assertIn('#include "psx_runtime.h"', main)
self.assertIn('void ov_a_func_80100000(CPUState *cpu);', main)
self.assertIn('void ov_b_func_80100004(CPUState *cpu);', main)
self.assertIn('/* dispatch */', main)
self.assertNotIn('ov_a_func_80100000(CPUState *cpu) {}', main)
p0 = open(written[1], encoding='utf-8').read()
self.assertIn('ov_a_func_80100000(CPUState *cpu) {}', p0)
self.assertIn('#include "psx_runtime.h"', p0)
self.assertNotIn('ov_b_func', p0)
def test_unchanged_part_keeps_mtime_and_stale_parts_are_removed(self):
parts = [part('ov_a', 'int a;', ['ov_a_func_80100000']),
part('ov_b', 'int b;', ['ov_b_func_80100004']),
part('ov_c', 'int c;', ['ov_c_func_80100008'])]
variants = [v for p in parts for v in p['variants']]
first = compile_overlays.write_static_outputs(
self.out, parts, variants, '/* d */\n')
self.assertEqual(len(first), 4)
old_mtime = os.stat(first[1]).st_mtime_ns
os.utime(first[1], ns=(old_mtime - 5_000_000_000,
old_mtime - 5_000_000_000))
old_mtime = os.stat(first[1]).st_mtime_ns
time.sleep(0.01)
# Drop the last part; part 0 unchanged, part 1 changed.
parts[1] = part('ov_b', 'int b2;', ['ov_b_func_80100004'])
parts = parts[:2]
variants = [v for p in parts for v in p['variants']]
second = compile_overlays.write_static_outputs(
self.out, parts, variants, '/* d */\n')
self.assertEqual(len(second), 3)
self.assertEqual(os.stat(second[1]).st_mtime_ns, old_mtime,
'unchanged part must not be rewritten')
self.assertIn('int b2;', open(second[2], encoding='utf-8').read())
self.assertFalse(os.path.exists(first[3]), 'stale part must be deleted')
self.assertEqual(sorted(os.listdir(self.tmp.name)),
['overlays_static.c', 'overlays_static_0000.c',
'overlays_static_0001.c'])
def test_single_file_mode_is_monolithic_and_cleans_parts(self):
parts = [part('ov_a', 'int a;', ['ov_a_func_80100000'])]
variants = parts[0]['variants']
compile_overlays.write_static_outputs(self.out, parts, variants, '/* d */\n')
self.assertTrue(os.path.exists(os.path.join(self.tmp.name,
'overlays_static_0000.c')))
written = compile_overlays.write_static_outputs(
self.out, parts, variants, '/* d */\n', single_file=True)
self.assertEqual(written, [self.out])
self.assertEqual(os.listdir(self.tmp.name), ['overlays_static.c'])
main = open(self.out, encoding='utf-8').read()
self.assertIn('int a;', main)
self.assertIn('/* d */', main)
self.assertNotIn('(CPUState *cpu);', main)
def test_static_part_paths_only_matches_numbered_siblings(self):
for name in ('overlays_static_0000.c', 'overlays_static_0001.c',
'overlays_static.c', 'overlays_static_x.c',
'other_0000.c'):
open(os.path.join(self.tmp.name, name), 'w').close()
found = [os.path.basename(p)
for p in compile_overlays.static_part_paths(self.out)]
self.assertEqual(found, ['overlays_static_0000.c',
'overlays_static_0001.c'])
class StaticWorkerContractTests(unittest.TestCase):
def test_worker_is_picklable_for_a_process_pool(self):
# ProcessPoolExecutor pickles the callable by qualified name; a nested
# closure would fail here. This is the guard for that regression.
fn = pickle.loads(pickle.dumps(compile_overlays.static_capture_job))
self.assertIs(fn, compile_overlays.static_capture_job)
def test_data_only_capture_reports_skip_with_captured_log(self):
import argparse
import base64
cap = {
'schema': 'psxrecomp overlay capture v2',
'load_addr': '0x80100000',
'size': 16,
'bytes_b64': base64.b64encode(b'\0' * 16).decode('ascii'),
'executed_pcs': [], 'dispatch_entry_pcs': [],
'function_entry_pcs': [],
}
args = argparse.Namespace(recompiler='unused', game_toml='unused',
cps=False, project_root=None)
res = compile_overlays.static_capture_job(cap, args, {}, set(),
'overlays_static.c')
self.assertEqual(res['outcome'], 'skip')
self.assertIsNone(res['part'])
self.assertIn('SKIP: no walk-root seeds', res['log'])
self.assertEqual(res['requested_entries'], set())
HOST_SRC = """#include "psx_runtime.h"
void ov_x_func_80100000(CPUState* cpu);
void ov_x_func_80100000(CPUState* cpu)
{
debug_server_log_call_entry(0x80100000u);
block_80100010:
cpu->pc = 1;
block_80100020:
cpu->pc = 2;
block_80100030:
cpu->pc = 3;
}
void ov_x_func_80100100(CPUState* cpu)
{
if (cpu->pc != 0u) {
uint32_t _cont = cpu->pc; cpu->pc = 0;
switch (_cont) {
case 0x80100100u: break;
default: cpu->pc = _cont; psx_native_bad_entry(cpu, 0x80100100u, _cont); return;
}
}
debug_server_log_call_entry(0x80100100u);
block_80100110:
cpu->pc = 4;
block_80100120:
cpu->pc = 5;
}
"""
class ResumeCaseBatchTests(unittest.TestCase):
"""add_cps_resume_cases must be byte-identical to looping
add_cps_resume_case entry by entry (the pre-batch implementation)."""
def _sequential(self, src, requests):
ok = {}
for host_symbol, host, entries in requests:
for entry in entries:
src, good = compile_overlays.add_cps_resume_case(
src, host_symbol, host, entry)
if good:
ok.setdefault(host_symbol, set()).add(entry)
return src, ok
def test_batch_matches_sequential_fresh_switch_and_existing_switch(self):
requests = [
('ov_x_func_80100000', 0x80100000,
[0x80100010, 0x80100020, 0x80100030, 0x80100999]), # no such block
('ov_x_func_80100100', 0x80100100, [0x80100110, 0x80100120]),
('ov_x_func_missing', 0x80100200, [0x80100210]), # no definition
]
seq_src, seq_ok = self._sequential(HOST_SRC, requests)
bat_src, bat_ok = compile_overlays.add_cps_resume_cases(HOST_SRC, requests)
self.assertEqual(bat_src, seq_src)
self.assertEqual({k: v for k, v in bat_ok.items() if v}, seq_ok)
self.assertIn('case 0x80100010u: goto block_80100010;', bat_src)
self.assertIn('case 0x80100120u: goto block_80100120;', bat_src)
self.assertNotIn('80100999', bat_src)
# Re-applying is a no-op (arms already present).
again_src, again_ok = compile_overlays.add_cps_resume_cases(bat_src, requests)
self.assertEqual(again_src, bat_src)
self.assertEqual(again_ok['ov_x_func_80100000'],
{0x80100010, 0x80100020, 0x80100030})
def test_batch_edits_are_confined_to_each_host_segment(self):
requests = [('ov_x_func_80100100', 0x80100100, [0x80100110])]
bat_src, _ = compile_overlays.add_cps_resume_cases(HOST_SRC, requests)
marker = 'void ov_x_func_80100100(CPUState* cpu)\n{'
head = bat_src[:bat_src.index(marker)]
self.assertEqual(head, HOST_SRC[:HOST_SRC.index(marker)])
if __name__ == '__main__':
unittest.main()