diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index 74ef72bf..9f7c7e00 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -869,6 +869,16 @@ if(BUILD_TESTING) --bios-recompiler $ --compiler ${CMAKE_C_COMPILER} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) + # Overlay shards keyed by segment (docs/SEGMENT_AWARE_CODE.md §5.7): + # capture views, per-segment cache directories and manifests, the real + # compile_overlays.py driver and the real overlay loader. + add_test( + NAME overlay_segment_keys + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_overlay_segment_keys.py + --recompiler $ + --compiler ${CMAKE_C_COMPILER} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) # BIOS segment variants (docs/SEGMENT_AWARE_CODE.md §5.4): OpenBIOS # with segment-qualified seeds, and SCPH-1001's KSEG1 kernel entry. add_test( diff --git a/recompiler/tests/test_aot_overlay_discovery.py b/recompiler/tests/test_aot_overlay_discovery.py index f84c9ed8..8cece023 100644 --- a/recompiler/tests/test_aot_overlay_discovery.py +++ b/recompiler/tests/test_aot_overlay_discovery.py @@ -1637,11 +1637,18 @@ def check_candidate_capacity_publication(): capacity_lock, cache_dirs = MOD._candidate_capacity_namespace(final) assert capacity_lock == os.path.join( tmp, 'GAME', '.overlay-candidate-capacity.lock') + # Each tier's leaf, then its per-segment shard directories + # (docs/SEGMENT_AWARE_CODE.md §5.7): one process-global table. assert cache_dirs == [ os.path.join(tmp, 'GAME', tier, 'win-x64', - 'cg9_a3003734_gc76b225b8') + 'cg9_a3003734_gc76b225b8', *sub) for tier in ('gcc', 'tcc') + for sub in ((), ('seg-kuseg',), ('seg-kseg1',)) ] + # A KUSEG shard counts against the same namespace and lock. + assert MOD._candidate_capacity_namespace(os.path.join( + leaf, 'seg-kuseg', '00010000_00000001.dll')) == ( + capacity_lock, cache_dirs) with tempfile.TemporaryDirectory() as tmp: first = os.path.join(tmp, '00010000_00000001.dll') diff --git a/recompiler/tests/test_coverage_vault.py b/recompiler/tests/test_coverage_vault.py index 332396da..d76ad665 100644 --- a/recompiler/tests/test_coverage_vault.py +++ b/recompiler/tests/test_coverage_vault.py @@ -106,6 +106,54 @@ class CoverageVaultHistoryTests(unittest.TestCase): self.assertEqual(merged['static_dispatch_entry_pcs'], ['0x80010004']) + def test_capture_merge_keeps_dispatch_segments(self): + # docs/SEGMENT_AWARE_CODE.md §5.7: a v3 record names the segments each + # dispatch entry entered through; a v2 record's entries are KSEG0. + with tempfile.TemporaryDirectory() as tmp: + vault = os.path.join(tmp, 'captures.json') + v2 = { + 'load_addr': '0x80010000', 'size': 8, + 'bytes_b64': 'AAAAAAAAAAA=', + 'dispatch_entry_pcs': ['0x80010000'], + } + with open(vault, 'w', encoding='utf-8') as out: + json.dump([v2], out) + v3 = dict(v2, dispatch_entry_pcs=['0x80010000', '0x80010004'], + dispatch_entry_segments={ + 'kuseg': ['0x00010000', '0x00010004'], + 'kseg0': [], 'kseg1': ['0xA0010004']}) + self.assertEqual(MOD.merge_capture_regions(vault, [v3]), (0, 0)) + with open(vault, encoding='utf-8') as source: + merged = json.load(source)[0] + self.assertEqual(merged['dispatch_entry_pcs'], + ['0x80010000', '0x80010004']) + self.assertEqual(merged['dispatch_entry_segments'], { + 'kuseg': ['0x00010000', '0x00010004'], + 'kseg0': ['0x80010000'], + 'kseg1': ['0xA0010004']}) + # v2 into v2 stays v2. + with open(vault, 'w', encoding='utf-8') as out: + json.dump([v2], out) + MOD.merge_capture_regions(vault, [dict(v2)]) + with open(vault, encoding='utf-8') as source: + self.assertNotIn('dispatch_entry_segments', json.load(source)[0]) + + def test_compaction_crops_dispatch_segments(self): + image = bytes(0x3004) + region = { + 'load_addr': '0x80010000', 'size': len(image), + 'bytes_b64': base64.b64encode(image).decode('ascii'), + 'executed_pcs': ['0x80010000', '0x80012000'], + 'dispatch_entry_pcs': ['0x80010000', '0x80012000'], + 'dispatch_entry_segments': { + 'kuseg': ['0x00010000'], 'kseg0': ['0x80012000'], + 'kseg1': ['0xA0012000']}, + } + runs, _entries, _invalid = MOD._compact_region(region) + self.assertEqual([r['dispatch_entry_segments'] for r in runs], [ + {'kuseg': ['0x00010000'], 'kseg0': [], 'kseg1': []}, + {'kuseg': [], 'kseg0': ['0x80012000'], 'kseg1': ['0xA0012000']}]) + def test_cache_merge_preserves_resident_sidecar(self): with tempfile.TemporaryDirectory() as tmp: src = os.path.join(tmp, 'source') diff --git a/recompiler/tests/test_overlay_segment_keys.py b/recompiler/tests/test_overlay_segment_keys.py new file mode 100644 index 00000000..2598af1e --- /dev/null +++ b/recompiler/tests/test_overlay_segment_keys.py @@ -0,0 +1,651 @@ +#!/usr/bin/env python3 +"""Overlay shards keyed by segment, end to end (docs/SEGMENT_AWARE_CODE.md §5.7). + +Overlay bytes are segment-free; execution is not. A PC's segment is part of +every link, EPC, I-cache tag and store-PC stamp a compiled body bakes, so rollout +PR E keys overlay shards by segment: + + - capture schema v3 records the segments each dispatch entry entered through + (`dispatch_entry_segments`); a capture without it is KSEG0 only; + - compile_overlays.py builds one shard per (region, segment with entries), + compiled at `segment | phys`, named and exported by that VA; + - KSEG0 shards keep their place and names in the cache tag directory, and + KUSEG/KSEG1 shards go in its seg-kuseg/ and seg-kseg1/ subdirectories + under the same {phys}_{crc} grammar (decision 4); their manifests carry + `S `; + - the loader runs a shard only for PCs in its own segment. + +This test drives the real pieces: + 1. capture_segment_views() on v2 and v3 records, and its refusals; + 2. the manifest writer and the runtime-contract parser (S record); + 3. the per-segment cache directories and candidate-capacity namespace; + 4. compile_overlays.py's real driver, in-process, on a v3 capture of a small + overlay entered at KUSEG and KSEG1 (the real psxrecomp-game compiles each + view; the C compiler builds the shards): the files land in their segment + directories, every PC each shard bakes is in its segment, a KSEG1 shard + charges a fetch before every instruction, a second run is served from the + cache per segment, the KSEG0 view of a v3 capture compiles exactly as the + same bytes captured as v2, and --static emits per-segment namespaces and + exact rows, including an isolated fragment per segment of one entry; + a record with execution evidence only compiles as its v2 reading, and a + --force-interior PC gets its KSEG0 view where no KSEG0 entry was seen; + each segment's shard reads only its own segment's prior manifest and + fragments; + 5. the real overlay loader (runtime/tests/overlay_pair_dedup_harness.c) on + those shards: each segment's PC runs its own shard through its CPS + continuations, links and stamps in its segment, and the fetch tags it hands + the I-cache model (runtime/src/psx_icache.c) cost exactly what Beetle's + fetch model charges for the instructions it executed; a KSEG0 PC, which + has no shard, is interpreted. + +Usage: python test_overlay_segment_keys.py --recompiler + [--compiler cc] +Exit 0 = PASS. +""" +from __future__ import annotations + +import argparse +import base64 +import contextlib +import importlib.util +import io +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +from unittest.mock import patch + +HERE = pathlib.Path(__file__).resolve().parent +ROOT = HERE.parents[1] +RUNTIME = ROOT / "runtime" +sys.path.insert(0, str(ROOT / "tools")) +sys.path.insert(0, str(RUNTIME / "tests")) + +import compile_overlays as co # noqa: E402 +import test_overlay_pair_dedup_runtime as pair # noqa: E402 + +KUSEG, KSEG0, KSEG1 = 0x00000000, 0x80000000, 0xA0000000 +PHYS = 0x000A0000 +GAME_TOML = ROOT / "tools" / "cycle_testrom" / "game.toml" +GAME_ID = "CYCT-00101" + + +def load(name: str, path: pathlib.Path): + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +GEN = load("gen_segment_exe", ROOT / "tools" / "segment_testrom" / "gen_segment_exe.py") +LEDGER = load("segment_ledger", HERE / "test_segment_aware_codegen.py") + +FAILURES: list[str] = [] + + +def check(cond: bool, what: str) -> None: + if not cond: + FAILURES.append(what) + print(" FAIL: " + what) + + +# --------------------------------------------------------------------------- +# The overlay: a framed function that calls a leaf with jal (a link in its +# segment), runs a straight line, stores and reloads $ra (a store-PC stamp in +# its segment) and returns. Its bytes are the same in every segment. +# --------------------------------------------------------------------------- +def overlay_image(): + a = GEN.Asm(KSEG0 | PHYS) + a.label("ov_run") + a.emit(GEN.addiu("sp", "sp", -8), GEN.sw("ra", 4, "sp")) + a.jal("ov_getpc") + a.emit(GEN.nop()) + a.label("ov_after") # the jal's return point + a.emit(*[GEN.addiu("t3", "t3", 1) for _ in range(4)]) + a.emit(GEN.lw("ra", 4, "sp"), GEN.addiu("sp", "sp", 8), + GEN.jr("ra"), GEN.nop()) + a.label("ov_getpc") + a.emit(GEN.jr("ra"), GEN.addu("v1", "ra", "zero")) + a.label("ov_end") + body = a.resolve() + labels = {k: v & co.PHYS_MASK for k, v in a.labels.items()} + return body + b"\0" * (0x1000 - len(body)), labels + + +def executed_sequence(labels): + """Physical PCs in the order one call executes them.""" + run = list(range(labels["ov_run"], labels["ov_after"], 4)) + leaf = list(range(labels["ov_getpc"], labels["ov_end"], 4)) + rest = list(range(labels["ov_after"], labels["ov_getpc"], 4)) + return run + leaf + rest + + +def capture(data, labels, segments=None, *, v2=False): + """A runtime-shaped capture record (overlay_capture.c write_json_window).""" + load_addr = KSEG0 | PHYS + executed = [f"0x{KSEG0 | pc:08X}" for pc in executed_sequence(labels)] + cap = { + "schema": "psxrecomp overlay capture v2" if v2 else + "psxrecomp overlay capture v3", + "load_addr": f"0x{load_addr:08X}", + "size": len(data), + "guard_bytes": 0, + "bytes_b64": base64.b64encode(data).decode(), + "executed_pcs": sorted(executed), + "dispatch_entry_pcs": [f"0x{load_addr:08X}"], + "function_entry_pcs": [], + "seeds": [f"0x{load_addr:08X}"], + } + if not v2: + cap["dispatch_entry_segments"] = { + name: [f"0x{seg | PHYS:08X}"] if seg in segments else [] + for name, seg in (("kuseg", KUSEG), ("kseg0", KSEG0), ("kseg1", KSEG1)) + } + return cap + + +def raw_capture(data, executed_phys, entries, function_entries=()): + """A v3 record from explicit evidence: `entries` maps a segment to the + physical dispatch entries seen there.""" + load_addr = KSEG0 | PHYS + dispatched = sorted({pc for pcs in entries.values() for pc in pcs}) + return { + "schema": "psxrecomp overlay capture v3", + "load_addr": f"0x{load_addr:08X}", + "size": len(data), + "guard_bytes": 0, + "bytes_b64": base64.b64encode(data).decode(), + "executed_pcs": [f"0x{KSEG0 | pc:08X}" for pc in sorted(executed_phys)], + "dispatch_entry_pcs": [f"0x{KSEG0 | pc:08X}" for pc in dispatched], + "dispatch_entry_segments": { + name: [f"0x{seg | pc:08X}" for pc in sorted(entries.get(seg, ()))] + for name, seg in (("kuseg", KUSEG), ("kseg0", KSEG0), ("kseg1", KSEG1))}, + "function_entry_pcs": [f"0x{KSEG0 | pc:08X}" for pc in function_entries], + "seeds": [f"0x{KSEG0 | pc:08X}" for pc in dispatched], + } + + +def fragment_image(): + """A root function, then an entry with no callable boundary (the probe's + position-independent ov_run shape): it compiles only as an isolated + fragment.""" + a = GEN.Asm(KSEG0 | PHYS) + a.label("root") + a.emit(GEN.jr("ra"), GEN.nop()) + a.label("pi") + a.emit(GEN.addu("t9", "ra", "zero")) + a.bgezal("zero", "pi_leaf") + a.emit(GEN.nop()) + a.emit(*[GEN.addiu("t3", "t3", 1) for _ in range(4)]) + a.emit(GEN.jr("t9"), GEN.addu("v0", "t9", "zero")) + a.label("pi_leaf") + a.emit(GEN.jr("ra"), GEN.addu("v1", "ra", "zero")) + a.label("end") + body = a.resolve() + labels = {k: v & co.PHYS_MASK for k, v in a.labels.items()} + return body + b"\0" * (0x1000 - len(body)), labels + + +# --------------------------------------------------------------------------- +# 1. capture views +# --------------------------------------------------------------------------- +def check_views(data, labels): + print("capture views") + v2 = capture(data, labels, v2=True) + check(co.capture_segment_views(v2) == [v2] and co.capture_segment_views(v2)[0] is v2, + "a v2 capture is returned unchanged") + v3 = capture(data, labels, {KUSEG, KSEG1}) + v3["static_dispatch_entry_pcs"] = [f"0x{KSEG0 | PHYS:08X}"] + v3["producer_ranges"] = [{"start": f"0x{KSEG0 | PHYS:08X}", + "end": f"0x{KSEG0 | PHYS + 0x40:08X}"}] + views = co.capture_segment_views(v3) + check([v["load_addr"] for v in views] == [f"0x{KUSEG | PHYS:08X}", + f"0x{KSEG1 | PHYS:08X}"], + "one view per segment with entries, spelled in it") + for v in views: + seg = int(v["load_addr"], 16) & co.SEGMENT_MASK + check("dispatch_entry_segments" not in v, "views carry no segment record") + check(v["dispatch_entry_pcs"] == [f"0x{seg | PHYS:08X}"], + f"{seg:08X}: dispatch entries are the segment's own") + check(all(int(pc, 16) & co.SEGMENT_MASK == seg for pc in v["executed_pcs"]) and + len(v["executed_pcs"]) == len(v3["executed_pcs"]), + f"{seg:08X}: execution evidence carried whole, respelled") + check(v["static_dispatch_entry_pcs"] == v["dispatch_entry_pcs"], + f"{seg:08X}: static dispatch entries stay a subset") + check(v["producer_ranges"][0]["start"] == f"0x{seg | PHYS:08X}", + f"{seg:08X}: producer ranges respelled") + check(v["bytes_b64"] == v3["bytes_b64"] and v["size"] == v3["size"], + f"{seg:08X}: the bytes are shared") + # An entry that no segment list names is KSEG0, as in v2. + mixed = capture(data, labels, {KUSEG}) + mixed["dispatch_entry_pcs"].append(f"0x{KSEG0 | PHYS + 0x30:08X}") + views = co.capture_segment_views(mixed) + check([v["dispatch_entry_pcs"] for v in views] == + [[f"0x{KSEG0 | PHYS + 0x30:08X}"], [f"0x{KUSEG | PHYS:08X}"]], + "an unlabelled dispatch entry is a KSEG0 entry") + # Execution evidence only (the runtime writes such regions): one KSEG0 + # view, read as a v2 reader reads the record. + bare = capture(data, labels, set()) + bare["dispatch_entry_pcs"] = bare["seeds"] = [] + views = co.capture_segment_views(bare) + as_v2 = {k: v for k, v in bare.items() if k != "dispatch_entry_segments"} + check(len(views) == 1 and views[0] == as_v2, + "a record with no dispatch entry is one KSEG0 view, as in v2") + # A demand from outside the capture names a segment it has no entry in. + views = co.capture_segment_views(capture(data, labels, {KUSEG}), {KSEG0}) + check([(v["load_addr"], v["dispatch_entry_pcs"]) for v in views] == + [(f"0x{KSEG0 | PHYS:08X}", []), + (f"0x{KUSEG | PHYS:08X}", [f"0x{KUSEG | PHYS:08X}"])], + "a required segment gets a view without entries of its own") + inside = KSEG0 | PHYS + 0x10 + declared = co.declared_view_segments + check(declared(v3, {}, {inside}) == {KSEG0} and + declared(v3, {}, {KSEG0 | PHYS + len(data)}) == set(), + "a --force-interior PC inside the record asks for its KSEG0 view") + check(declared(v3, {"overlays": [{"load_addr": f"0x{KUSEG | PHYS:08X}", + "entries": [f"0x{KUSEG | PHYS:08X}"]}]}) == {KUSEG} and + declared(v3, {"overlays": [{"load_addr": f"0x{KUSEG | PHYS:08X}"}]}) == set() and + declared(v3, {"overlays": [{"entry": f"0x{KSEG1 | PHYS + 8:08X}"}]}) == {KSEG1} and + declared(v3, {"overlays": [{"entry": f"0x{KSEG1 | PHYS + 0x2000:08X}"}]}) == set() and + declared(v3, {"overlays": [{"entry": f"0x{0xC0000000 | PHYS:08X}"}]}) == set(), + "a game.toml [[overlays]] table names the segment it spells, inside the record") + for bad, why in ((dict(v3, dispatch_entry_segments={"kseg2": []}), "unknown name"), + (dict(v3, dispatch_entry_segments={"kuseg": [f"0x{KSEG1 | PHYS:08X}"]}), + "PC in another segment"), + (dict(v3, dispatch_entry_segments=["0x000A0000"]), "not an object")): + try: + co.capture_segment_views(bad) + check(False, f"malformed segment record accepted ({why})") + except RuntimeError: + pass + + +# --------------------------------------------------------------------------- +# 2. manifests +# --------------------------------------------------------------------------- +def check_manifests(): + print("manifests") + ranges = [(KSEG0 | PHYS, 0x30)] + k0 = co.overlay_ranges_text([(KSEG0 | PHYS, 0x1234, ranges)], 0xABC) + check(k0 == ("# psxrecomp overlay code-range manifest v2 (entry+code_crc)\n" + "P 0000000000000ABC\nF 800A0000 00001234\nR 800A0000 30\n"), + "a KSEG0 manifest is the pre-§5.7 text") + ku = co.overlay_ranges_text([(KUSEG | PHYS, 0x1234, ranges)], 0xABC) + check(ku.splitlines()[1:4] == ["S 00000000", "P 0000000000000ABC", + "F 000A0000 00001234"] and + "R 800A0000 30" in ku, + "a KUSEG manifest: S first, full-VA F, KSEG0-spelled R") + try: + co.overlay_ranges_text([(KUSEG | PHYS, 1, ranges), (KSEG1 | PHYS + 8, 1, ranges)]) + check(False, "a manifest with two segments was written") + except ValueError: + pass + pid, funcs = co.parse_runtime_shard_manifest(ku, expected_segment=KUSEG) + check(pid == 0xABC and funcs and funcs[0][0] == KUSEG | PHYS, + "the parser keeps a KUSEG entry's full VA") + pid, funcs = co.parse_runtime_shard_manifest(k0, expected_segment=KSEG0) + check(funcs and funcs[0][0] == KSEG0 | PHYS, "a manifest without S is KSEG0") + for text, seg, why in ( + (ku, KSEG0, "KUSEG manifest in the KSEG0 directory"), + (k0, KUSEG, "KSEG0 manifest in seg-kuseg/"), + (ku.replace("F 000A0000", "F 800A0000"), KUSEG, "F outside S"), + (k0 + "S 80000000\n", None, "S after F"), + ("S 00000000\n" + ku, None, "S twice"), + (ku.replace("S 00000000", "S C0000000"), None, "S in KSEG2"), + (ku.replace("S 00000000", "S 00010000"), None, "S not a segment base")): + check(co.parse_runtime_shard_manifest(text, expected_segment=seg) == (None, []), + f"parser refuses {why}") + + +# --------------------------------------------------------------------------- +# 3. cache layout +# --------------------------------------------------------------------------- +def check_layout(): + print("cache layout") + leaf = os.path.join("C", "GAME", "gcc", "macos-arm64", "cg13_00000000_gc00000000_f0") + check(co.segment_cache_dir(leaf, KSEG0) == leaf and + co.segment_cache_dir(leaf, KUSEG) == os.path.join(leaf, "seg-kuseg") and + co.segment_cache_dir(leaf, KSEG1) == os.path.join(leaf, "seg-kseg1"), + "KSEG0 keeps the leaf; KUSEG/KSEG1 use seg-kuseg/ and seg-kseg1/") + check([co.cache_dir_segment(co.segment_cache_dir(leaf, s)) + for s in (KUSEG, KSEG0, KSEG1)] == [KUSEG, KSEG0, KSEG1], + "a directory names its segment") + lock, dirs = co._candidate_capacity_namespace(os.path.join(leaf, "00010000_00000001.so")) + lock_k, dirs_k = co._candidate_capacity_namespace( + os.path.join(leaf, "seg-kseg1", "00010000_00000001.so")) + check((lock, dirs) == (lock_k, dirs_k) and + os.path.abspath(os.path.join(leaf, "seg-kuseg")) in dirs and + os.path.abspath(os.path.join(leaf, "seg-kseg1")) in dirs, + "every segment directory shares one candidate namespace") + + +# --------------------------------------------------------------------------- +# 4. compile_overlays.py, in-process +# --------------------------------------------------------------------------- +def run_compile(recompiler, compiler, captures, out_dir, extra=(), toml=GAME_TOML): + """compile_overlays.main() on `captures`. The cache-tag stamp comes from a + runtime build this test does not need: the recompiler/stamp match and the + config hash are not under test here, so they are pinned.""" + cap_path = os.path.join(out_dir, "captures.json") + with open(cap_path, "w") as f: + json.dump(captures, f) + argv = ["compile_overlays.py", "--captures", cap_path, + "--game-toml", str(toml), "--recompiler", recompiler, + "--runtime-include", str(RUNTIME / "include"), + "--out-dir", out_dir, "--gcc", compiler, "--jobs", "1", *extra] + env = {k: v for k, v in os.environ.items() + if k not in ("PSX_OVERLAY_CACHE_DIR", "PSX_OVERLAY_CAPTURES", + "PSX_OVERLAY_FLAVOR")} + log = io.StringIO() + code = 0 + with patch.object(co, "verify_recompiler_matches_tag", lambda *a: None), \ + patch.object(co, "overlay_config_hash", lambda *a: 0), \ + patch.object(sys, "argv", argv), patch.dict(os.environ, env, clear=True), \ + contextlib.redirect_stdout(log): + try: + co.main() + except SystemExit as exc: + code = exc.code or 0 + return code, log.getvalue() + + +def leaf_dir(cache_root): + with patch.object(co, "overlay_config_hash", lambda *a: 0): + tag = co.cache_tag(str(RUNTIME / "include"), "", str(GAME_TOML), 0) + return os.path.join(cache_root, GAME_ID, "gcc", co.cache_arch_abi(), tag) + + +def manifest_entries(directory): + """Every F entry of every manifest in `directory`.""" + out = set() + for name in os.listdir(directory): + if name.endswith(".ranges"): + with open(os.path.join(directory, name)) as f: + out |= {int(ln.split()[1], 16) for ln in f if ln.startswith("F ")} + return out + + +def functions(c_text): + return {m.group(1): m.group(2) for m in re.finditer( + r"^void (func_[0-9A-F]{8})\(CPUState\* cpu\)\n\{(.*?)^\}", c_text, re.M | re.S)} + + +def check_compile(recompiler, compiler, data, labels, tmp): + print("compile_overlays.py") + crc = __import__("binascii").crc32(data) & 0xFFFFFFFF + stem = f"{PHYS:08X}_{crc:08X}" + cache = os.path.join(tmp, "cache") + os.makedirs(cache) + code, log = run_compile(recompiler, compiler, + [capture(data, labels, {KUSEG, KSEG1})], cache) + check(code == 0 and "built OK : 2" in log, "both views build") + leaf = leaf_dir(cache) + ext = co.overlay_ext() + check(not os.path.exists(os.path.join(leaf, stem + ext)), + "no KSEG0 shard: no entry was dispatched at KSEG0") + shards = {} + for seg, sub in ((KUSEG, "seg-kuseg"), (KSEG1, "seg-kseg1")): + d = os.path.join(leaf, sub) + check(os.path.isfile(os.path.join(d, stem + ext)), f"{sub}/{stem}{ext} exists") + with open(os.path.join(d, stem + ".ranges")) as f: + manifest = f.read() + check(f"S {seg:08X}\n" in manifest and + f"F {seg | labels['ov_run']:08X} " in manifest and + f"F {seg | labels['ov_getpc']:08X} " in manifest, + f"{sub}: manifest names its segment and full-VA entries") + with open(os.path.join(d, f"{crc:08X}_patched.c")) as f: + c_text = f.read() + bodies = functions(c_text) + check(set(bodies) == {f"func_{seg | labels['ov_run']:08X}", + f"func_{seg | labels['ov_getpc']:08X}"}, + f"{sub}: functions are named by their VA in the segment") + foreign = sorted({pc for body in bodies.values() + for gid, pattern in LEDGER.PC_SITES.items() + for pc in LEDGER.pc_constants(body, pattern) + if pc & co.SEGMENT_MASK != seg}) + check(not foreign, f"{sub}: every baked PC is in the segment" + + (f" (e.g. 0x{foreign[0]:08X})" if foreign else "")) + stamps = {pc for body in bodies.values() + for pc in LEDGER.pc_constants(body, LEDGER.PC_SITES["store-pc-segment"])} + check(stamps == {seg | (labels["ov_run"] + 4)}, + f"{sub}: the store-PC stamp is the store's PC in the segment") + shards[seg] = bodies + # KSEG1: a fetch before every instruction; KUSEG: line leaders only. + k1 = [pc for body in shards[KSEG1].values() + for pc in LEDGER.pc_constants(body, LEDGER.PC_SITES["fetch-tag-segment"])] + check(sorted(k1) == sorted(KSEG1 | pc for pc in executed_sequence(labels)), + "KSEG1 shard charges a fetch before every instruction") + ku = [pc for body in shards[KUSEG].values() + for pc in LEDGER.pc_constants(body, LEDGER.PC_SITES["fetch-tag-segment"])] + check(len(ku) < len(k1), "KUSEG shard keeps the cached line-leader rule") + + code, log = run_compile(recompiler, compiler, + [capture(data, labels, {KUSEG, KSEG1})], cache) + check(code == 0 and log.count("SKIP: runtime-valid current-byte cache serves") == 2, + "a second run is served from each segment's cache") + + # The KSEG0 view of a v3 capture compiles exactly as the same bytes as v2. + outs = {} + for name, cap in (("v2", capture(data, labels, v2=True)), + ("v3", capture(data, labels, {KSEG0}))): + out = os.path.join(tmp, "cache-" + name) + os.makedirs(out) + code, _log = run_compile(recompiler, compiler, [cap], out) + d = leaf_dir(out) + with open(os.path.join(d, stem + ".ranges")) as f: + manifest = f.read() + with open(os.path.join(d, f"{crc:08X}_patched.c")) as f: + outs[name] = (code, manifest, f.read()) + check(outs["v2"] == outs["v3"] and outs["v2"][0] == 0 and + "\nS " not in outs["v2"][1], + "a KSEG0 view compiles byte-identically to the v2 capture") + + # --static: per-segment namespaces and exact rows beside KSEG0 ones. + static_dir = os.path.join(tmp, "static") + os.makedirs(static_dir) + code, log = run_compile(recompiler, compiler, + [capture(data, labels, {KUSEG, KSEG0, KSEG1})], + static_dir, ("--static", "--static-single-file")) + with open(os.path.join(static_dir, "overlays_static.c")) as f: + static_c = f.read() + rows = {int(a, 16) for a in re.findall( + r"^ \{ 0x([0-9A-F]{8})u, \d+u, \d+u \},$", static_c, re.M)} + check(code == 0 and {seg | labels["ov_run"] for seg in (KUSEG, KSEG0, KSEG1)} <= rows, + "static dispatch has an exact row per segment") + check(f"ov_s0_{PHYS:08X}_" in static_c and f"ov_s5_{PHYS:08X}_" in static_c and + f"ov_{PHYS:08X}_" in static_c, + "static namespaces: ov_s0_ (KUSEG), ov_s5_ (KSEG1), ov_ (KSEG0)") + check("(addr & 0xE0000000u) != 0x00000000u" in static_c and + "(addr & 0xE0000000u) != 0xA0000000u" in static_c, + "the static segment gate admits every compiled segment") + return cache + + +def check_demands(recompiler, compiler, data, labels, tmp): + """Records and demands a view split must not drop (§5.7).""" + print("compile_overlays.py: demands without a dispatch entry") + run, after = KSEG0 | labels["ov_run"], KSEG0 | labels["ov_after"] + # Execution evidence only, plus an operator's --force-interior: the same + # fragment as the same bytes captured as v2. + outs = {} + for name, v2 in (("v2", True), ("v3", False)): + cap = capture(data, labels, set(), v2=v2) + cap["dispatch_entry_pcs"] = cap["seeds"] = [] + out = os.path.join(tmp, "bare-" + name) + os.makedirs(out) + code, log = run_compile(recompiler, compiler, [cap], out, + ("--force-interior", f"0x{after:08X}")) + d = leaf_dir(out) + outs[name] = (code, "built OK : 1" in log, sorted(os.listdir(d)), + {n: open(os.path.join(d, n), "rb").read() + for n in os.listdir(d) + if n.endswith((".ranges", "_patched.c"))}) + check(outs["v2"][0] == 0 and outs["v2"][1] and outs["v2"][2] and + outs["v3"] == outs["v2"], + "a dispatch-less v3 record compiles byte-identically to its v2 reading") + # A KUSEG-only record: --force-interior (a KSEG0 PC) still builds its + # KSEG0 fragment next to the KUSEG shard. + out = os.path.join(tmp, "forced-kuseg") + os.makedirs(out) + code, log = run_compile(recompiler, compiler, [capture(data, labels, {KUSEG})], out, + ("--force-interior", f"0x{run:08X}")) + d = leaf_dir(out) + check(code == 0 and run in manifest_entries(d) and + KUSEG | labels["ov_run"] in manifest_entries(os.path.join(d, "seg-kuseg")), + "a forced KSEG0 interior of a KUSEG-only record is compiled at KSEG0") + + +def check_static_fragments(recompiler, compiler, tmp): + """--static: one entry with no callable boundary, dispatched in two + segments, gets an isolated fragment in each (the image a demand compiles + against is its own segment's view).""" + print("compile_overlays.py --static: isolated fragments per segment") + data, labels = fragment_image() + crc = __import__("binascii").crc32(data) & 0xFFFFFFFF + pi = labels["pi"] + executed = set(range(labels["root"], labels["end"], 4)) + cap = raw_capture(data, executed, {KSEG0: {labels["root"]}, KUSEG: {pi}, KSEG1: {pi}}, + function_entries=(labels["root"],)) + out = os.path.join(tmp, "static-frag") + os.makedirs(out) + code, log = run_compile(recompiler, compiler, [cap], out, + ("--static", "--static-single-file")) + with open(os.path.join(out, "overlays_static.c")) as f: + static_c = f.read() + rows = {int(a, 16) for a in re.findall( + r"^ \{ 0x([0-9A-F]{8})u, \d+u, \d+u \},$", static_c, re.M)} + check(code == 0 and "SHARD FAIL" not in log, "every isolated fragment builds") + for seg in (KUSEG, KSEG1): + check(seg | pi in rows and + f"ov_frag_{PHYS:08X}_{crc:08X}_{seg | pi:08X}" in static_c, + f"{seg:08X}: the entry has its own isolated fragment and row") + check(KSEG0 | labels["root"] in rows, "the KSEG0 root has its row") + + +def check_segment_isolation(recompiler, compiler, data, labels, tmp): + """Each segment's shard reads only its own segment's cache: a KSEG0 + manifest and KSEG0 fragments of the same region never feed or serve a + KUSEG/KSEG1 view.""" + print("compile_overlays.py: per-segment prior manifests and fragments") + crc = __import__("binascii").crc32(data) & 0xFFFFFFFF + stem = f"{PHYS:08X}_{crc:08X}" + out = os.path.join(tmp, "prior") + os.makedirs(out) + leaf = leaf_dir(out) + os.makedirs(leaf) + # A KSEG0 manifest of these bytes that names an entry no KUSEG or KSEG1 + # capture saw. The prior-manifest reclassification merges only a view's + # own segment's manifest. + extra = KSEG0 | labels["ov_after"] + with open(os.path.join(leaf, stem + ".ranges"), "w") as f: + f.write(co.overlay_ranges_text( + [(KSEG0 | labels["ov_run"], 1, [(KSEG0 | labels["ov_run"], 4)]), + (extra, 1, [(extra, 4)])])) + code, log = run_compile(recompiler, compiler, + [capture(data, labels, {KUSEG, KSEG1})], out) + check(code == 0 and "built OK : 2" in log and "prior-manifest" not in log, + "a KUSEG/KSEG1 view merges no KSEG0 manifest") + for seg, sub in ((KUSEG, "seg-kuseg"), (KSEG1, "seg-kseg1")): + check(manifest_entries(os.path.join(leaf, sub)) == + {seg | labels["ov_run"], seg | labels["ov_getpc"]}, + f"{sub}: roots are the segment's own entries") + # Empty-primary reconciliation counts only the view's own segment's + # fragments as serving its roots. + k0_leaf = leaf_dir(os.path.join(tmp, "cache-v2")) # a KSEG0 shard only + both_leaf = leaf_dir(os.path.join(tmp, "cache")) # KUSEG and KSEG1 shards + abi = co.overlay_abi_tag(str(RUNTIME / "include"), 0) + for seg, cache, served in ((KUSEG, k0_leaf, False), (KSEG0, k0_leaf, True), + (KUSEG, both_leaf, True), (KSEG1, both_leaf, True)): + stats = co.ShardStats() + pending = [(f"view {seg:08X}", PHYS, seg | PHYS, len(data), data, + {seg | labels["ov_run"]})] + with contextlib.redirect_stdout(io.StringIO()): + co.reconcile_empty_primary_scans(pending, cache, abi, stats) + check((stats.skipped, stats.total_fail()) == ((1, 0) if served else (0, 1)), + f"{seg:08X} roots {'are' if served else 'are not'} served by " + f"{os.path.basename(os.path.dirname(cache + os.sep))}'s own-segment shards") + + +# --------------------------------------------------------------------------- +# 5. the loader +# --------------------------------------------------------------------------- +def check_loader(compiler, data, labels, cache, tmp): + print("overlay loader") + harness = pathlib.Path(tmp) / ("harness" + (".exe" if os.name == "nt" else "")) + pair.compile_harness(compiler, harness) + # The harness boots the loader for game "PAIR-TEST". + root = pathlib.Path(tmp) / "hcache" + shutil.copytree(os.path.join(cache, GAME_ID), root / pair.GAME) + image = pathlib.Path(tmp) / "image.bin" + image.write_bytes(data) + out = subprocess.run([str(harness), str(root), "segment-run", str(image), + f"{PHYS:X}"], capture_output=True, text=True) + check(out.returncode == 0, "harness runs: " + out.stderr.strip()) + runs = {} + for line in out.stdout.splitlines(): + if line.startswith("seg="): + fields = dict(item.split("=", 1) for item in line.split()) + runs[int(fields["seg"], 16)] = fields + elif line.startswith("segment_native="): + totals = dict(item.split("=", 1) for item in line.split()) + icache = LEDGER.ICacheModel(compiler, tmp) + after = labels["ov_after"] + for seg in (KUSEG, KSEG1): + r = runs.get(seg, {}) + check(r.get("interp_at") == "00000000" and r.get("end") == "0000BEE0", + f"{seg:08X}: every transfer ran natively, back to the caller") + check(r.get("v1") == f"{seg | after:08X}", + f"{seg:08X}: the jal link is in the segment (v1={r.get('v1')})") + check(r.get("t3") == "4" and r.get("sp") == "001FF000", + f"{seg:08X}: the body ran whole") + check(r.get("store_pc") == f"{seg | (labels['ov_run'] + 4):08X}", + f"{seg:08X}: the store-PC stamp the host saw is in the segment") + tags = [int(t, 16) for t in r.get("fetch", "").split(",") if t] + check(tags and all(t & co.SEGMENT_MASK == seg for t in tags), + f"{seg:08X}: every fetch tag is in the segment") + want = LEDGER.beetle_cycles([seg | pc for pc in executed_sequence(labels)]) + got = icache.cycles([tags])[0] if tags else -1 + check(got == want, f"{seg:08X}: fetch cycles {got} == Beetle's {want}") + k0 = runs.get(KSEG0, {}) + check(k0.get("native") == "0" and k0.get("interp_at") == f"{KSEG0 | PHYS:08X}", + "KSEG0 PC has no shard: interpreted") + check(totals.get("segment_native") == "6" and + totals.get("segment_alias_interp") == "0", + "KUSEG/KSEG1 dispatches all native; a KSEG0 miss is no alias") + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--recompiler", + default=str(ROOT / "recompiler" / "build" / "psxrecomp-game")) + ap.add_argument("--compiler", default=os.environ.get("CC") or + shutil.which("cc") or shutil.which("gcc")) + args = ap.parse_args() + if not os.path.isfile(args.recompiler): + raise SystemExit(f"recompiler not found: {args.recompiler}") + if not args.compiler: + raise SystemExit("a C compiler is required") + data, labels = overlay_image() + check_views(data, labels) + check_manifests() + check_layout() + with tempfile.TemporaryDirectory(prefix="psx-overlay-segment-", + ignore_cleanup_errors=True) as tmp: + cache = check_compile(args.recompiler, args.compiler, data, labels, tmp) + check_demands(args.recompiler, args.compiler, data, labels, tmp) + check_static_fragments(args.recompiler, args.compiler, tmp) + check_segment_isolation(args.recompiler, args.compiler, data, labels, tmp) + check_loader(args.compiler, data, labels, cache, tmp) + if FAILURES: + print(f"FAIL: {len(FAILURES)} check(s)") + return 1 + print("PASS: overlay shards are compiled, cached and dispatched per segment") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index 105407e4..7f9aade8 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -24,6 +24,7 @@ import argparse import itertools import io import contextlib +import contextvars import base64 import binascii from bisect import bisect_left @@ -259,6 +260,207 @@ def cache_arch_abi() -> str: return f'{os_tag}-{arch}' +# --------------------------------------------------------------------------- +# Code segments (docs/SEGMENT_AWARE_CODE.md §5.7) +# --------------------------------------------------------------------------- +# The bytes of an overlay are segment-free; execution is not. A PC's segment +# (KUSEG 0x0xxxxxxx, KSEG0 0x8xxxxxxx, KSEG1 0xAxxxxxxx) is part of every link, +# EPC, I-cache tag and store-PC stamp a compiled body bakes, so a shard serves +# only the segment it was compiled for. Capture schema v3 records the segments +# each dispatch entry entered through (`dispatch_entry_segments`); a capture +# without it is KSEG0 only. This tool compiles one shard per (region, segment +# with entries): the image is compiled at `segment | phys`, so its functions, +# manifest F entries and exports are named by that VA (§5.1). KSEG0 shards keep +# their place and names in the cache tag directory, so no existing cache is +# invalidated; KUSEG and KSEG1 shards go in its seg-kuseg/ and seg-kseg1/ +# subdirectories (decision 4), under the same {phys}_{crc} filename grammar, +# and their manifests carry `S `. R records stay KSEG0-spelled byte +# extents: they validate bytes, not execution. +KUSEG, KSEG0, KSEG1 = 0x00000000, 0x80000000, 0xA0000000 +SEGMENT_MASK, PHYS_MASK = 0xE0000000, 0x1FFFFFFF +SEGMENT_NAMES = {KUSEG: 'kuseg', KSEG0: 'kseg0', KSEG1: 'kseg1'} +SEGMENT_BY_NAME = {name: seg for seg, name in SEGMENT_NAMES.items()} +SEGMENT_CACHE_SUBDIRS = {KUSEG: 'seg-kuseg', KSEG1: 'seg-kseg1'} + +# The segment of the image being compiled. Helpers that name PCs without an +# image address at hand spell them in it (_canonical_guest_addr); every +# per-image entry point below sets it for its duration (image_segment). The +# default is KSEG0, the only segment before §5.7, so a KSEG0 compile is +# unchanged. A context variable, not a global: captures compile on threads. +_IMAGE_SEGMENT = contextvars.ContextVar('psx_overlay_image_segment', + default=KSEG0) + + +def segment_of(addr: int) -> int: + return addr & SEGMENT_MASK + + +@contextmanager +def image_segment(seg: int): + if seg not in SEGMENT_NAMES: + raise ValueError(f'0x{seg:08X} is not a segment that maps RAM') + token = _IMAGE_SEGMENT.set(seg) + try: + yield seg + finally: + _IMAGE_SEGMENT.reset(token) + + +def _canonical_guest_addr(addr: int) -> int: + """``addr``'s bytes, spelled in the segment of the image being compiled.""" + return (addr & PHYS_MASK) | _IMAGE_SEGMENT.get() + + +def segment_cache_dir(cache_dir: str, seg: int) -> str: + """The directory a shard compiled for ``seg`` lives in (decision 4).""" + sub = SEGMENT_CACHE_SUBDIRS.get(seg) + return os.path.join(cache_dir, sub) if sub else cache_dir + + +def cache_dir_segment(directory: str) -> int: + """The segment of the shards in ``directory`` (overlay_loader.c's + cache_seg_from_path): a seg-kuseg/ or seg-kseg1/ directory, else KSEG0.""" + name = os.path.basename(os.path.normpath(directory)).lower() + for seg, sub in SEGMENT_CACHE_SUBDIRS.items(): + if name == sub: + return seg + return KSEG0 + + +def _respell(addr: int, seg: int) -> int: + return (addr & PHYS_MASK) | seg + + +def capture_segment_views(cap: dict, + required_segments=frozenset()) -> list[dict]: + """Split one capture record into one per segment with entries (§5.7). + + A v2 capture (no ``dispatch_entry_segments``) is KSEG0 only and is + returned unchanged, so every existing capture compiles exactly as before. + For v3, ``dispatch_entry_pcs`` keeps its v2 meaning (dispatched in any + segment, spelled at KSEG0) and ``dispatch_entry_segments`` names the + segments: {"kuseg": [...], "kseg0": [...], "kseg1": [...]}, each PC in its + own segment. A dispatch entry that no segment list names is KSEG0. + + A view is the capture seen from one segment: ``load_addr`` and every PC in + it are spelled in that segment, and its dispatch entries are that + segment's alone. The execution evidence (executed_pcs) has no segment -- + direct edges keep the entry's segment -- so each view carries all of it. + The bytes, size and guard words are shared. + + ``required_segments`` also gets a view where the record has no dispatch + entry: the segments an operator or config demand names inside it + (declared_view_segments). A record with no view at all -- execution + evidence only, which the runtime writes -- is read as a v2 reader reads + it: one KSEG0 view, so its classification, forced interiors, declared + entries and prior manifests apply exactly as before. + """ + segments = cap.get('dispatch_entry_segments') + if segments is None: + return [cap] + load = _parse_addr(cap['load_addr']) + all_entries = {a & PHYS_MASK + for a in _parse_addr_list(cap.get('dispatch_entry_pcs', []))} + by_seg = {seg: set() for seg in SEGMENT_NAMES} + labelled = set() + if not isinstance(segments, dict): + raise RuntimeError('dispatch_entry_segments must be an object') + for name, pcs in segments.items(): + seg = SEGMENT_BY_NAME.get(name) + if seg is None: + raise RuntimeError(f'dispatch_entry_segments: unknown segment {name!r}') + for pc in _parse_addr_list(pcs): + if segment_of(pc) != seg: + raise RuntimeError( + f'dispatch_entry_segments.{name}: 0x{pc:08X} is not in {name}') + by_seg[seg].add(pc & PHYS_MASK) + labelled.add(pc & PHYS_MASK) + # Entries with no segment record are KSEG0, as in a v2 capture. + by_seg[KSEG0] |= all_entries - labelled + + def respell_list(values, seg, keep=None): + out = [] + for value in _parse_addr_list(values or []): + if keep is not None and (value & PHYS_MASK) not in keep: + continue + out.append(f'0x{_respell(value, seg):08X}') + return sorted(out) + + wanted = {seg for seg, entries in by_seg.items() if entries} + wanted |= {seg for seg in required_segments if seg in by_seg} + if not wanted: + wanted = {KSEG0} + views = [] + for seg in (KSEG0, KUSEG, KSEG1): + if seg not in wanted: + continue + entries = by_seg[seg] + view = {k: v for k, v in cap.items() if k != 'dispatch_entry_segments'} + view['load_addr'] = f'0x{_respell(load, seg):08X}' + view['dispatch_entry_pcs'] = [f'0x{_respell(a, seg):08X}' + for a in sorted(entries)] + view['seeds'] = list(view['dispatch_entry_pcs']) + if 'static_dispatch_entry_pcs' in cap: + view['static_dispatch_entry_pcs'] = respell_list( + cap['static_dispatch_entry_pcs'], seg, entries) + for field in ('executed_pcs', 'observed_pcs', 'function_entry_pcs', + 'static_discovery_entry_pcs', + 'static_call_continuation_pcs'): + if field in cap: + view[field] = respell_list(cap[field], seg) + if cap.get('producer_ranges'): + view['producer_ranges'] = [ + dict(r, start=f'0x{_respell(_parse_addr(r["start"]), seg):08X}', + end=f'0x{_respell(_parse_addr(r["end"]), seg):08X}') + for r in cap['producer_ranges']] + if cap.get('static_alias_ranges'): + view['static_alias_ranges'] = [ + dict(r, entry=f'0x{_respell(_parse_addr(r["entry"]), seg):08X}', + start=f'0x{_respell(_parse_addr(r["start"]), seg):08X}', + end=f'0x{_respell(_parse_addr(r["end"]), seg):08X}') + for r in cap['static_alias_ranges']] + views.append(view) + return views + + +def declared_view_segments(cap: dict, toml_doc: dict, + forced_interiors=frozenset()) -> set[int]: + """Segments a demand from outside the capture names inside ``cap`` (§5.7). + + A ``--force-interior`` PC is KSEG0 (a physical and a KUSEG PC are the same + number), so one inside the record asks for its KSEG0 view. A game.toml + ``[[overlays]]`` table matches the load address it spells, and one without + a load address applies its entries by their own spelling, so each names + the segment it is written in. capture_segment_views() gives every such + segment a view even where the capture saw no dispatch entry, so these + demands compile as they did before captures carried segments. Byte-CRC + checks stay with the compile (_collect_toml_overlay_entries): a detached + table's view compiles nothing.""" + load = _parse_addr(cap['load_addr']) + lo = load & PHYS_MASK + hi = lo + int(cap['size']) + inside = lambda pc: lo <= (pc & PHYS_MASK) < hi + segs = {KSEG0} if any(inside(pc) for pc in forced_interiors) else set() + for ov in (toml_doc or {}).get('overlays', []) or []: + if not isinstance(ov, dict): + continue + entries = set() + for key in ('entry', 'entries', 'function_entry_pcs', 'function_entries'): + val = ov.get(key) + if isinstance(val, list): + entries.update(_parse_addr_list(val)) + elif val is not None: + entries.add(_parse_addr(val)) + ov_load = ov.get('load_addr') or ov.get('load_address') + if ov_load is not None: + ov_load = _parse_addr(ov_load) + if entries and (ov_load & PHYS_MASK) == lo: + segs.add(segment_of(ov_load)) + else: + segs.update(segment_of(pc) for pc in entries if inside(pc)) + return {seg for seg in segs if seg in SEGMENT_NAMES} + + # --------------------------------------------------------------------------- # Shard build accounting — make failures LOUD # --------------------------------------------------------------------------- @@ -2066,7 +2268,7 @@ def walk_root_seed_entries(seeds: list[str]) -> set[int]: not parts[0].startswith('0x')): continue try: - entries.add((int(parts[-1], 16) & 0x1FFFFFFF) | 0x80000000) + entries.add(_canonical_guest_addr(int(parts[-1], 16))) except ValueError: pass return entries @@ -2330,12 +2532,14 @@ def patch_generated_c_static(src: str, load_addr: int, size: int) -> tuple: return f'call_by_address(cpu, 0x{addr:08X}u)' src = re.sub(r'\bfunc_([0-9A-Fa-f]{8})\(cpu\)', fix_call, src) - # 4. Collect in-overlay function definition addresses (no export annotation needed) + # 4. Collect in-overlay function definition addresses (no export annotation + # needed), spelled in the image's segment (§5.7). func_virt_addrs = [] + image_seg = segment_of(load_addr) def collect_fn(m): addr = int(m.group(1), 16) if in_overlay(addr): - func_virt_addrs.append(0x80000000 | (addr & 0x1FFFFFFF)) + func_virt_addrs.append(image_seg | (addr & PHYS_MASK)) return m.group(0) src = re.sub(r'^void func_([0-9A-Fa-f]{8})\(CPUState\* cpu\)$', collect_fn, src, flags=re.MULTILINE) @@ -2360,7 +2564,7 @@ def namespace_generated_static(src: str, namespace: str, src = re.sub(rf'\b{helper}\b', f'{namespace}_{helper}', src) def rename_func(m): - addr = (int(m.group(1), 16) & 0x1FFFFFFF) | 0x80000000 + addr = _canonical_guest_addr(int(m.group(1), 16)) if addr not in func_set: return m.group(0) return f'{namespace}_func_{addr:08X}' @@ -2368,7 +2572,7 @@ def namespace_generated_static(src: str, namespace: str, src = re.sub(r'\bfunc_([0-9A-Fa-f]{8})\b', rename_func, src) def rename_alias(m): - addr = (int(m.group(1), 16) & 0x1FFFFFFF) | 0x80000000 + addr = _canonical_guest_addr(int(m.group(1), 16)) if addr not in func_set: return m.group(0) return f'{namespace}_alias_body_{addr:08X}' @@ -2392,12 +2596,12 @@ def parse_cps_continuation_owners(src: str) -> dict: definitions = list(definition_re.finditer(src)) owners = {} for index, match in enumerate(definitions): - host = (int(match.group(1), 16) & 0x1FFFFFFF) | 0x80000000 + host = _canonical_guest_addr(int(match.group(1), 16)) end = definitions[index + 1].start() if index + 1 < len(definitions) else len(src) body = src[match.end():end] for block in re.finditer(r'^block_([0-9A-Fa-f]{8}):', body, re.MULTILINE): - entry = (int(block.group(1), 16) & 0x1FFFFFFF) | 0x80000000 + entry = _canonical_guest_addr(int(block.group(1), 16)) owners.setdefault(entry, host) return owners @@ -2532,10 +2736,13 @@ def generate_overlay_dispatch(variants: list) -> str: unique = [] seen = set() for variant in variants: - # The lookup keys on the exact PC; every body is compiled for KSEG0. - if variant['addr'] & 0xE0000000 != 0x80000000: + # The lookup keys on the exact PC, and each body is compiled for the + # segment of its PC (§5.7): a KUSEG, KSEG0 or KSEG1 entry of the same + # bytes is its own row. PC 0 is the hash table's empty marker. + if (segment_of(variant['addr']) not in SEGMENT_NAMES or + variant['addr'] == 0): raise ValueError(f"static overlay entry 0x{variant['addr']:08X} " - "is not a KSEG0 PC") + "is not a KUSEG, KSEG0 or KSEG1 code PC") ranges = tuple((lo & 0x1FFFFFFF, length) for lo, length in variant['ranges']) key = (variant['addr'], variant['crc'], ranges) @@ -2690,6 +2897,23 @@ def generate_overlay_dispatch(variants: list) -> str: lines.append(f'static uint64_t psx_ov_entry_hits[{n_entries}];') lines.append('') + present = sorted({segment_of(v['addr']) for v in unique} or {KSEG0}) + if present == [KSEG0]: + segment_gate = [ + ' /* Entries are the KSEG0 PCs their bodies were compiled for', + ' * (PSX_OVERLAY_CODE_SEGMENT, overlay_loader.h). A KUSEG or KSEG1', + ' * alias of the same bytes is a different architectural PC and', + ' * stays in the interpreter. */', + ' if ((addr & 0xE0000000u) != 0x80000000u) {', + ] + else: + segment_gate = [ + ' /* Entries are the exact PCs their bodies were compiled for, in', + ' * their own segment (docs/SEGMENT_AWARE_CODE.md §5.7). A PC in a', + ' * segment no body was compiled for stays in the interpreter. */', + ' if (' + ' &&\n '.join( + f'(addr & 0xE0000000u) != 0x{seg:08X}u' for seg in present) + ') {', + ] lines += [ f'/* {n_entries} dispatch addresses, {len(flat_variants)} variants, ' f'{size}-slot table (load {n_entries / size:.2f}), ' @@ -2716,11 +2940,7 @@ def generate_overlay_dispatch(variants: list) -> str: 'static uint32_t psx_ov_found_entry = 0;', '', 'static const PsxOvVariant *psx_overlay_static_find_variant(uint32_t addr) {', - ' /* Entries are the KSEG0 PCs their bodies were compiled for', - ' * (PSX_OVERLAY_CODE_SEGMENT, overlay_loader.h). A KUSEG or KSEG1', - ' * alias of the same bytes is a different architectural PC and', - ' * stays in the interpreter. */', - ' if ((addr & 0xE0000000u) != 0x80000000u) {', + ] + segment_gate + [ ' psx_ov_static_address_misses++;', ' return 0;', ' }', @@ -2903,7 +3123,8 @@ def parse_overlay_func_ids(src_path: str, data: bytes, load_addr: int, crc = binascii.crc32(data[off:off + length], crc) if not ok: continue - ev = (entry & 0x1FFFFFFF) | 0x80000000 + # The entry VA in the image's segment (§5.1, §5.7). + ev = (entry & PHYS_MASK) | segment_of(load_addr) out.append((ev, crc & 0xFFFFFFFF, ranges)) return out @@ -2930,6 +3151,7 @@ def audit_func_id_delay_slots(func_ids: list, data: bytes, """Return unsafe emitted dependency errors for exact function identities.""" errors = [] base = load_addr & 0x1FFFFFFF + seg = segment_of(load_addr) for entry, _crc, ranges in func_ids: if len(ranges) > 16: errors.append((entry, None, @@ -2942,7 +3164,7 @@ def audit_func_id_delay_slots(func_ids: list, data: bytes, off = phys - base if (phys & 3) or (length & 3) or length < 4 or \ off < 0 or off + length > len(data): - errors.append((entry, phys | 0x80000000, + errors.append((entry, phys | seg, 'malformed/out-of-capture code range')) malformed = True break @@ -2961,18 +3183,18 @@ def audit_func_id_delay_slots(func_ids: list, data: bytes, instr = int.from_bytes(data[off:off + 4], 'little') kind = _mips_control_kind(instr) if kind < 0: - errors.append((entry, pc | 0x80000000, + errors.append((entry, pc | seg, 'reserved/unsupported branch encoding')) elif kind > 0: controls.add(pc) if not contains_word(pc + 4): - errors.append((entry, pc | 0x80000000, - f'delay slot 0x{(pc + 4) | 0x80000000:08X} ' + errors.append((entry, pc | seg, + f'delay slot 0x{(pc + 4) | seg:08X} ' 'is not identity-hashed')) delay_pcs = {pc + 4 for pc in controls} nested = sorted(controls & delay_pcs) for pc in nested: - errors.append((entry, pc | 0x80000000, + errors.append((entry, pc | seg, 'control transfer in a delay slot')) return errors @@ -2991,13 +3213,27 @@ def overlay_ranges_text(func_ids: list, pair_id: int | None = None, out_lines = ['# psxrecomp overlay code-range manifest v2 (entry+code_crc)\n'] if provenance is not None: out_lines.append(f'# psxrecomp overlay provenance {provenance}\n') + # F entries are full VAs in the shard's segment (§5.7). A KUSEG or KSEG1 + # shard says so with S before its first F; a KSEG0 manifest keeps the + # pre-§5.7 text exactly (no S means KSEG0). + segments = {segment_of(ev) for ev, _crc, _ranges in func_ids} + if len(segments) > 1: + raise ValueError('one manifest names entries in several segments: ' + + ', '.join(f'0x{seg:08X}' for seg in sorted(segments))) + seg = segments.pop() if segments else KSEG0 + if seg not in SEGMENT_NAMES: + raise ValueError(f'manifest entries in segment 0x{seg:08X}, ' + 'which maps no RAM') + if seg != KSEG0: + out_lines.append(f'S {seg:08X}\n') if pair_id is not None: out_lines.append(f'P {pair_id & 0xFFFFFFFFFFFFFFFF:016X}\n') for ev, crc, ranges in func_ids: out_lines.append(f'F {ev:08X} {crc & 0xFFFFFFFF:08X}\n') for lo, length in ranges: + # Byte extents, KSEG0-spelled in every segment's manifest. out_lines.append( - f'R {(lo & 0x1FFFFFFF) | 0x80000000:08X} {length:X}\n') + f'R {(lo & PHYS_MASK) | KSEG0:08X} {length:X}\n') return ''.join(out_lines) @@ -3099,9 +3335,15 @@ def load_region_entry_set(cache_dir: str, phys_addr: int, def parse_runtime_shard_manifest(manifest: str, - require_pair: bool = True + require_pair: bool = True, + expected_segment: int | None = None ) -> tuple[int | None, list]: - """Parse only identities representable by the runtime manifest contract.""" + """Parse only identities representable by the runtime manifest contract. + + F entries are full VAs in the manifest's segment: ``S `` before + the first F, KSEG0 when absent (§5.7). ``expected_segment`` is the segment + of the directory the shard lives in; the runtime refuses a manifest that + names another, and so does this.""" def hex_field(token: str, max_digits: int) -> int: if not re.fullmatch(rf'[0-9A-Fa-f]{{1,{max_digits}}}', token): raise ValueError(f'invalid manifest hex field: {token!r}') @@ -3110,6 +3352,7 @@ def parse_runtime_shard_manifest(manifest: str, pair_ids = [] funcs = [] current = None + segment = None if '\0' in manifest: return None, [] # Runtime reads physical records with fgets(), so only LF terminates a @@ -3127,7 +3370,16 @@ def parse_runtime_shard_manifest(manifest: str, if stripped else []) if not parts: continue - if parts[0] == 'P': + if parts[0] == 'S': + if len(parts) != 2 or segment is not None or funcs: + return None, [] + try: + segment = hex_field(parts[1], 8) + except ValueError: + return None, [] + if segment not in SEGMENT_NAMES: + return None, [] + elif parts[0] == 'P': if len(parts) != 2: return None, [] try: @@ -3138,11 +3390,11 @@ def parse_runtime_shard_manifest(manifest: str, if len(parts) != 3: return None, [] try: - current = [((hex_field(parts[1], 8) & 0x1FFFFFFF) | - 0x80000000), - hex_field(parts[2], 8), []] + current = [hex_field(parts[1], 8), hex_field(parts[2], 8), []] except ValueError: return None, [] + if segment_of(current[0]) != (KSEG0 if segment is None else segment): + return None, [] funcs.append(current) elif parts[0] == 'R' and current is not None: if len(parts) != 3: @@ -3154,6 +3406,9 @@ def parse_runtime_shard_manifest(manifest: str, return None, [] if len(pair_ids) > 1 or (require_pair and len(pair_ids) != 1) or not funcs: return None, [] + if (expected_segment is not None and + (KSEG0 if segment is None else segment) != expected_segment): + return None, [] entries = [entry & 0x1FFFFFFF for entry, _crc, _ranges in funcs] if len(set(entries)) != len(entries): return None, [] @@ -3197,7 +3452,8 @@ def load_shard_func_ids(dll_path: str, manifest_declares_supplemental_provenance(manifest)): return [] pair_id, valid_funcs = parse_runtime_shard_manifest( - manifest, require_pair=False) + manifest, require_pair=False, + expected_segment=cache_dir_segment(os.path.dirname(dll_path))) if (not valid_funcs or not _dll_runtime_exports_match( dll_path, expected_abi, pair_id, {entry for entry, _crc, _ranges in valid_funcs})): @@ -3394,12 +3650,15 @@ def unsupported_guest_branch_rejection(reason: str) -> bool: def reconcile_empty_primary_scans(pending, cache_dir, expected_abi, stats): - """Accept an empty primary only if guarded fragments serve every root.""" + """Accept an empty primary only if guarded fragments serve every root. + + Each capture view is checked against its own segment's shards (§5.7).""" for label, physical, load, size, data, roots in pending: + seg = segment_of(load) current, _ = load_region_current_variant_coverage( - cache_dir, physical, data, load, size, + segment_cache_dir(cache_dir, seg), physical, data, load, size, expected_abi) - served = {(entry & 0x1FFFFFFF) | 0x80000000 for entry in current} + served = {(entry & PHYS_MASK) | seg for entry in current} if roots and roots <= served: print(f' primary scan recovered by guarded fragments: {label}') stats.add_skip() @@ -3506,9 +3765,10 @@ def static_fragment_job(entry: int, image_key, args): Returns (part or None, reason).""" data, load_addr, size, phys_addr, guard_bytes = \ _STATIC_IMAGE_SOURCES[image_key] - return generate_interior_fragment_static( - entry, data, load_addr, size, phys_addr, args, - guard_bytes=guard_bytes) + with image_segment(segment_of(load_addr)): + return generate_interior_fragment_static( + entry, data, load_addr, size, phys_addr, args, + guard_bytes=guard_bytes) def generate_interior_fragment_static(interior: int, data: bytes, @@ -3581,7 +3841,7 @@ def generate_interior_fragment_static(interior: int, data: bytes, if set(func_addrs) - set(ids_by_addr): return None, 'static-ranges: dispatchable function(s) lack exact ranges' - entry = (interior & 0x1FFFFFFF) | 0x80000000 + entry = (interior & PHYS_MASK) | segment_of(load_addr) if entry not in ids_by_addr or entry not in set(func_addrs): return None, 'requested-entry-audit: fragment omitted its own entry' namespace = (f'ov_frag_{phys_addr:08X}_{image_crc:08X}_' @@ -3637,7 +3897,7 @@ def validate_fragment_requested_ids(requested: set[int], frag_ids: list, representable identity whose guarded bytes include the entry itself. """ for requested_entry in sorted(requested): - entry = (requested_entry & 0x1FFFFFFF) | 0x80000000 + entry = _canonical_guest_addr(requested_entry) if entry not in generated_defs: return f'requested entry 0x{entry:08X} missing from generated C' matches = [ @@ -3650,7 +3910,7 @@ def validate_fragment_requested_ids(requested: set[int], frag_ids: list, if contract_error: return contract_error for requested_entry in sorted(requested): - entry = (requested_entry & 0x1FFFFFFF) | 0x80000000 + entry = _canonical_guest_addr(requested_entry) matches = [ (ev, crc, ranges) for ev, crc, ranges in frag_ids if ev == entry ] @@ -3786,9 +4046,11 @@ def make_interior_fragment_job(phys_addr: int, load_addr: int, size: int, seed_audit.get('static_interval_fragment_demands', set())) static_demands = static_exact_demands | static_interval_demands region_hi = phys_addr + size + # --force-interior names a PC, so it forces the view of its own segment. forced = { a for a in forced_interiors - if phys_addr <= (a & 0x1FFFFFFF) < region_hi + if phys_addr <= (a & 0x1FFFFFFF) < region_hi and + segment_of(a) == segment_of(load_addr) } if not (((interiors or dispatch_roots) and executed) or observed_dispatch or static_demands or forced): @@ -3872,10 +4134,6 @@ def merge_fragment_jobs_by_recipe(jobs: list[dict], return [grouped[key] for key in sorted(grouped)] -def _canonical_guest_addr(addr: int) -> int: - return (addr & 0x1FFFFFFF) | 0x80000000 - - def _normalized_func_identity(func_id) -> tuple: entry, code_crc, ranges = func_id return (_canonical_guest_addr(entry), code_crc & 0xFFFFFFFF, @@ -4376,7 +4634,7 @@ def compile_fragment_batch(requested_entries, data: bytes, load_addr: int, (None, reason) on failure/skip so the caller can tally WHY a fragment dropped instead of the old silent None.""" requested = sorted({ - (entry & 0x1FFFFFFF) | 0x80000000 + (entry & PHYS_MASK) | segment_of(load_addr) for entry in requested_entries }) if not requested: @@ -5015,8 +5273,15 @@ def _candidate_capacity_namespace(dll_path: str) -> tuple[str, list[str]]: may still use ``cgN_hash``). The runtime candidate table is process-global, so serialize/count every compiler tier with the same GAME/arch/cg leaf. Tests and hand-built paths fall back to their containing directory. + + The KUSEG and KSEG1 shards of a leaf live in its seg-kuseg/ and seg-kseg1/ + subdirectories (§5.7) and take slots from the same process-global table, + so a shard there belongs to its leaf's namespace, and every namespace + lists each leaf's segment directories after the leaf. """ leaf = os.path.dirname(os.path.abspath(dll_path)) + if cache_dir_segment(leaf) != KSEG0: + leaf = os.path.dirname(leaf) cg_name = os.path.basename(leaf) arch_dir = os.path.dirname(leaf) compiler_dir = os.path.dirname(arch_dir) @@ -5026,12 +5291,36 @@ def _candidate_capacity_namespace(dll_path: str) -> tuple[str, list[str]]: compiler_name in ('gcc', 'tcc')): arch_name = os.path.basename(arch_dir) tier_dirs = [ - os.path.join(game_dir, tier, arch_name, cg_name) + directory for tier in ('gcc', 'tcc') + for directory in _with_segment_dirs( + os.path.join(game_dir, tier, arch_name, cg_name)) ] return (os.path.join(game_dir, '.overlay-candidate-capacity.lock'), tier_dirs) - return (os.path.join(leaf, '.overlay-candidate-capacity.lock'), [leaf]) + return (os.path.join(leaf, '.overlay-candidate-capacity.lock'), + _with_segment_dirs(leaf)) + + +def _with_segment_dirs(leaf: str) -> list[str]: + """A cache leaf, then its per-segment subdirectories (§5.7).""" + return [leaf] + [os.path.join(leaf, SEGMENT_CACHE_SUBDIRS[seg]) + for seg in (KUSEG, KSEG1)] + + +def _segment_leaf(directory: str) -> str: + """The cache leaf a (possibly per-segment) shard directory belongs to.""" + directory = os.path.abspath(directory) + if cache_dir_segment(directory) != KSEG0: + return os.path.dirname(directory) + return directory + + +def _segment_basename(path: str) -> tuple[int, str]: + """A shard's identity within one compiler tier: its segment and name. + The same {phys}_{crc} name is a different shard in each segment.""" + return (cache_dir_segment(os.path.dirname(os.path.abspath(path))), + os.path.normcase(os.path.basename(path))) def cache_candidate_capacity_snapshot( @@ -5060,7 +5349,7 @@ def candidate_capacity_saturated_in_tier(total: int, counts: dict, candidate_cap: int, cache_dir: str) -> bool: """Whether one active tier has reached any physical/runtime capacity.""" - active = os.path.normcase(os.path.abspath(cache_dir)) + active = os.path.normcase(_segment_leaf(cache_dir)) saturated = (total >= candidate_cap or getattr(counts, 'raw_total', sum(counts.values())) >= 2 * candidate_cap or @@ -5069,7 +5358,7 @@ def candidate_capacity_saturated_in_tier(total: int, counts: dict, getattr(counts, 'file_total', len(counts)) >= RUNTIME_CACHE_FILE_CAP) return (saturated and bool(counts) and all( - os.path.normcase(os.path.abspath(os.path.dirname(path))) == active + os.path.normcase(_segment_leaf(os.path.dirname(path))) == active for path in counts )) @@ -5223,7 +5512,8 @@ def _load_candidate_inventory_record(dll: str, with open(ranges_path, encoding='ascii', newline='') as source: manifest = source.read() pair_id, funcs = parse_runtime_shard_manifest( - manifest, require_pair=False) + manifest, require_pair=False, + expected_segment=cache_dir_segment(os.path.dirname(dll))) raw_count = len(funcs) range_link_count = _runtime_manifest_range_link_count(funcs) if (not funcs or not _dll_runtime_exports_match( @@ -5282,7 +5572,7 @@ def cache_candidate_inventory(cache_dirs: list[str], # before scanning the next compiler tier, so it cannot shadow # a valid lower-tier basename. continue - basename = os.path.normcase(name) + basename = _segment_basename(dll) if basename in seen_basenames: continue # Runtime scans gcc before tcc and suppresses a lower-tier path by @@ -5341,10 +5631,10 @@ def _projected_cache_capacity_usage(total: int, counts: dict, for index, path in enumerate(cache_dirs) } final_rank = ranks.get(final_dir, 0) - basename = os.path.normcase(os.path.basename(final_dll)) + basename = _segment_basename(final_dll) selected_path = next(( path for path in counts - if os.path.normcase(os.path.basename(path)) == basename + if _segment_basename(path) == basename ), None) records = [(path, count, getattr(counts, 'identities', {}).get(path), @@ -5563,7 +5853,8 @@ def _runtime_valid_shard_pair_locked(dll: str, ranges: str, with open(ranges, encoding='ascii', newline='') as f: manifest = f.read() pair_id, funcs = parse_runtime_shard_manifest( - manifest, require_pair=False) + manifest, require_pair=False, + expected_segment=cache_dir_segment(os.path.dirname(dll))) except (OSError, UnicodeError): return False return (bool(funcs) and _dll_runtime_exports_match( @@ -5739,8 +6030,8 @@ def _dll_runtime_exports_match(dll_path: str, expected_abi: int | None, getattr(library, 'overlay_init') getattr(library, 'overlay_flush_cycles') for entry in entries: - virt = (entry & 0x1FFFFFFF) | 0x80000000 - getattr(library, f'func_{virt:08X}') + # Exports are named by the entry's full VA (§5.1, §5.7). + getattr(library, f'func_{entry:08X}') return True except (OSError, AttributeError): return False @@ -5858,6 +6149,14 @@ def static_capture_job(cap: dict, args, toml: dict, forced_interiors: set, def _static_capture_job(cap, args, toml, forced_interiors, static_out, result): + # Runs in a worker process too, so it sets its image segment itself. + with image_segment(segment_of(int(cap['load_addr'], 16))): + _static_capture_job_in_segment(cap, args, toml, forced_interiors, + static_out, result) + + +def _static_capture_job_in_segment(cap, args, toml, forced_interiors, + static_out, result): load_addr = int(cap['load_addr'], 16) size = int(cap['size']) data = base64.b64decode(cap['bytes_b64']) @@ -5875,7 +6174,7 @@ def _static_capture_job(cap, args, toml, forced_interiors, static_out, result): result['fail'] = (cls, detail) for captured_entry in _parse_addr_list(cap.get('dispatch_entry_pcs', [])): - entry = ((captured_entry & 0x1FFFFFFF) | 0x80000000) + entry = _canonical_guest_addr(captured_entry) key = (entry, crc32) result['requested_entries'].add(key) result['entry_sources'][key] = ( @@ -5887,8 +6186,9 @@ def _static_capture_job(cap, args, toml, forced_interiors, static_out, result): region_hi = phys_addr + size for forced_entry in forced_interiors: forced_phys = forced_entry & 0x1FFFFFFF - if phys_addr <= forced_phys < region_hi: - entry = forced_phys | 0x80000000 + if (phys_addr <= forced_phys < region_hi and + segment_of(forced_entry) == segment_of(load_addr)): + entry = forced_entry key = (entry, crc32) result['requested_entries'].add(key) result['entry_sources'][key] = ( @@ -5996,7 +6296,11 @@ def _static_capture_job(cap, args, toml, forced_interiors, static_out, result): # identical bytes to coexist without symbol clashes. cov_blob = ','.join(f'{a:08X}' for a in sorted(func_addrs)).encode() cov_crc = binascii.crc32(cov_blob) & 0xFFFFFFFF - namespace = f'ov_{phys_addr:08X}_{crc32:08X}_{cov_crc:08X}' + # A KUSEG or KSEG1 image of the same bytes links beside the KSEG0 one + # (§5.7): ov_s0_… and ov_s5_…; KSEG0 keeps its name. + seg_tag = ('' if segment_of(load_addr) == KSEG0 + else f's{segment_of(load_addr) >> 29}_') + namespace = f'ov_{seg_tag}{phys_addr:08X}_{crc32:08X}_{cov_crc:08X}' src, symbols = namespace_generated_static(src, namespace, func_addrs) variants = [] for ev in sorted(func_addrs): @@ -6215,8 +6519,10 @@ def main(): else: target_os = None set_target_os(target_os) + # Canonicalized to KSEG0 as before (a physical and a KUSEG PC are the + # same number); a forced interior therefore forces KSEG0 views. forced_interiors = { - (int(v, 0) & 0x1FFFFFFF) | 0x80000000 + (int(v, 0) & 0x1FFFFFFF) | KSEG0 for v in args.force_interior } @@ -6325,6 +6631,25 @@ def main(): if not captures: ap.error('--only-region did not match any capture') + # One view per (capture, segment with entries), §5.7. A capture whose + # segment record is malformed is reported and not compiled. + segment_view_errors = [] + views = [] + for cap in captures: + try: + views.extend(capture_segment_views( + cap, declared_view_segments(cap, toml, forced_interiors))) + except (KeyError, TypeError, ValueError, RuntimeError) as exc: + segment_view_errors.append( + (f'overlay {cap.get("load_addr", "?")}', str(exc))) + if len(views) != len(captures) or segment_view_errors: + by_seg = Counter(SEGMENT_NAMES[segment_of(int(v['load_addr'], 16))] + for v in views) + print(f'Segment views: {len(captures)} capture(s) -> {len(views)} ' + 'view(s) (' + ', '.join(f'{n} {name}' for name, n in + sorted(by_seg.items())) + ')') + captures = views + print(f'Captures: {len(captures)} overlay(s) to process\n') # B-2 static mode: accumulate privately-namespaced generated C plus exact @@ -6368,6 +6693,15 @@ def main(): # or passed per-region (region_coverage_cache / interior_frag_jobs), so a # worker owning a region owns every mutable it touches. def _do_capture(cap, region_coverage_cache, interior_frag_jobs, stats): + # A capture view compiles in its own segment (§5.7): helpers that name + # PCs spell them in it, and its shard lives in that segment's cache + # directory. + with image_segment(segment_of(int(cap['load_addr'], 16))): + _do_capture_in_segment(cap, region_coverage_cache, + interior_frag_jobs, stats) + + def _do_capture_in_segment(cap, region_coverage_cache, interior_frag_jobs, + stats): load_addr = int(cap['load_addr'], 16) size = int(cap['size']) data = base64.b64decode(cap['bytes_b64']) @@ -6384,6 +6718,9 @@ def main(): _merge_static_result(static_capture_job( cap, args, toml, forced_interiors, static_out)) return + seg_cache_dir = segment_cache_dir(cache_dir, segment_of(load_addr)) + # Coverage is per segment: each segment's shards are their own. + coverage_key = (segment_of(load_addr), phys_addr) # Reclassify prior F entry addresses from an identical image. Callable # entries may become current roots; other entries re-enter as dispatch @@ -6391,7 +6728,7 @@ def main(): # ranges: later roots can repartition the same bytes incompatibly. if not args.static: ranges_name = f'{phys_addr:08X}_{crc32:08X}.ranges' - prior_ranges_path = os.path.join(cache_dir, ranges_name) + prior_ranges_path = os.path.join(seg_cache_dir, ranges_name) if os.path.exists(prior_ranges_path): prior_entries = [] with open(prior_ranges_path) as pf: @@ -6441,7 +6778,8 @@ def main(): interior_frag_jobs.append(fragment_job) if not args.static: - dll_path = os.path.join(cache_dir, f'{phys_addr:08X}_{crc32:08X}{overlay_ext()}') + dll_path = os.path.join(seg_cache_dir, + f'{phys_addr:08X}_{crc32:08X}{overlay_ext()}') print(f'Overlay load=0x{load_addr:08X} size={size} crc32=0x{crc32:08X}' + (f' guard={guard_bytes}B (delay-slot only, not analysed)' @@ -6479,10 +6817,10 @@ def main(): else: current_entries, _current_ranges = ( load_region_current_variant_coverage( - cache_dir, phys_addr, data, load_addr, size, + seg_cache_dir, phys_addr, data, load_addr, size, expected_abi)) if demanded_root_entries <= { - (entry & 0x1FFFFFFF) | 0x80000000 + _canonical_guest_addr(entry) for entry in current_entries}: reconcile_bios_resident_marker(dll_path, cap, False) print(f' SKIP: runtime-valid current-byte cache serves all ' @@ -6608,12 +6946,12 @@ def main(): this_set = {(ev, crc) for ev, crc, _ in this_ids} with cov_lock: - covered = region_coverage_cache.get(phys_addr) + covered = region_coverage_cache.get(coverage_key) if covered is None: covered = load_region_coverage( - cache_dir, phys_addr, + seg_cache_dir, phys_addr, overlay_abi_tag(args.runtime_include, args.flavor)) - region_coverage_cache[phys_addr] = covered + region_coverage_cache[coverage_key] = covered fully_covered = (bool(this_set) and this_set <= covered and not args.force and cap.get('producer') != @@ -6708,7 +7046,7 @@ def main(): # DLLs at a region.) with cov_lock: covered |= load_region_coverage( - cache_dir, phys_addr, + seg_cache_dir, phys_addr, overlay_abi_tag( args.runtime_include, args.flavor)) if published: @@ -6755,6 +7093,18 @@ def main(): # interior PC (recovers no host). Isolated => a bad fragment fails alone and # never poisons a region's trusted DLL. def _do_frags(interior_frag_jobs, stats): + # Fragments compile per segment, like region shards (§5.7): a job's + # PCs, its shards and their directory are its capture view's segment. + # KSEG0 always runs (as before); the others only when they have jobs. + by_seg = {} + for job in interior_frag_jobs: + by_seg.setdefault(segment_of(job['load_addr']), []).append(job) + for seg in [KSEG0] + [s for s in (KUSEG, KSEG1) if s in by_seg]: + with image_segment(seg): + _do_frags_in_segment(by_seg.get(seg, []), stats, + segment_cache_dir(cache_dir, seg)) + + def _do_frags_in_segment(interior_frag_jobs, stats, cache_dir): frag_env = dict(os.environ) if args.cps: frag_env['PSX_CPS'] = '1' @@ -7398,6 +7748,9 @@ def main(): # combined-C mode has no runtime cache publication and keeps its historical # capture order. stats = ShardStats() + for label, detail in segment_view_errors: + print(f' SEGMENT RECORD REJECTED: {label}: {detail}') + stats.add_fail(label, 'segments', detail) if args.static: if args.jobs > 1 and len(captures) > 1: # Each capture is an independent recompiler subprocess plus a @@ -7611,7 +7964,11 @@ def main(): (entry & 0x1FFFFFFF) not in forced_phys): fragment_memo_skipped += 1 continue - image_key = (image_crc, phys_addr, size) + # One image per segment view (§5.7): every view of a capture has + # the same bytes, phys and size, but a demand compiles against its + # own view's load address, which fixes the segment it is spelled, + # named and baked in. + image_key = (image_crc, phys_addr, size, segment_of(load_addr)) image_sources.setdefault(image_key, source) fragment_jobs.append((key, memo_key, image_key)) diff --git a/tools/coverage_vault.py b/tools/coverage_vault.py index db473c3f..f1daff22 100644 --- a/tools/coverage_vault.py +++ b/tools/coverage_vault.py @@ -36,6 +36,39 @@ CAP_NAME = "overlay_captures.json" CACHE_SUB = "cache" EVIDENCE_FIELDS = ("executed_pcs", "dispatch_entry_pcs", "static_dispatch_entry_pcs", "function_entry_pcs", "seeds") +# Capture schema v3 (docs/SEGMENT_AWARE_CODE.md §5.7): the segments each +# dispatch entry entered through, {"kuseg": [...], "kseg0": [...], +# "kseg1": [...]}, each PC spelled in its segment. A record without it is +# KSEG0 only. +SEGMENT_FIELD = "dispatch_entry_segments" +SEGMENT_BASES = {"kuseg": 0x00000000, "kseg0": 0x80000000, "kseg1": 0xA0000000} + + +def _explicit_segments(record): + """The record's dispatch segments with the v2 default made explicit: + a dispatch entry that no segment list names is a KSEG0 entry.""" + out = {name: set() for name in SEGMENT_BASES} + labelled = set() + for name, pcs in (record.get(SEGMENT_FIELD) or {}).items(): + for pc in pcs or []: + out.setdefault(name, set()).add(pc) + labelled.add(_address(pc, SEGMENT_FIELD) & 0x1FFFFFFF) + for pc in record.get("dispatch_entry_pcs", []) or []: + phys = _address(pc, "dispatch_entry_pcs") & 0x1FFFFFFF + if phys not in labelled: + out["kseg0"].add("0x%08X" % (0x80000000 | phys)) + return out + + +def _union_segments(target, record): + """Fold `record`'s segments into `target` (call before unioning + dispatch_entry_pcs). Only when either side carries segments: merging v2 + records keeps them v2, and a v2 side's entries stay KSEG0 entries.""" + if SEGMENT_FIELD not in target and SEGMENT_FIELD not in record: + return + a, b = _explicit_segments(target), _explicit_segments(record) + target[SEGMENT_FIELD] = {name: sorted(a.get(name, set()) | b.get(name, set())) + for name in SEGMENT_BASES} def _variant_key(region): b = region.get("bytes_b64", "") or "" @@ -68,6 +101,7 @@ def _load_list(path): index[k] = dict(r) else: tgt = index[k] + _union_segments(tgt, r) for fld in ("executed_pcs", "dispatch_entry_pcs", "function_entry_pcs", "seeds"): tgt[fld] = sorted(set(tgt.get(fld, [])) | @@ -209,7 +243,7 @@ def _compact_region(region, drop_invalid_evidence=False): run_hi = min(phys_hi, ((previous + 1) << 12) + 4) compact = {k: v for k, v in region.items() if k not in EVIDENCE_FIELDS and - k not in ("load_addr", "size", "bytes_b64")} + k not in ("load_addr", "size", "bytes_b64", SEGMENT_FIELD)} compact["load_addr"] = "0x%08X" % (load + run_lo - phys_lo) compact["size"] = run_hi - run_lo compact["bytes_b64"] = base64.b64encode( @@ -217,6 +251,11 @@ def _compact_region(region, drop_invalid_evidence=False): for field in EVIDENCE_FIELDS: compact[field] = sorted({text for phys, text in parsed[field] if run_lo <= phys < run_hi}) + if isinstance(region.get(SEGMENT_FIELD), dict): + compact[SEGMENT_FIELD] = { + name: sorted(pc for pc in pcs or [] + if run_lo <= (_address(pc, SEGMENT_FIELD) & 0x7FFFFF) < run_hi) + for name, pcs in region[SEGMENT_FIELD].items()} runs.append(compact) if page is None: break @@ -244,6 +283,7 @@ def compact_capture_manifest(source, output=None, drop_invalid_evidence=False): index[key] = record continue target = index[key] + _union_segments(target, record) for field in EVIDENCE_FIELDS: target[field] = sorted(set(target.get(field, [])) | set(record.get(field, []))) @@ -511,6 +551,7 @@ def merge_capture_regions(vault_json, src): new_pcs += len(r.get("executed_pcs", [])) else: tgt = index[k] + _union_segments(tgt, r) for fld in ("executed_pcs", "dispatch_entry_pcs", "static_dispatch_entry_pcs", "function_entry_pcs", "seeds"): diff --git a/tools/test_compile_overlays_static_split.py b/tools/test_compile_overlays_static_split.py index f00a2265..8282ac4c 100644 --- a/tools/test_compile_overlays_static_split.py +++ b/tools/test_compile_overlays_static_split.py @@ -283,11 +283,75 @@ class StaticDispatchApiTests(unittest.TestCase): '-o', exe], check=True) subprocess.run([exe], check=True) - def test_non_kseg0_entry_is_rejected(self): - with self.assertRaises(ValueError): - compile_overlays.generate_overlay_dispatch( - [{'addr': 0x0000281C, 'symbol': 'ov_func_k', 'crc': 1, - 'ranges': ((0x0000281C, 16),)}]) + def test_segment_rows_dispatch_their_own_pc(self): + # docs/SEGMENT_AWARE_CODE.md §5.7: a KUSEG body of the same bytes is + # its own row, dispatched for its exact PC; KSEG1, with no body, + # stays in the interpreter. + cc = shutil.which('gcc') or shutil.which('clang') + if cc is None: + self.skipTest('no C compiler available') + variants = [ + {'addr': 0x8000281C, 'symbol': 'ov_func_k0', 'crc': 0x11111111, + 'ranges': ((0x0000281C, 16),)}, + {'addr': 0x0000281C, 'symbol': 'ov_func_ku', 'crc': 0x11111111, + 'ranges': ((0x0000281C, 16),)}, + ] + dispatch = compile_overlays.generate_overlay_dispatch(variants) + self.assertIn('(addr & 0xE0000000u) != 0x00000000u', dispatch) + self.assertIn('(addr & 0xE0000000u) != 0x80000000u', dispatch) + with tempfile.TemporaryDirectory() as tmp: + Path(tmp, 'psx_runtime.h').write_text( + '#include \n' + 'typedef struct CPUState { uint32_t pc; } CPUState;\n', + encoding='utf-8') + exe = os.path.join(tmp, 'rows.exe' if os.name == 'nt' else 'rows') + src = os.path.join(tmp, 'rows.c') + Path(src).write_text(textwrap.dedent(f''' + #include "psx_runtime.h" + + void ov_func_k0(CPUState *cpu); + void ov_func_ku(CPUState *cpu); + + {dispatch} + + static int k0, ku; + + int psx_overlay_static_code_matches( + const uint32_t *lo_len_pairs, uint32_t count, + uint32_t expected_crc) + {{ + (void)lo_len_pairs; + (void)count; + return expected_crc == 0x11111111u; + }} + + void ov_func_k0(CPUState *cpu) {{ (void)cpu; k0++; }} + void ov_func_ku(CPUState *cpu) {{ (void)cpu; ku++; }} + + int main(void) + {{ + CPUState cpu = {{0}}; + if (!psx_overlay_dispatch(&cpu, 0x0000281Cu)) return 1; + if (ku != 1 || k0 != 0) return 2; + if (!psx_overlay_dispatch(&cpu, 0x8000281Cu)) return 3; + if (ku != 1 || k0 != 1) return 4; + if (psx_overlay_dispatch(&cpu, 0xA000281Cu)) return 5; + if (psx_overlay_static_can_dispatch(0x2000281Cu)) return 6; + return 0; + }} + '''), encoding='utf-8') + subprocess.run([cc, src, '-I', tmp, '-std=c99', '-Wall', '-Werror', + '-o', exe], check=True) + subprocess.run([exe], check=True) + + def test_unmapped_segment_entry_is_rejected(self): + # 0x20000000-0x7FFFFFFF and KSEG2 map no RAM; PC 0 is the hash + # table's empty marker. + for addr in (0x2000281C, 0xC000281C, 0x00000000): + with self.assertRaises(ValueError): + compile_overlays.generate_overlay_dispatch( + [{'addr': addr, 'symbol': 'ov_func_k', 'crc': 1, + 'ranges': ((0x0000281C, 16),)}]) class StaticWorkerContractTests(unittest.TestCase):