mirror/psxrecomp
1
0
Fork 0
mirror of https://github.com/RetroPortingToolKit/psxrecomp synced 2026-10-25 18:05:12 -04:00
Code Issues Packages Projects Releases Wiki Activity
Files
2c92ac3ff1d176b198f4a040f09637af61ba12d4
psxrecomp/tools/compile_overlays.py
T
Shokunin 9714fb678a Linux tcc bundle: hermetic includes and a real-shard smoke test 
- compile_overlays: the bundle layout now compiles with -nostdinc and only
  the bundle's own headers (tcc's include/, then musl's libc-include/), so a
  host with libc6-dev cannot change what a shard compiles against.
- runtime/tests/test_linux_tcc_bundle_smoke.py (ctest
  linux_tcc_bundle_smoke_test): stages the bundle exactly as a release does
  (pinned tcc 0.9.27 built from source, musl 1.2.5 headers, runtime
  headers), compiles an emitter-shaped overlay shard through
  compile_overlays' own tcc path (preamble shim, call_by_address rewrite,
  exports) for PGXP flavors 0 and 2 with PATH emptied, loads the .so and
  checks overlay_abi() and the exported function. Linux x86_64 only (skips
  77 elsewhere). Passes in the Steam Runtime sniper SDK image.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-11 22:38:53 -07:00

8264 lines
377 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""
compile_overlays.py — Offline overlay compilation primitive (B-2 / A-2 shared path).
Reads overlay_captures.json, compiles each code overlay to a DLL, and writes
it to the cache directory. The runtime's A-1 LoadLibrary path reads from the
same cache.
Usage:
python3 tools/compile_overlays.py \\
--captures build-dev/overlay_captures.json \\
--game-toml game.toml \\
--recompiler psxrecomp/recompiler/build/psxrecomp-game.exe \\
--runtime-include psxrecomp/runtime/include \\
--out-dir build-dev/cache
Each DLL is written to <out-dir>/<game-id>/<crc32hex>.dll.
Each DLL exports:
overlay_init(dispatch_fn) — call once after LoadLibrary to wire dispatch
func_XXXXXXXX(CPUState*) — one export per compiled function entry point
"""
import argparse
import itertools
import io
import contextlib
import contextvars
import base64
import binascii
from bisect import bisect_left
from collections import Counter, deque
from contextlib import contextmanager, nullcontext
import glob
import hashlib
import heapq
import os
import re
import struct
import subprocess
import sys
import tempfile
import threading
import time
import uuid
try:
import tomllib # Python 3.11+
except ImportError:
try:
import tomli as tomllib
except ImportError:
print("ERROR: need tomllib (Python 3.11+) or 'pip install tomli'")
sys.exit(1)
import json
import platform
import re
import sysconfig
def codegen_ver(runtime_include: str) -> int:
"""Parse PSX_OVERLAY_CODEGEN_VER from overlay_api.h so the cache path version
is the SAME value the runtime (overlay_loader.c) uses — the two can't drift.
The cache is namespaced gcc/<arch-abi>/cg<N>/, so a build with new emitter
output reads+writes a FRESH dir and never reuses a stale DLL."""
hdr = os.path.join(runtime_include, 'overlay_api.h')
with open(hdr) as f:
m = re.search(r'#define\s+PSX_OVERLAY_CODEGEN_VER\s+(\d+)', f.read())
if not m:
raise SystemExit(f'PSX_OVERLAY_CODEGEN_VER not found in {hdr}')
return int(m.group(1))
def overlay_abi_tag(runtime_include: str, flavor: int) -> int:
"""Return the exact ABI/flavor export expected from a cached shard."""
hdr = os.path.join(runtime_include, 'overlay_api.h')
with open(hdr) as source:
match = re.search(
r'#define\s+PSX_OVERLAY_ABI_VERSION\s+(\d+)', source.read())
if not match:
raise SystemExit(f'PSX_OVERLAY_ABI_VERSION not found in {hdr}')
return (int(match.group(1)) & 0xFFFF) | ((int(flavor) & 0xFFFF) << 16)
def overlay_candidate_cap(runtime_include: str) -> int:
"""Return the loader's process-lifetime manifest-candidate capacity."""
hdr = os.path.join(runtime_include, 'overlay_api.h')
with open(hdr) as source:
match = re.search(
r'#define\s+PSX_OVERLAY_CANDIDATE_CAP\s+(\d+)', source.read())
if not match:
raise SystemExit(f'PSX_OVERLAY_CANDIDATE_CAP not found in {hdr}')
value = int(match.group(1))
if value <= 0:
raise SystemExit(f'invalid PSX_OVERLAY_CANDIDATE_CAP in {hdr}: {value}')
return value
def codegen_hash(runtime_include: str) -> int:
"""Parse PSX_OVERLAY_CODEGEN_HASH from the build-generated overlay_codegen_hash.h
(next to overlay_api.h). Folded into the cache path as cg<N>_<hash> so ANY
emitter change auto-invalidates the cache (no stale-but-cgN reuse). Falls back
to 0 when the header is absent (a tree not yet built) — matching overlay_api.h's
__has_include fallback, so the loader and compiler still agree on the path."""
hdr = os.path.join(runtime_include, 'overlay_codegen_hash.h')
try:
with open(hdr) as f:
m = re.search(r'#define\s+PSX_OVERLAY_CODEGEN_HASH\s+0x([0-9A-Fa-f]+)', f.read())
if m:
return int(m.group(1), 16)
except FileNotFoundError:
pass
return 0
def verify_recompiler_matches_tag(recompiler: str, tag_hash: int) -> None:
"""Stale-recompiler-binary guard. The cg tag hash is computed from the
EMITTER SOURCES (via --runtime-include's overlay_codegen_hash.h), but the
code is emitted by the --recompiler BINARY — nothing else ties the two.
A recompiler built before the last emitter change happily emits OLD code
that gets stamped with the CURRENT tag: read tag == write tag, content
stale (the 2026-07-01 Tomba pause-menu wedge; loaded-save scene shards
corrupted the display-list queue while every tag check passed).
psxrecomp-game bakes the same hash at ITS build time and prints it via
--codegen-hash. Any mismatch — including a binary too old to support the
flag — is a HARD failure: rebuild the recompiler, never build shards."""
import subprocess
try:
out = subprocess.run([recompiler, '--codegen-hash'],
capture_output=True, text=True, timeout=30)
except Exception as e:
raise SystemExit(f'FATAL: cannot execute {recompiler} for --codegen-hash '
f'staleness check: {e}')
line = (out.stdout or '').strip().splitlines()
baked = line[0].strip() if line else ''
if out.returncode != 0 or not re.fullmatch(r'[0-9a-fA-F]{8}', baked or ''):
raise SystemExit(
f'FATAL: {recompiler} does not support --codegen-hash (or errored).\n'
f' This binary predates the stale-recompiler guard and CANNOT be\n'
f' verified against the emitter sources. Rebuild it:\n'
f' cmake --build <recompiler-build-dir> --target psxrecomp-game')
if int(baked, 16) != tag_hash:
raise SystemExit(
f'FATAL: STALE RECOMPILER BINARY.\n'
f' {recompiler}\n'
f' was built from emitter sources hashing {baked}, but the runtime\n'
f' tree stamps cache tag hash {tag_hash:08x}. Shards it emits would\n'
f' carry the current tag with OLD codegen semantics (the silent\n'
f' stale-shard class). Rebuild the recompiler first:\n'
f' cmake --build <recompiler-build-dir> --target psxrecomp-game')
print(f'recompiler codegen hash verified: {baked} == cg tag hash')
def cache_tag(runtime_include: str, recompiler: str, game_toml: str,
flavor: int = 0) -> str:
"""THE cache-namespace string. Single source of truth.
The release packagers used to rebuild this string themselves in PowerShell.
That parallel implementation went stale the moment the flavor suffix was
added: the packager derived cg<N>_<hash>_gc<hash> while this tool wrote
cg<N>_<hash>_gc<hash>_f<flavor>, so its tag filter matched nothing and a
perfectly good regenerated cache staged ZERO shards (measured on Tomba 2,
2026-09-01). Anything that needs the tag must call this, never reformat it.
Must stay identical to overlay_loader.c scan_cache_dir().
"""
return 'cg%d_%08x_gc%08x_f%d' % (
codegen_ver(runtime_include),
codegen_hash(runtime_include),
overlay_config_hash(recompiler, game_toml),
int(flavor))
def overlay_config_hash(recompiler: str, game_toml: str) -> int:
"""Ask the recompiler for the canonical hash of config fields that affect
generated overlay code. Keeping the serializer in the shared C++ config
loader means the runtime and producer cannot disagree about which
widescreen/patch settings define a cache namespace."""
import subprocess
try:
out = subprocess.run(
[recompiler, '--overlay-config-hash', os.path.abspath(game_toml)],
capture_output=True, text=True, timeout=30)
except Exception as e:
raise SystemExit(
f'FATAL: cannot execute {recompiler} for --overlay-config-hash: {e}')
line = (out.stdout or '').strip().splitlines()
value = line[0].strip() if line else ''
if out.returncode != 0 or not re.fullmatch(r'[0-9a-fA-F]{8}', value):
detail = (out.stderr or out.stdout or '').strip()
raise SystemExit(
f'FATAL: {recompiler} could not hash overlay codegen config.\n'
f' This binary may predate --overlay-config-hash; rebuild it.\n'
f' detail: {detail}')
return int(value, 16)
_TARGET_OS = None
def set_target_os(target_os: str | None) -> None:
global _TARGET_OS
_TARGET_OS = target_os
def is_windows() -> bool:
if _TARGET_OS == 'win':
return True
if _TARGET_OS in ('linux', 'macos'):
return False
return (os.name == 'nt'
or platform.system() == 'Windows'
or platform.system().startswith(('MSYS', 'CYGWIN', 'MINGW')))
def overlay_ext() -> str:
"""Use the host platform's conventional shared-library suffix."""
return '.dll' if is_windows() else '.so'
def native_path(p: str) -> str:
"""Absolute form of `p`, in the RUNNING interpreter's own path flavor, for
handing to native tools (gcc, tcc). Two hard-won rules live here
(2026-07-15, runtime `cmd.exe /C python` resolved to devkitPro's MSYS
python and every in-game shard compile died exit-1/empty-stderr):
1. ABSOLUTE is mandatory: gcc invoked from cmd.exe silently fails on
relative forward-slash paths.
2. Do NOT translate flavors here. Under an MSYS-flavored python, POSIX
args (/f/..., /home/...) are translated to Windows form AT THE EXEC
BOUNDARY by the msys runtime using its OWN mount table — verified
correct (gcc -v showed /home/... arriving as C:/Users/...). Translating
here with `cygpath`/heuristics uses whatever foreign cygpath is first
on PATH (e.g. Git-for-Windows') and corrupts the path. The actual
silent-failure root cause was never the args — it was the child PATH
flavor (_toolchain_env below)."""
return os.path.abspath(p)
# The arch the shards are built for: the RUNTIME's, not this interpreter's.
# None means "this interpreter's" (manual/offline use). The spawning runtime
# pins it through PSX_OVERLAY_ARCH_ABI (autocompile.c), the same way it pins
# the cache dir and the flavor, because the interpreter, the compiler and the
# runtime need not share an architecture on macOS: an x86_64 runtime under
# Rosetta runs an x86_64 Python, but Xcode's clang is an arm64-only binary and
# defaults to arm64. Without -arch every shard it built was arm64, landed in
# the macos-x64 cache and never loaded, so the x64 runtime ran every overlay
# on the interpreter.
_TARGET_ARCH = None
_ARCH_ABI_OSES = ('win', 'linux', 'macos')
_ARCH_ABI_ARCHES = ('x64', 'arm64', 'x86', 'unknown')
# Apple's driver names for the arch-abi arches (clang -arch / gcc -arch).
_DARWIN_ARCH_NAMES = {'x64': 'x86_64', 'arm64': 'arm64', 'x86': 'i386'}
def _arch_abi_arch(machine: str) -> str:
"""A machine/arch name (uname, PE, MSYS2 tag) in arch-abi spelling."""
m = machine.lower()
if m in ('amd64', 'x86_64', 'x64'):
return 'x64'
if m in ('arm64', 'aarch64'):
return 'arm64'
if m in ('i386', 'i686', 'x86'):
return 'x86'
return 'unknown'
def interpreter_arch() -> str:
"""Arch of THIS Python process, in arch-abi spelling: the only
architecture ctypes can load here.
macOS and Linux: platform.machine() is uname's, which is the process's
own (x86_64 for a Python running under Rosetta).
Native Windows: platform.machine() is the MACHINE's, not the process's
(Win32_Processor through WMI since Python 3.12, PROCESSOR_ARCHITEW6432
for a WOW64 process before), so an x64 Python emulated on Windows on ARM
says ARM64 and a 32-bit one on x64 Windows says AMD64. The interpreter's
own build names the process there: sysconfig.get_platform() is
win-amd64 / win-arm64 / win32 for python.org builds (from the compiler
tag in sys.version) and mingw_<arch>_<crt>_<cc> for MSYS2's."""
if os.name == 'nt':
plat = sysconfig.get_platform().lower()
if plat == 'win32':
return 'x86'
m = re.match(r'(?:win-|mingw_)(amd64|arm64|x86_64|aarch64|i686)(?:_|$)', plat)
if m:
return _arch_abi_arch(m.group(1))
# Older or unrecognized Python builds have no process-architecture
# tag. Preserve the previous machine-name fallback for these callers.
return _arch_abi_arch(platform.machine())
return _arch_abi_arch(platform.machine())
def target_os_tag() -> str:
"""The "<os>" half of the arch-abi tag the shards are built for."""
if _TARGET_OS in _ARCH_ABI_OSES:
return _TARGET_OS
if is_windows():
return 'win'
return {'Darwin': 'macos'}.get(platform.system(), 'linux')
def set_target_arch(arch: str | None) -> None:
global _TARGET_ARCH
_TARGET_ARCH = arch
def target_arch() -> str:
"""The "<arch>" half: the runtime's when it said so, else this interpreter's."""
return _TARGET_ARCH or interpreter_arch()
def cache_arch_abi() -> str:
"""Canonical cache arch-abi tag, IDENTICAL to overlay_loader.h's
PSX_OVERLAY_ARCH_ABI ("<os>-<arch>": win|linux|macos + x64|arm64|x86).
gcc DLLs are namespaced under <game_id>/gcc/<arch-abi>/ so same-OS
different-arch caches never comingle. Keep this
mapping in lockstep with overlay_loader.h."""
return f'{target_os_tag()}-{target_arch()}'
def apply_runtime_arch_abi(value: str | None, source: str) -> str | None:
"""Build for the runtime's "<os>-<arch>" (PSX_OVERLAY_ARCH_ABI or
--arch-abi). Returns None when adopted (or nothing was given), else why
not. The OS half must agree with the OS the shards are built for: a
disagreement means the cache would be written where no loader reads."""
if not value:
return None
value = value.strip()
os_tag, sep, arch = value.partition('-')
if not sep or os_tag not in _ARCH_ABI_OSES or arch not in _ARCH_ABI_ARCHES:
return (f'{source}={value!r} is not "<os>-<arch>" '
f'(os: {"|".join(_ARCH_ABI_OSES)}; '
f'arch: {"|".join(_ARCH_ABI_ARCHES)})')
if os_tag != target_os_tag():
return (f'{source}={value!r} names {os_tag}, but these shards are '
f'built for {target_os_tag()}')
set_target_arch(arch)
return None
def target_arch_flags(compiler: str = 'gcc') -> list[str]:
"""Compiler flags that make a shard's architecture the target's.
macOS only: Apple's cc defaults to the architecture of the compiler
binary, not of whoever runs it, so the target is always named. Other
platforms are left exactly as they were (their toolchains target the one
architecture the runtime and the bundled Python share)."""
if compiler != 'gcc' or target_os_tag() != 'macos':
return []
name = _DARWIN_ARCH_NAMES.get(target_arch())
# Non-Apple GCC passed via --gcc is unsupported on macOS; it may ignore -arch.
return ['-arch', name] if name else []
def interpreter_arch_mismatch() -> str | None:
"""Why this interpreter cannot build shards for the target, or None.
Every shard is validated by loading it into this interpreter (ctypes:
overlay_abi, pair id, exports) before it counts, and a process can only
load its own architecture. So a mismatch cannot produce a usable cache;
say so instead of writing shards nothing will load."""
have = interpreter_arch()
want = target_arch()
if have == want or have == 'unknown':
# 'unknown': nothing proves a mismatch; each shard's load check
# still decides, as before this guard existed.
return None
return (f'the runtime is {cache_arch_abi()} but this Python runs as '
f'{have} ({sys.executable}); compiled shards are validated by '
f'loading them here, so the Python must run as {want} too. Use a '
f'Python built for {want} (a universal one runs as its caller\'s '
f'architecture).')
# ---------------------------------------------------------------------------
# Code segments (docs/SEGMENT_AWARE_CODE.md §5.7)
# ---------------------------------------------------------------------------
# The bytes of an overlay are segment-free; execution is not. A PC's segment
# (KUSEG 0x0xxxxxxx, KSEG0 0x8xxxxxxx, KSEG1 0xAxxxxxxx) is part of every link,
# EPC, I-cache tag and store-PC stamp a compiled body bakes, so a shard serves
# only the segment it was compiled for. Capture schema v3 records the segments
# each dispatch entry entered through (`dispatch_entry_segments`); a capture
# without it is KSEG0 only. This tool compiles one shard per (region, segment
# with entries): the image is compiled at `segment | phys`, so its functions,
# manifest F entries and exports are named by that VA (§5.1). KSEG0 shards keep
# their place and names in the cache tag directory, so no existing cache is
# invalidated; KUSEG and KSEG1 shards go in its seg-kuseg/ and seg-kseg1/
# subdirectories (decision 4), under the same {phys}_{crc} filename grammar,
# and their manifests carry `S <segment>`. R records stay KSEG0-spelled byte
# extents: they validate bytes, not execution.
KUSEG, KSEG0, KSEG1 = 0x00000000, 0x80000000, 0xA0000000
SEGMENT_MASK, PHYS_MASK = 0xE0000000, 0x1FFFFFFF
SEGMENT_NAMES = {KUSEG: 'kuseg', KSEG0: 'kseg0', KSEG1: 'kseg1'}
SEGMENT_BY_NAME = {name: seg for seg, name in SEGMENT_NAMES.items()}
SEGMENT_CACHE_SUBDIRS = {KUSEG: 'seg-kuseg', KSEG1: 'seg-kseg1'}
# The segment of the image being compiled. Helpers that name PCs without an
# image address at hand spell them in it (_canonical_guest_addr); every
# per-image entry point below sets it for its duration (image_segment). The
# default is KSEG0, the only segment before §5.7, so a KSEG0 compile is
# unchanged. A context variable, not a global: captures compile on threads.
_IMAGE_SEGMENT = contextvars.ContextVar('psx_overlay_image_segment',
default=KSEG0)
def segment_of(addr: int) -> int:
return addr & SEGMENT_MASK
@contextmanager
def image_segment(seg: int):
if seg not in SEGMENT_NAMES:
raise ValueError(f'0x{seg:08X} is not a segment that maps RAM')
token = _IMAGE_SEGMENT.set(seg)
try:
yield seg
finally:
_IMAGE_SEGMENT.reset(token)
def _canonical_guest_addr(addr: int) -> int:
"""``addr``'s bytes, spelled in the segment of the image being compiled."""
return (addr & PHYS_MASK) | _IMAGE_SEGMENT.get()
def segment_cache_dir(cache_dir: str, seg: int) -> str:
"""The directory a shard compiled for ``seg`` lives in (decision 4)."""
sub = SEGMENT_CACHE_SUBDIRS.get(seg)
return os.path.join(cache_dir, sub) if sub else cache_dir
def cache_dir_segment(directory: str) -> int:
"""The segment of the shards in ``directory`` (overlay_loader.c's
cache_seg_from_path): a seg-kuseg/ or seg-kseg1/ directory, else KSEG0."""
name = os.path.basename(os.path.normpath(directory)).lower()
for seg, sub in SEGMENT_CACHE_SUBDIRS.items():
if name == sub:
return seg
return KSEG0
def _respell(addr: int, seg: int) -> int:
return (addr & PHYS_MASK) | seg
def capture_segment_views(cap: dict,
required_segments=frozenset()) -> list[dict]:
"""Split one capture record into one per segment with entries (§5.7).
A v2 capture (no ``dispatch_entry_segments``) is KSEG0 only and is
returned unchanged, so every existing capture compiles exactly as before.
For v3, ``dispatch_entry_pcs`` keeps its v2 meaning (dispatched in any
segment, spelled at KSEG0) and ``dispatch_entry_segments`` names the
segments: {"kuseg": [...], "kseg0": [...], "kseg1": [...]}, each PC in its
own segment. A dispatch entry that no segment list names is KSEG0.
A view is the capture seen from one segment: ``load_addr`` and every PC in
it are spelled in that segment, and its dispatch entries are that
segment's alone. The execution evidence (executed_pcs) has no segment --
direct edges keep the entry's segment -- so each view carries all of it.
The bytes, size and guard words are shared.
``required_segments`` also gets a view where the record has no dispatch
entry: the segments an operator or config demand names inside it
(declared_view_segments). A record with no view at all -- execution
evidence only, which the runtime writes -- is read as a v2 reader reads
it: one KSEG0 view, so its classification, forced interiors, declared
entries and prior manifests apply exactly as before.
"""
segments = cap.get('dispatch_entry_segments')
if segments is None:
return [cap]
load = _parse_addr(cap['load_addr'])
all_entries = {a & PHYS_MASK
for a in _parse_addr_list(cap.get('dispatch_entry_pcs', []))}
by_seg = {seg: set() for seg in SEGMENT_NAMES}
labelled = set()
if not isinstance(segments, dict):
raise RuntimeError('dispatch_entry_segments must be an object')
for name, pcs in segments.items():
seg = SEGMENT_BY_NAME.get(name)
if seg is None:
raise RuntimeError(f'dispatch_entry_segments: unknown segment {name!r}')
for pc in _parse_addr_list(pcs):
if segment_of(pc) != seg:
raise RuntimeError(
f'dispatch_entry_segments.{name}: 0x{pc:08X} is not in {name}')
by_seg[seg].add(pc & PHYS_MASK)
labelled.add(pc & PHYS_MASK)
# Entries with no segment record are KSEG0, as in a v2 capture.
by_seg[KSEG0] |= all_entries - labelled
def respell_list(values, seg, keep=None):
out = []
for value in _parse_addr_list(values or []):
if keep is not None and (value & PHYS_MASK) not in keep:
continue
out.append(f'0x{_respell(value, seg):08X}')
return sorted(out)
wanted = {seg for seg, entries in by_seg.items() if entries}
wanted |= {seg for seg in required_segments if seg in by_seg}
if not wanted:
wanted = {KSEG0}
views = []
for seg in (KSEG0, KUSEG, KSEG1):
if seg not in wanted:
continue
entries = by_seg[seg]
view = {k: v for k, v in cap.items() if k != 'dispatch_entry_segments'}
view['load_addr'] = f'0x{_respell(load, seg):08X}'
view['dispatch_entry_pcs'] = [f'0x{_respell(a, seg):08X}'
for a in sorted(entries)]
view['seeds'] = list(view['dispatch_entry_pcs'])
if 'static_dispatch_entry_pcs' in cap:
view['static_dispatch_entry_pcs'] = respell_list(
cap['static_dispatch_entry_pcs'], seg, entries)
for field in ('executed_pcs', 'observed_pcs', 'function_entry_pcs',
'static_discovery_entry_pcs',
'static_call_continuation_pcs'):
if field in cap:
view[field] = respell_list(cap[field], seg)
if cap.get('producer_ranges'):
view['producer_ranges'] = [
dict(r, start=f'0x{_respell(_parse_addr(r["start"]), seg):08X}',
end=f'0x{_respell(_parse_addr(r["end"]), seg):08X}')
for r in cap['producer_ranges']]
if cap.get('static_alias_ranges'):
view['static_alias_ranges'] = [
dict(r, entry=f'0x{_respell(_parse_addr(r["entry"]), seg):08X}',
start=f'0x{_respell(_parse_addr(r["start"]), seg):08X}',
end=f'0x{_respell(_parse_addr(r["end"]), seg):08X}')
for r in cap['static_alias_ranges']]
views.append(view)
return views
def declared_view_segments(cap: dict, toml_doc: dict,
forced_interiors=frozenset()) -> set[int]:
"""Segments a demand from outside the capture names inside ``cap`` (§5.7).
A ``--force-interior`` PC is KSEG0 (a physical and a KUSEG PC are the same
number), so one inside the record asks for its KSEG0 view. A game.toml
``[[overlays]]`` table matches the load address it spells, and one without
a load address applies its entries by their own spelling, so each names
the segment it is written in. capture_segment_views() gives every such
segment a view even where the capture saw no dispatch entry, so these
demands compile as they did before captures carried segments. Byte-CRC
checks stay with the compile (_collect_toml_overlay_entries): a detached
table's view compiles nothing."""
load = _parse_addr(cap['load_addr'])
lo = load & PHYS_MASK
hi = lo + int(cap['size'])
inside = lambda pc: lo <= (pc & PHYS_MASK) < hi
segs = {KSEG0} if any(inside(pc) for pc in forced_interiors) else set()
for ov in (toml_doc or {}).get('overlays', []) or []:
if not isinstance(ov, dict):
continue
entries = set()
for key in ('entry', 'entries', 'function_entry_pcs', 'function_entries'):
val = ov.get(key)
if isinstance(val, list):
entries.update(_parse_addr_list(val))
elif val is not None:
entries.add(_parse_addr(val))
ov_load = ov.get('load_addr') or ov.get('load_address')
if ov_load is not None:
ov_load = _parse_addr(ov_load)
if entries and (ov_load & PHYS_MASK) == lo:
segs.add(segment_of(ov_load))
else:
segs.update(segment_of(pc) for pc in entries if inside(pc))
return {seg for seg in segs if seg in SEGMENT_NAMES}
# ---------------------------------------------------------------------------
# Shard build accounting — make failures LOUD
# ---------------------------------------------------------------------------
# Historically a per-shard failure (recompiler error, generated-C audit reject,
# gcc/tcc compile error, dropped interior fragment) only printed a line into an
# ephemeral log and the script still exited 0. The runtime's autocompile watcher
# keys "did it work?" off the exit code, so a header change that broke EVERY
# shard looked identical to a fully-successful run: the affected code just ran
# interpreted forever. ShardStats fixes that — every build outcome is tallied
# (thread-safe, since captures compile on a pool), a SUMMARY prints at the end,
# a machine-readable PSX_SHARD_RESULT line is emitted for the runtime to parse,
# and main() exits non-zero when any shard that SHOULD have built failed.
class ShardStats:
def __init__(self):
self._lock = threading.Lock()
self.ok = 0
self.skipped = 0
self.capacity_fastpath = 0
self.fail_by_class = Counter()
self.failures = [] # [(label, cls, detail), ...] (detail truncated)
def add_ok(self, n=1):
with self._lock:
self.ok += n
def add_skip(self, n=1):
with self._lock:
self.skipped += n
def add_capacity_fastpath(self, n=1):
with self._lock:
self.capacity_fastpath += n
def add_fail(self, label, cls, detail=''):
# Print immediately AND record, so the failure is visible in the live
# stream (per-worker buffered) and in the end-of-run summary.
d = (detail or '').strip().replace('\n', ' ')
if len(d) > 240:
d = d[:237] + '...'
print(f' SHARD FAIL [{cls}] {label}{(": " + d) if d else ""}')
with self._lock:
self.fail_by_class[cls] += 1
self.failures.append((label, cls, d))
def total_fail(self):
with self._lock:
return sum(self.fail_by_class.values())
def print_summary(self):
with self._lock:
fail = sum(self.fail_by_class.values())
print('\n=== SHARD BUILD SUMMARY ===')
print(f' built OK : {self.ok}')
print(f' skipped : {self.skipped} '
'(cached / data-only / safe coverage loss)')
if self.capacity_fastpath:
print(f' cap-fast : {self.capacity_fastpath} cache recipe '
'job(s) skipped at an already-full fixed point')
print(f' FAILED : {fail}')
for cls, n in sorted(self.fail_by_class.items()):
print(f' - {cls}: {n}')
for label, cls, detail in self.failures[:40]:
print(f' ! [{cls}] {label}{(": " + detail) if detail else ""}')
if len(self.failures) > 40:
print(f' ... {len(self.failures) - 40} more')
# Stable, grep-able result line the runtime (autocompile.c) parses to
# surface shard_ok / shard_fail without depending on the exit code.
print(f'PSX_SHARD_RESULT ok={self.ok} failed={fail} '
f'skipped={self.skipped} '
f'capacity_fastpath={self.capacity_fastpath}')
return fail
# ---------------------------------------------------------------------------
# PS-EXE fake header
# ---------------------------------------------------------------------------
# Analysis-bound tag read by the recompiler (recompiler/include/ps1_exe_parser.h,
# namespace exe_tag). Offsets are into the 2048-byte PS-EXE header, whose tail
# is zero-filled in every retail image; the 8-byte magic makes a false positive
# on a genuine EXE effectively impossible.
GUARD_TAG_MAGIC_OFFSET = 0x7E0
GUARD_TAG_COUNT_OFFSET = 0x7E8
GUARD_TAG_MAGIC = b'PSXRGRD1'
CAPTURE_PAGE_BYTES = 4096
def capture_guard_bytes(cap: dict, size: int, label: str = '') -> int:
"""Trailing bytes of a captured region that are guard words ONLY.
overlay_capture.c write_json_window appends one coherent guard instruction
past the end of a dirty-page run so that a MIPS branch at the run's final
word (...FFC) has its architectural delay slot (...000) available. Those
bytes must stay READABLE by the recompiler and must NEVER be discovered as
code: the word after a guard word is not in the image, so a control
transfer AT the guard word could not emit its own delay slot. The
recompiler is TOLD the count (see make_psxexe) rather than inferring it.
Precedence:
1. the capture's explicit ``guard_bytes`` field — the writer stating its
own intent, which survives any future change to capture granularity;
2. for captures recorded BEFORE that field existed, reconstruct from the
format invariant write_json_window has always obeyed: a dirty-page run
is a whole number of 4 KiB pages, plus one 4-byte guard word whenever
the next word is still inside RAM. ``size % 4096 == 4`` is exactly
that signature and means one guard word.
Every OTHER shape reconstructs to zero, and that is ordinary traffic rather
than an anomaly, so this stays quiet:
* ``size % 4096 == 0`` — a page run whose guard word was suppressed
because it would have left RAM, or a pre-2026-07-25 page-exact capture.
* any other residue — a STATIC extraction, not a RAM page run. Those
records carry an exact file extent (tools/aot_overlay_spike/
extract_generic.py; they are the ones with static_dispatch_entry_pcs /
producer_ranges) and append no guard word. MEASURED: 52 of the 946
records in the SCUS-94454 vault are this shape, with residues from 112
to 3996 bytes and one that is not even 4-byte aligned.
Zero is also the fail-closed answer. If some future producer ever appends a
guard word in a shape this cannot recognise, that word is analysed as code
and the emitter's mandatory-delay-slot check refuses the transfer LOUDLY at
generation time; it never silently emits a transfer without its slot. The
way to opt in is the explicit field, not a wider guess here.
"""
declared = cap.get('guard_bytes') if cap else None
if declared is not None:
try:
guard = int(declared)
except (TypeError, ValueError):
guard = -1
if guard in (0, 4) and guard < size:
return guard
# A malformed DECLARATION is a producer bug, unlike the shapes above.
print(f' WARNING: {label or "capture"} declares guard_bytes='
f'{declared!r} with size {size}; ignoring (treating as 0)')
return 0
if size % CAPTURE_PAGE_BYTES == 4 and size > 4:
return 4
return 0
def make_psxexe(load_addr: int, entry_pc: int, data: bytes, *,
guard_bytes: int) -> bytes:
"""Wrap raw overlay bytes in a minimal PS-EXE header.
``guard_bytes`` is keyword-only and has NO default on purpose: every
producer of an overlay image must state how many of its trailing bytes are
delay-slot guard words (see capture_guard_bytes). A new call site that
forgets is a TypeError, not a silently mis-analysed shard.
"""
if guard_bytes < 0 or guard_bytes % 4 or guard_bytes >= len(data):
raise ValueError(
f'invalid guard_bytes={guard_bytes} for {len(data)}-byte image')
header = bytearray(2048)
header[0:8] = b'PS-X EXE'
struct.pack_into('<I', header, 0x10, entry_pc) # initial PC
struct.pack_into('<I', header, 0x14, 0) # initial GP
struct.pack_into('<I', header, 0x18, load_addr) # load address
struct.pack_into('<I', header, 0x1C, len(data)) # text size
if guard_bytes:
header[GUARD_TAG_MAGIC_OFFSET:
GUARD_TAG_MAGIC_OFFSET + len(GUARD_TAG_MAGIC)] = GUARD_TAG_MAGIC
struct.pack_into('<I', header, GUARD_TAG_COUNT_OFFSET, guard_bytes)
return bytes(header) + data
# ---------------------------------------------------------------------------
# Overlay seed and generated-C audits
# ---------------------------------------------------------------------------
INCLUDE_REASONS = {
'DISPATCH_ENTRY',
# Export/header/jump-table entry proven directly by a static extractor.
# It passes the same callable-boundary gate as a generic dispatch entry,
# but retains provenance that may nominate the address in sibling bytes.
'STATIC_DISPATCH_ENTRY',
'DIRECT_JAL_TARGET',
'STATIC_INDIRECT_TARGET',
'STATIC_BRANCH_ROOT',
# Play-free normal-mode discovery entry that also passes a target-local
# callable-boundary/CFG proof. Stronger than a generic captured pointer.
'STATIC_DISCOVERY_ROOT',
'FUNCTION_POINTER_TARGET',
'TOML_DECLARED_ENTRY',
# Dispatch-proven PC with no callable boundary (mid-function code reached
# through a function pointer or dispatch chain). Compiled as an
# overlapping-alias entry into its host function — written to the seeds
# file as 'interior 0x...' so the recompiler never uses it as a walk root
# (a walk root there would truncate the host: the mid-function-seed
# softlock class).
'DISPATCH_INTERIOR',
# Kernel-window-only promotion of an ORPHAN dispatch interior: a
# dispatch-proven PC in kernel RAM [0, 0x10000) that no rooted walk
# covers. There is no host to alias into and no host a root could
# truncate — the static recompiler's install-slot hooks tail-dispatch
# into exactly such PCs (e.g. RAM 0xCF0, the SIO data-byte stub).
# Written to the seeds file as 'dispatch_root 0x...': a trusted walk
# root, exempt from the recompiler's boundary re-check. Overlay regions
# are NOT eligible: per-PC dispatch evidence persists across scene
# variants there, so an orphan may belong to a non-resident variant
# whose bytes in THIS image are data (the 0xE889C class) — rooting it
# would walk garbage.
'DISPATCH_ROOT',
}
EXACT_FRAGMENT_REASONS = INCLUDE_REASONS - {'DISPATCH_INTERIOR'}
# Cross-variant propagation uses only explicit static code-flow evidence or a
# statically extracted export/header dispatch entry. Runtime observations,
# generic pointer scans, TOML entries, and promoted dispatch roots are useful
# local evidence, but are not durable sibling-variant nominations.
CROSS_VARIANT_DONOR_REASONS = {
'DIRECT_JAL_TARGET',
'STATIC_INDIRECT_TARGET',
'STATIC_BRANCH_ROOT',
'STATIC_DISCOVERY_ROOT',
'STATIC_DISPATCH_ENTRY',
}
# A recipient host is not inferred: it must already be a runtime-valid exact F
# entry for these bytes and retain one of the classifier's local root reasons.
HOSTED_OWNER_REASONS = EXACT_FRAGMENT_REASONS - {'DISPATCH_ROOT'}
FATAL_SEED_REASONS = {'BRANCH_TARGET_ONLY', 'OBSERVED_PC_ONLY', 'UNKNOWN'}
def recompiler_project_root_args(args) -> list:
"""--project-root arguments for a psxrecomp-game invocation, or [].
Every recompiler spawn below runs with cwd = dirname(game.toml), because
the seeds/captures paths in the config are relative to it. That cwd is
also where the recompiler probes for the BIOS profile when the config has
no explicit [recompiler] bios_config — and for a PACKAGED game.toml
(packaging/release/game.toml) it holds neither bios/SCPH1001.toml nor a
vendored framework one level down. Every shard then died with
"FATAL: no BIOS profile found" and the cache silently failed to build,
which is what the recipe printed by package_release.ps1 used to tell
people to run (issue #72).
Default the root to the framework, derived from the required
--runtime-include (<framework>/runtime/include). Derivation is used ONLY
when it actually locates a profile, so this can fix a broken invocation
but never alter one that already worked.
Every lookup is defensive: in-process callers (the test suite, the
packager's importlib entry) build a lightweight args object that carries
only the fields their recipe needs, so a missing attribute must degrade to
"pass no flag" — today's behaviour — never raise.
"""
explicit = getattr(args, 'project_root', None)
if explicit:
return ['--project-root', os.path.abspath(explicit)]
runtime_include = getattr(args, 'runtime_include', None)
if not runtime_include:
return []
derived = os.path.abspath(
os.path.join(os.path.abspath(runtime_include), '..', '..'))
if os.path.isfile(os.path.join(derived, 'bios', 'SCPH1001.toml')):
return ['--project-root', derived]
return []
BIOS_RESIDENT_PRODUCER = 'bios_resident_manifest'
BIOS_RESIDENT_MARKER = 'psxrecomp bios resident shard v1'
# Hosted aliases are supplemental optimizations, never correctness authority.
# Keep every phase deterministically bounded so adversarial/malformed capture
# sets cannot turn one compile invocation into unbounded shard fan-out.
HOSTED_NOMINATION_CAP = 4096
# Python selection precedes the C++ organic-block proof, so most candidates are
# expected to be rejected without publication. The emitted-identity cap below
# remains the tighter runtime-facing limit.
HOSTED_SELECTED_ALIAS_CAP = 4096
HOSTED_SELECTED_HOST_CAP = 512
HOSTED_BATCH_HOST_CAP = 64
HOSTED_BATCH_ALIAS_CAP = 256
# A terminal partition/partial-success tree over N targets has at most 2N-1
# nodes. Budget the entire selected universe so a second invocation can never
# resume an attempt-capped suffix.
HOSTED_COMPILE_ATTEMPT_CAP = 2 * HOSTED_SELECTED_ALIAS_CAP - 1
HOSTED_EMITTED_IDENTITY_CAP = 1024
MANIFEST_LINE_MAX = 128
MANIFEST_PHYSICAL_LINE_MAX = 159
MANIFEST_ASCII_WHITESPACE = ' \t\r\v\f'
HOSTED_MANIFEST_PROVENANCE = 'hosted-v1'
ORPHAN_MANIFEST_PROVENANCE = 'orphan-v1'
NON_AUTHORITY_MANIFEST_PROVENANCES = {
HOSTED_MANIFEST_PROVENANCE, ORPHAN_MANIFEST_PROVENANCE,
}
HOSTED_MANIFEST_MARKER = (
f'# psxrecomp overlay provenance {HOSTED_MANIFEST_PROVENANCE}')
HOSTED_UNIQUE_GUARDED_BYTE_CAP = 1024 * 1024
# Full 8 MiB host capacity (PSX_MAIN_RAM_BACKING_BYTES): 8 MB-mod captures carry
# high-bank enhancement code and their ranges must validate.
PSX_RAM_SIZE = 8 * 1024 * 1024
class ShardCandidateCapacityError(RuntimeError):
"""A cache publication would exceed the runtime's fixed F-record table."""
def manifest_has_hosted_provenance(manifest: str) -> bool:
"""Recognize the exact compiler-owned marker under LF or CRLF records."""
return manifest_provenance(manifest) == HOSTED_MANIFEST_PROVENANCE
def manifest_declares_supplemental_provenance(manifest: str) -> bool:
"""Fail closed for any compiler provenance record, known or malformed."""
prefix = '# psxrecomp overlay provenance '
return any(
(raw_line[:-1] if raw_line.endswith('\r') else raw_line).startswith(
prefix)
for raw_line in manifest.split('\n')
)
def manifest_provenance(manifest: str) -> str | None:
"""Return one exact compiler-owned non-authority provenance record."""
prefix = '# psxrecomp overlay provenance '
found = []
for raw_line in manifest.split('\n'):
line = raw_line[:-1] if raw_line.endswith('\r') else raw_line
if line.startswith(prefix):
found.append(line[len(prefix):])
if len(found) != 1 or found[0] not in NON_AUTHORITY_MANIFEST_PROVENANCES:
return None
return found[0]
def seed_line_for_reason(addr: int, reason: str) -> str:
"""Serialize one classifier reason without weakening its root contract."""
if reason == 'DISPATCH_INTERIOR':
return f'interior 0x{addr:08X}'
if reason == 'DISPATCH_ROOT':
return f'dispatch_root 0x{addr:08X}'
if reason in ('DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET',
'STATIC_BRANCH_ROOT', 'STATIC_DISCOVERY_ROOT'):
return f'call_root 0x{addr:08X}'
return f'0x{addr:08X}'
def update_bios_resident_marker(dll_path: str, cap: dict) -> None:
"""Publish/remove the opt-in marker consumed by the runtime preloader.
Resident BIOS helpers use a synthetic page envelope, so their filename's
region key cannot equal the dirty-run start recovered at runtime. The
marker tells the loader to register the shard at cache scan time; normal
per-function code-CRC validation remains the execution authority.
"""
marker = os.path.splitext(dll_path)[0] + '.resident'
if cap.get('producer') != BIOS_RESIDENT_PRODUCER:
try:
os.remove(marker)
except FileNotFoundError:
pass
return
payload = {
'schema': BIOS_RESIDENT_MARKER,
'bios_sha256': str(cap.get('bios_sha256', '')),
'producer_name': str(cap.get('producer_name', 'BIOS resident code')),
}
serialized = json.dumps(payload, sort_keys=True) + '\n'
try:
with open(marker, 'r', encoding='utf-8') as f:
if f.read(len(serialized) + 1) == serialized:
return
except (FileNotFoundError, OSError, UnicodeError):
pass
tmp = f'{marker}.tmp.{uuid.uuid4().hex}'
try:
with open(tmp, 'w', encoding='utf-8') as f:
f.write(serialized)
os.replace(tmp, marker)
finally:
_best_effort_unlink(tmp)
def reconcile_bios_resident_marker(dll_path: str, cap: dict,
published: bool) -> None:
"""Keep the winner's residency, while letting resident evidence self-heal.
A losing ordinary writer must not remove a resident winner's sidecar. A
resident capture may always ensure the marker: identical region bytes make
the preserved shard equally safe to preload, and this repairs a crash
between pair publication and sidecar publication.
"""
# Once this exact canonical region identity has verified resident evidence,
# retaining the marker is safe even if an ordinary capture won the compile
# race: every function still passes the live per-range CRC guard. Never let
# an ordinary writer race a resident writer by removing that evidence.
if cap.get('producer') == BIOS_RESIDENT_PRODUCER:
update_bios_resident_marker(dll_path, cap)
def _parse_addr(value) -> int:
if isinstance(value, int):
return value
return int(str(value), 16)
def _parse_addr_list(values) -> set[int]:
out = set()
for v in values or []:
try:
out.add(_parse_addr(v))
except (TypeError, ValueError):
pass
return out
def optional_enrichment_fallback_capture(cap: dict) -> dict | None:
"""Return a conservative retry recipe declared by a static extractor.
The richer recipe is always attempted first. If its generated C or host
compile is rejected, these independently proven roots preserve the safe
shard while all optional aliases and normal-mode discoveries are dropped.
"""
key = 'optional_enrichment_fallback_entry_pcs'
recipe = cap.get(key, [])
fallback = dict(cap)
if isinstance(recipe, dict):
functions = sorted(_parse_addr_list(
recipe.get('function_entry_pcs', [])))
dispatch = sorted(_parse_addr_list(
recipe.get('dispatch_entry_pcs', [])))
static = sorted(_parse_addr_list(
recipe.get('static_discovery_entry_pcs', [])))
static_dispatch = sorted(_parse_addr_list(
recipe.get('static_dispatch_entry_pcs', [])))
if not (functions or dispatch or static):
return None
fallback['function_entry_pcs'] = [
f'0x{entry:08X}' for entry in functions]
fallback['dispatch_entry_pcs'] = [
f'0x{entry:08X}' for entry in dispatch]
fallback['static_dispatch_entry_pcs'] = [
f'0x{entry:08X}' for entry in static_dispatch
if entry in set(dispatch)]
fallback['static_discovery_entry_pcs'] = [
f'0x{entry:08X}' for entry in static]
fallback['seeds'] = fallback['function_entry_pcs']
if 'producer_ranges' in recipe:
fallback['producer_ranges'] = recipe['producer_ranges']
if 'strict_producer_ranges' in recipe:
fallback['strict_producer_ranges'] = bool(
recipe['strict_producer_ranges'])
else:
entries = sorted(_parse_addr_list(recipe))
if not entries:
return None
encoded = [f'0x{entry:08X}' for entry in entries]
fallback['function_entry_pcs'] = encoded
fallback['dispatch_entry_pcs'] = encoded
fallback['static_dispatch_entry_pcs'] = []
fallback['static_discovery_entry_pcs'] = encoded
fallback['seeds'] = encoded
fallback.pop('static_alias_ranges', None)
fallback.pop('static_jump_table_proofs', None)
fallback.pop('static_call_continuation_pcs', None)
fallback.pop('_prior_aliases', None)
fallback.pop(key, None)
return fallback
def _word_at(data: bytes, load_addr: int, addr: int):
off = addr - load_addr
if off < 0 or off + 4 > len(data):
return None
return struct.unpack_from('<I', data, off)[0]
def _is_jr_ra(word) -> bool:
return word == 0x03E00008
def _is_addiu_sp_neg(word) -> bool:
if word is None:
return False
return ((word >> 26) & 0x3F) == 0x09 \
and ((word >> 21) & 0x1F) == 29 \
and ((word >> 16) & 0x1F) == 29 \
and (word & 0x8000) != 0
def _is_control_flow(word) -> bool:
if word is None:
return False
op = (word >> 26) & 0x3F
fn = word & 0x3F
return (op in (0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x14, 0x15, 0x16, 0x17) or
(op == 0 and fn in (0x08, 0x09)) or
(op in (0x11, 0x12) and ((word >> 21) & 0x1F) == 0x08))
def _is_valid_mips_word(word) -> bool:
if word is None or word in (0xFFFFFFFF, 0xFFFFFFFD):
return False
op = (word >> 26) & 0x3F
fn = word & 0x3F
rt = (word >> 16) & 0x1F
if op == 0x00:
return fn in {
0x00, 0x02, 0x03, 0x04, 0x06, 0x07,
0x08, 0x09, 0x0C, 0x0D,
0x10, 0x11, 0x12, 0x13,
0x18, 0x19, 0x1A, 0x1B,
0x20, 0x21, 0x22, 0x23,
0x24, 0x25, 0x26, 0x27,
0x2A, 0x2B,
}
if op == 0x01:
return rt in (0x00, 0x01, 0x10, 0x11)
return op in {
0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
0x10, 0x12,
0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26,
0x28, 0x29, 0x2A, 0x2B, 0x2E,
0x30, 0x32, 0x38, 0x3A,
}
def _jump_target(pc: int, word: int) -> int:
return ((pc + 4) & 0xF0000000) | ((word & 0x03FFFFFF) << 2)
def _branch_target(pc: int, word: int) -> int:
imm = word & 0xFFFF
if imm & 0x8000:
imm -= 0x10000
return pc + 4 + (imm << 2)
def _classify_cf(pc: int, word: int) -> tuple[str, int]:
op = (word >> 26) & 0x3F
fn = word & 0x3F
rs = (word >> 21) & 0x1F
if op == 0 and fn == 0x08:
return ('jr_ra' if rs == 31 else 'jr', 0)
if op == 0 and fn == 0x09:
return ('jalr', 0)
if op == 0x02:
return ('j', _jump_target(pc, word))
if op == 0x03:
return ('jal', _jump_target(pc, word))
rt = (word >> 16) & 0x1F
if op in (0x04, 0x14) and rs == rt:
return ('j', _branch_target(pc, word))
if op == 0x05 and rs == rt:
return ('branch_never', 0)
if op == 0x15 and rs == rt:
return ('branch_never_likely', 0)
if op in (0x06, 0x16) and rs == 0:
return ('j', _branch_target(pc, word))
if op == 0x07 and rs == 0:
return ('branch_never', 0)
if op == 0x17 and rs == 0:
return ('branch_never_likely', 0)
if op == 0x01 and rs == 0:
if rt in (0x00, 0x10): # bltz / bltzal
return ('branch_never', 0)
if rt in (0x02, 0x12): # bltzl / bltzall
return ('branch_never_likely', 0)
if rt in (0x01, 0x03): # bgez / bgezl
return ('j', _branch_target(pc, word))
if rt in (0x11, 0x13): # bal / ball aliases: target + continuation
return ('jal', _branch_target(pc, word))
if op in (0x11, 0x12) and rs == 0x08:
return ('branch', _branch_target(pc, word))
if op in (0x01, 0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17):
return ('branch', _branch_target(pc, word))
return ('normal', 0)
def _find_jump_table_targets(data: bytes, load_addr: int, size: int,
entry: int, hard_cap: int,
jr_pc: int, jr_rs: int,
producer_range=None, proof_out=None) -> set[int]:
"""Resolve one tightly bounded canonical ``sltiu``/``jr`` switch.
This is deliberately a structural reaching-definition proof, not the old
40-word bag-of-opcodes backscan. The accepted suffix is::
sltiu guard,index,count; beq guard,zero,reject; nop
sll scaled,index,2; addu address,scaled,table_base
lw target,offset(address); [nop x 0..1]; jr target
Also accept ``sltiu; beq; lui; addiu; sll``: the LUI executes in the
guard's delay slot, then ADDIU completes the base on the in-range path.
That exact pair may not overwrite the checked index, and direct edges
may not bypass the bound (including edges from switch cases).
``table_base`` must be a nearest-definition LUI, optionally followed by a
nearest-definition ADDIU. The lower instruction may rename the
register (``lui v0; addiu s0,v0,lo``), as used by Ape Escape. Any other
control flow between that constant and the suffix, an inbound edge that
skips the definition, a register mismatch/clobber, or one non-local table
entry rejects the entire candidate.
"""
lo = load_addr
hi = load_addr + size
if not (entry <= jr_pc < hard_cap and 0 < jr_rs < 31):
return set()
jr_word = _word_at(data, load_addr, jr_pc)
if jr_word != ((jr_rs << 21) | 0x08):
return set()
# Permit only the compiler's zero-or-one scheduling NOP between LW/JR.
lw_pc = jr_pc - 4
for _ in range(1):
if _word_at(data, load_addr, lw_pc) != 0:
break
lw_pc -= 4
lw_word = _word_at(data, load_addr, lw_pc)
if lw_word is None or ((lw_word >> 26) & 0x3F) != 0x23 \
or ((lw_word >> 16) & 0x1F) != jr_rs:
return set()
address_reg = (lw_word >> 21) & 0x1F
if not address_reg:
return set()
lw_offset = lw_word & 0xFFFF
if lw_offset & 0x8000:
lw_offset -= 0x10000
addu_pc = lw_pc - 4
addu_word = _word_at(data, load_addr, addu_pc)
if addu_word is None or ((addu_word >> 26) & 0x3F) != 0 \
or (addu_word & 0x3F) != 0x21 \
or ((addu_word >> 6) & 0x1F) != 0 \
or ((addu_word >> 11) & 0x1F) != address_reg:
return set()
addu_inputs = ((addu_word >> 21) & 0x1F,
(addu_word >> 16) & 0x1F)
sll_pc = addu_pc - 4
sll_word = _word_at(data, load_addr, sll_pc)
if sll_word is None or ((sll_word >> 26) & 0x3F) != 0 \
or (sll_word & 0x3F) != 0 \
or ((sll_word >> 21) & 0x1F) != 0 \
or ((sll_word >> 6) & 0x1F) != 2:
return set()
scaled_reg = (sll_word >> 11) & 0x1F
index_reg = (sll_word >> 16) & 0x1F
if not scaled_reg or not index_reg or scaled_reg not in addu_inputs:
return set()
table_reg = addu_inputs[1] if addu_inputs[0] == scaled_reg else addu_inputs[0]
if not table_reg or table_reg == scaled_reg:
return set()
# The bound must guard this exact index on the sole fallthrough into SLL.
sltiu_pc = sll_pc - 12
branch_pc = sll_pc - 8
scheduled_base = _word_at(data, load_addr, sll_pc - 4) != 0
if scheduled_base:
# R3000 load delay: JR must not consume the immediately preceding LW.
if lw_pc != jr_pc - 8:
return set()
sltiu_pc -= 4
branch_pc -= 4
upper = _word_at(data, load_addr, sll_pc - 8)
lower = _word_at(data, load_addr, sll_pc - 4)
if (upper is None or lower is None or
(upper >> 26) != 0x0F or ((upper >> 21) & 31) != 0 or
(lower >> 26) != 0x09 or ((lower >> 16) & 31) != table_reg or
((upper >> 16) & 31) == 0 or
((upper >> 16) & 31) != ((lower >> 21) & 31) or
index_reg in (table_reg, (upper >> 16) & 31)):
return set()
if sltiu_pc < entry or (sltiu_pc >= entry + 4 and
_is_control_flow(_word_at(data, load_addr, sltiu_pc - 4))):
return set()
sltiu_word = _word_at(data, load_addr, sltiu_pc)
branch_word = _word_at(data, load_addr, branch_pc)
if sltiu_word is None or ((sltiu_word >> 26) & 0x3F) != 0x0B \
or ((sltiu_word >> 21) & 0x1F) != index_reg:
return set()
guard_reg = (sltiu_word >> 16) & 0x1F
table_count = sltiu_word & 0xFFFF
if not guard_reg or guard_reg == index_reg or not (1 <= table_count < 512):
return set()
if branch_word is None or ((branch_word >> 26) & 0x3F) != 0x04:
return set()
branch_rs = (branch_word >> 21) & 0x1F
branch_rt = (branch_word >> 16) & 0x1F
if {branch_rs, branch_rt} != {0, guard_reg}:
return set()
reject_target = _branch_target(branch_pc, branch_word)
if (not (entry <= reject_target < hard_cap) or
sll_pc <= reject_target < jr_pc + 8 or
(scheduled_base and sltiu_pc < reject_target < jr_pc + 8)):
return set()
def nearest_writer(reg: int, before: int):
for pc in range(before - 4, max(entry, before - 0x80) - 1, -4):
word = _word_at(data, load_addr, pc)
if word is None:
return None
if _instruction_writes_gpr(word, reg):
return pc, word
return None
table_writer = nearest_writer(table_reg, addu_pc)
if table_writer is None:
return set()
table_def_pc, table_def_word = table_writer
def_op = (table_def_word >> 26) & 0x3F
def_rs = (table_def_word >> 21) & 0x1F
def_rt = (table_def_word >> 16) & 0x1F
if def_op == 0x0F and def_rs == 0 and def_rt == table_reg:
constant_pc = table_def_pc
table_base = (table_def_word & 0xFFFF) << 16
elif def_op == 0x09 and def_rt == table_reg and def_rs:
upper_writer = nearest_writer(def_rs, table_def_pc)
if upper_writer is None:
return set()
constant_pc, upper_word = upper_writer
if ((upper_word >> 26) & 0x3F) != 0x0F \
or ((upper_word >> 21) & 0x1F) != 0 \
or ((upper_word >> 16) & 0x1F) != def_rs:
return set()
low = table_def_word & 0xFFFF
upper = (upper_word & 0xFFFF) << 16
if low & 0x8000:
low -= 0x10000
table_base = (upper + low) & 0xFFFFFFFF
else:
return set()
if scheduled_base and (constant_pc != branch_pc + 4 or
table_def_pc != sll_pc - 4):
return set()
# Calls could clobber the proven constant. Other branches could enter the
# suffix through an unproved path. The one exact bounds branch is allowed.
for pc in range(constant_pc + 4, sll_pc, 4):
word = _word_at(data, load_addr, pc)
if word is None:
return set()
if _is_control_flow(word) and pc != branch_pc:
return set()
predecessor = _word_at(data, load_addr, constant_pc - 4)
if (constant_pc >= entry + 4 and _is_control_flow(predecessor) and
not scheduled_base):
return set()
table_base = (table_base + lw_offset) & 0xFFFFFFFF
table_end = table_base + table_count * 4
range_lo, range_hi = producer_range or (lo, hi)
if (table_base & 3) or not (range_lo <= table_base < table_end <= range_hi):
return set()
targets = set()
for index in range(table_count):
target = _word_at(data, load_addr, table_base + index * 4)
if (target is None or (target & 3) or
not (entry <= target < hard_cap) or
(scheduled_base and sltiu_pc < target < jr_pc + 8) or
not (range_lo <= target < range_hi) or
not _is_valid_mips_word(_word_at(data, load_addr, target))):
return set()
targets.add(target)
# Establish which later sources are themselves reached only after the
# proven table dispatch. Such case blocks may safely loop to the suffix
# (Ape does this); an arbitrary earlier/later byte sequence may not jump
# into the protected definitions and manufacture domination.
case_reachable = set()
work = deque(targets)
while work and len(case_reachable) < 2048:
pc = work.popleft()
if (pc in case_reachable or not (entry <= pc < hard_cap) or
not (range_lo <= pc < range_hi)):
continue
word = _word_at(data, load_addr, pc)
if not _is_valid_mips_word(word):
continue
case_reachable.add(pc)
kind, target = _classify_cf(pc, word)
delay = pc + 4
if kind == 'normal':
work.append(pc + 4)
elif kind == 'branch':
case_reachable.add(delay)
work.extend((pc + 8, target))
elif kind == 'branch_never':
case_reachable.add(delay)
work.append(pc + 8)
elif kind == 'branch_never_likely':
work.append(pc + 8)
elif kind == 'j':
case_reachable.add(delay)
work.append(target)
elif kind in ('jal', 'jalr'):
case_reachable.add(delay)
work.append(pc + 8)
elif kind in ('jr', 'jr_ra'):
case_reachable.add(delay)
protected_pc = sltiu_pc if scheduled_base else constant_pc
for source in range(entry, hard_cap, 4):
source_word = _word_at(data, load_addr, source)
if source_word is None:
return set()
kind, target = _classify_cf(source, source_word)
if (kind in ('branch', 'j', 'jal') and
protected_pc < target <= jr_pc and
not (protected_pc <= source <= jr_pc) and
(scheduled_base or source not in case_reachable)):
return set()
if proof_out is not None:
proof_out.append({
'host_entry': entry,
'jr_pc': jr_pc,
'table_base': table_base,
'count': table_count,
'targets': sorted(targets),
})
return targets
def _callable_legacy_seed(data: bytes, load_addr: int, addr: int) -> bool:
"""Conservative fallback for old captures that only had a `seeds` list."""
word = _word_at(data, load_addr, addr)
prev = _word_at(data, load_addr, addr - 4)
prev8 = _word_at(data, load_addr, addr - 8)
if not _is_valid_mips_word(word):
return False
if addr == load_addr:
return True
if _is_jr_ra(prev8):
return True
return _is_addiu_sp_neg(word) and not _is_control_flow(prev)
def _dense_local_pointer_table(data: bytes, load_addr: int,
addr: int, count: int = 3) -> bool:
"""Reject instruction-shaped data at a supposed callable target."""
hi = load_addr + len(data)
for index in range(count):
value = _word_at(data, load_addr, addr + index * 4)
if value is None or (value & 3) or not (load_addr <= value < hi):
return False
return True
def _callable_direct_jal_target(data: bytes, load_addr: int, addr: int) -> bool:
"""True only when a direct JAL target has independent boundary evidence."""
return (not _dense_local_pointer_table(data, load_addr, addr) and
_callable_legacy_seed(data, load_addr, addr))
def _instruction_writes_gpr(word: int, reg: int) -> bool:
if not reg:
return False
op = (word >> 26) & 0x3F
rs = (word >> 21) & 0x1F
rt = (word >> 16) & 0x1F
rd = (word >> 11) & 0x1F
fn = word & 0x3F
if op == 0:
if fn in (0x08, 0x0C, 0x0D, 0x11, 0x13,
0x18, 0x19, 0x1A, 0x1B):
return False
return rd == reg
if op == 0x03:
return reg == 31
if op == 0x01 and rt in (0x10, 0x11):
return reg == 31
if op in range(0x08, 0x10) or op in range(0x20, 0x27) \
or op in (0x30, 0x32):
return rt == reg
if op in (0x10, 0x12) and rs in (0, 2):
return rt == reg
return False
def _resolve_constant_transfer(data: bytes, load_addr: int,
entry: int, transfer_pc: int,
target_reg: int):
"""Resolve a same-basic-block lui + addiu/ori constant feeding jr/jalr."""
low = None
low_is_addiu = False
for count in range(1, 17):
pc = transfer_pc - count * 4
if pc < entry:
break
word = _word_at(data, load_addr, pc)
if word is None:
break
op = (word >> 26) & 0x3F
rs = (word >> 21) & 0x1F
rt = (word >> 16) & 0x1F
if low is None and op in (0x09, 0x0D) \
and rs == target_reg and rt == target_reg:
low = word & 0xFFFF
low_is_addiu = op == 0x09
continue
if op == 0x0F and rt == target_reg:
# A LUI in a control-transfer delay slot is not a local reaching
# definition for the following suffix. A callee may clobber the
# register before returning at LUI+4, while a branch/jump can leave
# the linear path entirely. Fail closed without path/call proof.
predecessor = _word_at(data, load_addr, pc - 4)
if pc >= entry + 4 and _is_control_flow(predecessor):
return None
# Reject a syntactic constant pair when a direct control-flow edge
# can enter after this LUI. Example: `j low; lui; low: addiu;
# jalr` never executes the LUI, despite looking like a pair to a
# raw backward scan.
crossed_inbound_boundary = False
for source in range(entry, transfer_pc, 4):
source_word = _word_at(data, load_addr, source)
if source_word is None:
continue
source_kind, source_target = _classify_cf(source, source_word)
if (source_kind in ('branch', 'j', 'jal') and
pc < source_target <= transfer_pc):
crossed_inbound_boundary = True
break
if crossed_inbound_boundary:
return None
upper = (word & 0xFFFF) << 16
if low is None:
return upper
if low_is_addiu and low & 0x8000:
low -= 0x10000
return (upper + low) & 0xFFFFFFFF if low_is_addiu else upper | low
if _instruction_writes_gpr(word, target_reg) or _is_control_flow(word):
break
return None
def plausible_callable_target(data: bytes, load_addr: int, size: int,
target: int, producer_hi: int) -> bool:
"""CFG proof for a cross-producer call target without a classic prologue.
Direct calls are sufficient boundary evidence inside one linked producer.
Across a synthetic adjacent-producer envelope, additionally require a
bounded, valid CFG with a reachable return and no sequential fallthrough.
This retains real frameless exports while rejecting pointer tables/NOP
runways that only happen to decode as instructions.
"""
first = _word_at(data, load_addr, target)
if first in (None, 0) or not _is_valid_mips_word(first):
return False
if _dense_local_pointer_table(data, load_addr, target):
return False
cap = min(load_addr + size, producer_hi, target + 0x4000)
work = deque([target])
visited = set()
saw_return = False
bad = False
def inside(addr: int) -> bool:
return target <= addr < cap and (addr & 3) == 0
def add_delay(pc: int) -> None:
nonlocal bad
delay = pc + 4
if not inside(delay):
bad = True
return
word = _word_at(data, load_addr, delay)
if not _is_valid_mips_word(word):
bad = True
else:
visited.add(delay)
while work and not bad and len(visited) < 2048:
pc = work.popleft()
if pc in visited:
continue
if not inside(pc):
bad = True # sequential fallthrough past the bounded probe
break
word = _word_at(data, load_addr, pc)
if not _is_valid_mips_word(word):
bad = True
break
visited.add(pc)
kind, branch_target = _classify_cf(pc, word)
if kind == 'normal':
work.append(pc + 4)
elif kind == 'branch':
add_delay(pc)
work.append(pc + 8) # conditional fallthrough must remain bounded
if inside(branch_target):
work.append(branch_target)
# An out-of-probe branch target is a shared external code edge.
elif kind == 'branch_never':
add_delay(pc)
work.append(pc + 8)
elif kind == 'branch_never_likely':
work.append(pc + 8)
elif kind in ('jal', 'jalr'):
add_delay(pc)
work.append(pc + 8)
elif kind == 'j':
add_delay(pc)
if inside(branch_target):
work.append(branch_target)
# Otherwise this is a bounded tail edge.
elif kind == 'jr_ra':
add_delay(pc)
saw_return = True
elif kind == 'jr':
add_delay(pc) # bounded indirect/tail exit
return saw_return and not bad and not work and len(visited) < 2048
def _frameless_dispatch_root_proven(data: bytes, load_addr: int, size: int,
addr: int, producer_hi: int) -> bool:
"""CFG proof for a dispatch entry admitted only by the word before it.
`_callable_legacy_seed` accepts an address on three different grades of
evidence: it is the image entry, it opens a stack frame, or the word two
slots back is `jr $ra`. The third is the weak one, and it is weak exactly
where it fires most: the first word *after* a function's return is also
the first word of whatever the linker laid down next, which in these
images is routinely a pointer table, a packed record array or zero fill.
A dispatch PC there is real -- a sibling occupant of the shared band has
code at that address -- but for *this* image it is data, and promoting it
to a walk root sends the linear walk through the table to the image end.
So require the same bounded CFG probe the discovery roots already use
when the classic prologue is absent. Failing it does not discard the
entry: the caller demotes it to DISPATCH_INTERIOR, which keeps the
dispatch evidence and the isolated-fragment demand and only declines to
start a walk there.
"""
word = _word_at(data, load_addr, addr)
prev = _word_at(data, load_addr, addr - 4)
if addr == load_addr:
return True
if _is_addiu_sp_neg(word) and not _is_control_flow(prev):
return True
return plausible_callable_target(data, load_addr, size, addr, producer_hi)
def _walk_overlay_function(data: bytes, load_addr: int, size: int,
entry: int, hard_cap: int,
producer_ranges=(),
allow_cross_producer_calls=True) -> dict:
lo = load_addr
hi = load_addr + size
work = deque([entry])
visited = set()
direct_jals = set()
static_indirect_targets = set()
call_continuations = set()
branch_targets = set()
jump_table_targets = set()
jump_table_proofs = []
forward_branch_targets = set()
rejected_cross_producer_calls = set()
accepted_cross_producer_calls = set()
def producer_for(addr: int):
for range_lo, range_hi in producer_ranges:
if range_lo <= addr < range_hi:
return range_lo, range_hi
return None
entry_producer = producer_for(entry)
def in_function(addr: int) -> bool:
return (lo <= addr < hi and entry <= addr < hard_cap and
(addr & 3) == 0 and
(not producer_ranges or
(entry_producer is not None and
producer_for(addr) == entry_producer)))
def callable_transfer_target(source: int, target: int) -> bool:
if _dense_local_pointer_table(data, load_addr, target):
rejected_cross_producer_calls.add(target)
return False
if not producer_ranges:
return True
source_range = producer_for(source)
target_range = producer_for(target)
if source_range is None or target_range is None:
rejected_cross_producer_calls.add(target)
return False
if source_range is not None and source_range == target_range:
return True
if not allow_cross_producer_calls:
rejected_cross_producer_calls.add(target)
return False
# Adjacent-producer composites are a byte envelope, not proof that a
# call encoded by producer A is linked to arbitrary bytes at the same
# address in every producer-B variant. Cross-boundary calls therefore
# need independent local boundary evidence. This keeps real exports
# while rejecting pointer/data words that happen to be valid MIPS.
if _callable_direct_jal_target(data, load_addr, target):
accepted_cross_producer_calls.add(target)
return True
if (target_range is not None and
plausible_callable_target(
data, load_addr, size, target, target_range[1])):
accepted_cross_producer_calls.add(target)
return True
rejected_cross_producer_calls.add(target)
return False
while work:
pc = work.popleft()
if pc in visited or not in_function(pc):
continue
word = _word_at(data, load_addr, pc)
if word is None:
continue
visited.add(pc)
kind, target = _classify_cf(pc, word)
delay = pc + 4
if kind == 'normal':
if in_function(pc + 4):
work.append(pc + 4)
elif kind == 'branch':
if in_function(delay):
visited.add(delay)
if in_function(pc + 8):
work.append(pc + 8)
branch_targets.add(pc + 8)
if in_function(target):
work.append(target)
branch_targets.add(target)
elif lo <= target < hi and target >= hard_cap and (target & 3) == 0:
# A reachable direct branch may jump across a sibling entry.
# Exact-entry partitioning hard-caps the current walk there,
# but the target remains mechanically proven code reachability.
forward_branch_targets.add(target)
elif kind == 'branch_never':
if in_function(delay):
visited.add(delay)
if in_function(pc + 8):
work.append(pc + 8)
branch_targets.add(pc + 8)
elif kind == 'branch_never_likely':
if in_function(pc + 8):
work.append(pc + 8)
branch_targets.add(pc + 8)
elif kind == 'j':
if in_function(delay):
visited.add(delay)
if in_function(target):
work.append(target)
branch_targets.add(target)
elif lo <= target < hi and target >= hard_cap and (target & 3) == 0:
forward_branch_targets.add(target)
elif kind == 'jal':
if in_function(delay):
visited.add(delay)
if in_function(pc + 8):
work.append(pc + 8)
branch_targets.add(pc + 8)
call_continuations.add(pc + 8)
if (lo <= target < hi and (target & 3) == 0 and
_is_valid_mips_word(_word_at(data, load_addr, target)) and
callable_transfer_target(pc, target)):
direct_jals.add(target)
elif kind == 'jalr':
if in_function(delay):
visited.add(delay)
if in_function(pc + 8):
work.append(pc + 8)
branch_targets.add(pc + 8)
call_continuations.add(pc + 8)
target_reg = (word >> 21) & 0x1F
target = _resolve_constant_transfer(data, load_addr, entry, pc,
target_reg)
if (target is not None and lo <= target < hi and (target & 3) == 0
and callable_transfer_target(pc, target)):
static_indirect_targets.add(target)
elif kind == 'jr':
if in_function(delay):
visited.add(delay)
jr_rs = (word >> 21) & 0x1F
target = _resolve_constant_transfer(data, load_addr, entry, pc, jr_rs)
if target is not None and lo <= target < hi and (target & 3) == 0:
if in_function(target):
branch_targets.add(target)
work.append(target)
elif callable_transfer_target(pc, target):
static_indirect_targets.add(target)
else:
proofs = []
for jt in _find_jump_table_targets(data, load_addr, size,
entry, hard_cap, pc, jr_rs,
entry_producer, proofs):
jump_table_targets.add(jt)
branch_targets.add(jt)
if in_function(jt):
work.append(jt)
jump_table_proofs.extend(proofs)
elif kind == 'jr_ra':
if in_function(delay):
visited.add(delay)
return {
'visited': visited,
'direct_jals': direct_jals,
'static_indirect_targets': static_indirect_targets,
'call_continuations': call_continuations,
'branch_targets': branch_targets,
'jump_table_targets': jump_table_targets,
'jump_table_proofs': jump_table_proofs,
'forward_branch_targets': forward_branch_targets,
'rejected_cross_producer_calls': rejected_cross_producer_calls,
'accepted_cross_producer_calls': accepted_cross_producer_calls,
}
def _masked_region_crc(data: bytes, load_addr: int, ignore) -> int:
"""CRC32 of a captured region with declared-volatile byte ranges zeroed."""
if not ignore:
return binascii.crc32(data) & 0xFFFFFFFF
buf = bytearray(data)
hi = load_addr + len(data)
for raw in ignore:
if isinstance(raw, dict):
start, end = _parse_addr(raw['start']), _parse_addr(raw['end'])
else:
raise RuntimeError(
'bytes_crc_ignore entries must be tables with start/end')
if not (load_addr <= start < end <= hi):
raise RuntimeError(
f'bytes_crc_ignore range 0x{start:08X}..0x{end:08X} outside '
f'capture 0x{load_addr:08X}..0x{hi:08X}')
buf[start - load_addr:end - load_addr] = b'\0' * (end - start)
return binascii.crc32(bytes(buf)) & 0xFFFFFFFF
def _collect_toml_overlay_entries(toml_doc: dict, load_addr: int, crc32: int,
data: bytes = b'') -> set[int]:
entries = set()
for ov in toml_doc.get('overlays', []) or []:
if not isinstance(ov, dict):
continue
ov_load = ov.get('load_addr') or ov.get('load_address')
if ov_load is not None and _parse_addr(ov_load) != load_addr:
continue
ov_crc = ov.get('bytes_crc') or ov.get('crc32') or ov.get('crc')
if ov_crc is not None:
ignore = ov.get('bytes_crc_ignore')
live = _masked_region_crc(data, load_addr, ignore) if ignore else crc32
if _parse_addr(ov_crc) != live:
print(f' declared entries DETACHED for 0x{load_addr:08X}: '
f'bytes_crc=0x{_parse_addr(ov_crc):08X} but region hashes '
f'0x{live:08X}'
f'{" (masked)" if ignore else ""} -- update game.toml',
file=sys.stderr)
continue
for key in ('entry', 'entries', 'function_entry_pcs', 'function_entries'):
if key not in ov:
continue
val = ov[key]
if isinstance(val, list):
entries.update(_parse_addr_list(val))
else:
entries.add(_parse_addr(val))
return entries
def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
crc32: int, toml_doc: dict) -> tuple[list[str], dict]:
lo = load_addr
hi = load_addr + size
region = lambda a: lo <= a < hi and (a & 3) == 0
producer_ranges = []
for raw_range in cap.get('producer_ranges', []) or []:
if not isinstance(raw_range, dict):
raise RuntimeError('producer_ranges entries must be objects')
try:
range_lo = _parse_addr(raw_range['start'])
range_hi = _parse_addr(raw_range['end'])
except (KeyError, TypeError, ValueError) as exc:
raise RuntimeError(f'invalid producer_ranges entry: {raw_range}') from exc
if not (lo <= range_lo < range_hi <= hi):
raise RuntimeError(
f'producer range 0x{range_lo:08X}..0x{range_hi:08X} '
f'outside capture 0x{lo:08X}..0x{hi:08X}')
producer_ranges.append((range_lo, range_hi))
producer_ranges.sort()
if any(producer_ranges[i-1][1] > producer_ranges[i][0]
for i in range(1, len(producer_ranges))):
raise RuntimeError('producer_ranges overlap')
def capture_producer_for(addr: int):
for producer_lo, producer_hi in producer_ranges:
if producer_lo <= addr < producer_hi:
return producer_lo, producer_hi
return None
static_alias_ranges = set()
allow_cross_producer_calls = not bool(cap.get('strict_producer_ranges'))
for raw_alias in cap.get('static_alias_ranges', []) or []:
if not isinstance(raw_alias, dict):
raise RuntimeError('static_alias_ranges entries must be objects')
try:
entry = _parse_addr(raw_alias['entry'])
range_lo = _parse_addr(raw_alias['start'])
range_hi = _parse_addr(raw_alias['end'])
except (KeyError, TypeError, ValueError) as exc:
raise RuntimeError(
f'invalid static_alias_ranges entry: {raw_alias}') from exc
if not (lo <= range_lo <= entry < range_hi <= hi and
(entry & 3) == 0 and (range_lo & 3) == 0 and
(range_hi & 3) == 0):
raise RuntimeError(
f'static alias 0x{entry:08X} -> '
f'0x{range_lo:08X}..0x{range_hi:08X} outside/misaligned for '
f'capture 0x{lo:08X}..0x{hi:08X}')
if producer_ranges:
owners = [index for index, (producer_lo, producer_hi) in
enumerate(producer_ranges)
if (producer_lo <= range_lo < producer_hi and
producer_lo <= entry < producer_hi and
producer_lo <= range_hi - 1 < producer_hi)]
if not owners:
raise RuntimeError(
f'static alias 0x{entry:08X} crosses producer boundary')
static_alias_ranges.add((entry, range_lo, range_hi))
schema_keys = {'executed_pcs', 'observed_pcs', 'dispatch_entry_pcs',
'static_dispatch_entry_pcs', 'function_entry_pcs'}
has_split_schema = any(k in cap for k in schema_keys)
legacy_seeds = _parse_addr_list(cap.get('seeds', []))
executed_pcs = _parse_addr_list(cap.get('executed_pcs',
cap.get('observed_pcs',
legacy_seeds if not has_split_schema else [])))
dispatch_entry_pcs = _parse_addr_list(cap.get('dispatch_entry_pcs', []))
# New extractors preserve which dispatch entries came from a static
# export/header/jump table. Treat malformed superset values as ordinary
# unknown metadata, never as cross-variant authority.
declared_static_dispatch = _parse_addr_list(
cap.get('static_dispatch_entry_pcs', []))
if not declared_static_dispatch <= dispatch_entry_pcs:
extra = sorted(declared_static_dispatch - dispatch_entry_pcs)
raise RuntimeError(
'static_dispatch_entry_pcs must be a subset of '
'dispatch_entry_pcs: ' +
', '.join(f'0x{addr:08X}' for addr in extra))
static_dispatch_entries = declared_static_dispatch
captured_function_entries = _parse_addr_list(cap.get('function_entry_pcs', []))
static_discovery_entries = _parse_addr_list(
cap.get('static_discovery_entry_pcs', []))
toml_entries = _collect_toml_overlay_entries(toml_doc, load_addr, crc32, data)
legacy_callable_seeds = {a for a in legacy_seeds
if region(a) and _callable_legacy_seed(data, load_addr, a)}
included: dict[int, str] = {}
excluded: dict[int, str] = {}
# Shared CFG ownership may later reject a hostless interior. Preserve its
# validated dispatch evidence separately so executed entries can still use
# the isolated, audited fragment path without truncating a shared host.
dispatch_fragment_demands = set()
fragment_hi = hi - capture_guard_bytes(cap, size)
game_text = _game_text_range(toml_doc)
# A dispatch into dirty RAM can land on a jump-table case label. That
# proves coverage, not a callable function boundary, so discover those
# labels before promoting dispatch entries to seeds.
pre_roots = (set(legacy_callable_seeds) | set(captured_function_entries) |
set(static_discovery_entries) | set(toml_entries))
pre_roots.update(a for a in dispatch_entry_pcs
if region(a) and _callable_legacy_seed(data, load_addr, a))
jump_table_targets = set()
pre_roots_sorted = sorted(a for a in pre_roots if region(a))
for i, entry in enumerate(pre_roots_sorted):
hard_cap = pre_roots_sorted[i + 1] if i + 1 < len(pre_roots_sorted) else hi
walk = _walk_overlay_function(
data, load_addr, size, entry, hard_cap, producer_ranges,
allow_cross_producer_calls)
jump_table_targets.update(walk['jump_table_targets'])
def impossible_entry_start(addr: int) -> bool:
word = _word_at(data, load_addr, addr)
if word is None:
return True
if not _is_valid_mips_word(word):
return True
kind, target = _classify_cf(addr, word)
if kind == 'j':
return _classify_target(target, load_addr, size, game_text) == 'UNKNOWN_BAD'
if kind == 'jal' and region(target):
return not _callable_direct_jal_target(data, load_addr, target)
return False
def include(addr: int, reason: str):
if not region(addr):
excluded[addr] = 'UNKNOWN'
return
# Composite padding has bytes but no producer identity. A root there
# would otherwise get entry_producer=None and could walk through every
# disconnected zero-fill gap as though those gaps shared ownership.
if producer_ranges and capture_producer_for(addr) is None:
excluded[addr] = 'UNKNOWN'
return
# Boundary gate: a dispatched-to PC is proof of *code reachability*, not
# of a *callable function boundary*. A jr-driven jump-table case label
# or a mid-function pointer target can be dispatched-to yet have no
# prologue and no preceding jr $ra. Promoting it to a WALK ROOT would
# truncate its host function (mid-function-seed softlock class) or run
# off into adjacent data. Such PCs are dispatch-proven code, though —
# classify them DISPATCH_INTERIOR: the recompiler emits them as
# overlapping-alias entries into their host (never walk roots).
# Invalid words stay excluded. Call-edge-proven reasons
# (DIRECT_JAL_TARGET, FUNCTION_POINTER_TARGET, TOML_DECLARED_ENTRY) are
# exempt — they carry their own proof.
# A dispatch entry that has no prologue and is callable only because a
# `jr $ra` precedes it is the head of whatever follows the last
# function, data included, so it needs the bounded CFG probe before it
# may be a walk root (_frameless_dispatch_root_proven).
if reason in ('DISPATCH_ENTRY', 'STATIC_DISPATCH_ENTRY'):
if impossible_entry_start(addr):
excluded[addr] = 'UNKNOWN'
return
if addr + 4 <= fragment_hi:
dispatch_fragment_demands.add(addr)
producer = capture_producer_for(addr)
if (addr in jump_table_targets or
not _callable_legacy_seed(data, load_addr, addr) or
not _frameless_dispatch_root_proven(
data, load_addr, size, addr,
producer[1] if producer else hi)):
included.setdefault(addr, 'DISPATCH_INTERIOR')
return
elif (reason not in ('DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET',
'STATIC_BRANCH_ROOT',
'STATIC_DISCOVERY_ROOT',
'TOML_DECLARED_ENTRY') and
addr in jump_table_targets and
not _callable_legacy_seed(data, load_addr, addr)):
excluded[addr] = 'BRANCH_TARGET_ONLY'
return
if reason in FATAL_SEED_REASONS:
raise RuntimeError(f'BUG: refusing to include 0x{addr:08X} as {reason}')
old = included.get(addr)
if reason in ('DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET',
'STATIC_BRANCH_ROOT', 'STATIC_DISCOVERY_ROOT',
'STATIC_DISPATCH_ENTRY'):
# Static call-edge evidence outranks heuristic capture metadata.
included[addr] = reason
elif old is None or old in ('FUNCTION_POINTER_TARGET', 'DISPATCH_INTERIOR'):
included[addr] = reason
for addr in dispatch_entry_pcs:
include(addr, ('STATIC_DISPATCH_ENTRY'
if addr in static_dispatch_entries
else 'DISPATCH_ENTRY'))
for addr in captured_function_entries:
include(addr, 'FUNCTION_POINTER_TARGET')
for addr in static_discovery_entries:
if not region(addr):
excluded[addr] = 'UNKNOWN'
elif (_callable_legacy_seed(data, load_addr, addr) or
plausible_callable_target(data, load_addr, size, addr, hi)):
include(addr, 'STATIC_DISCOVERY_ROOT')
else:
# Preserve the weaker classification. Overlay mode will perform
# its ordinary boundary recheck and fail closed if needed.
include(addr, 'FUNCTION_POINTER_TARGET')
for addr in toml_entries:
include(addr, 'TOML_DECLARED_ENTRY')
legacy_seed_mode = bool(legacy_seeds) and not cap.get('schema')
if legacy_seed_mode:
# Old capture files used `seeds` for interpreter-observed PCs. Include
# only PCs whose surrounding bytes look callable; classify the rest
# after we know branch targets from accepted roots.
for addr in legacy_callable_seeds:
include(addr, 'FUNCTION_POINTER_TARGET')
# Current-capture alias recipes contribute only an interior ENTRY candidate;
# their historical host range is not trusted. Final exact reachability below
# must discover a current host or the entry is dropped to the interpreter.
for addr, _range_lo, _range_hi in static_alias_ranges:
if region(addr):
included[addr] = 'DISPATCH_INTERIOR'
# Walk roots: callable entries only. DISPATCH_INTERIOR addresses are NOT
# roots — as roots they would hard-cap (truncate) the sibling walk that
# owns them.
known = {a for a, r in included.items() if r != 'DISPATCH_INTERIOR'}
initial_known = set(known)
derived_reasons = {}
promoted_roots = set()
suppressed_derived = set()
suppression_passes = 0
all_branch_targets = set()
kernel_window = (load_addr & 0x1FFFFFFF) < 0x10000
while True:
# Recompute call/branch evidence from the CURRENT partition. Evidence is
# not monotonic: a new hard cap can make an old source PC unreachable.
# Dropping stale candidates here prevents an old speculative walk from
# resurrecting a target in the final shard.
discovery_round = 0
seen_states = {}
state_history = []
while True:
discovery_round += 1
state = (tuple(sorted(known)), tuple(sorted(derived_reasons)))
cycle_start = seen_states.get(state)
if discovery_round > 64 or cycle_start is not None:
unstable = set()
if cycle_start is not None:
cycle_states = state_history[cycle_start:]
known_sets = [set(item[0]) for item in cycle_states]
derived_sets = [set(item[1]) for item in cycle_states]
if known_sets:
unstable |= set.union(*known_sets) - set.intersection(*known_sets)
if derived_sets:
unstable |= (set.union(*derived_sets) -
set.intersection(*derived_sets))
unstable_text = ','.join(f'{addr:08X}'
for addr in sorted(unstable)[:12])
unstable -= initial_known | promoted_roots
if unstable and suppression_passes < 64:
suppression_passes += 1
suppressed_derived.update(unstable)
print(' WARNING: ownership discovery cycle; suppressing '
f'{len(unstable)} unstable derived roots'
+ (f' ({unstable_text})' if unstable_text else ''))
known = initial_known | promoted_roots
derived_reasons = {}
discovery_round = 0
seen_states = {}
state_history = []
continue
print(' WARNING: ownership discovery did not converge; '
'falling back to explicit/dispatch roots'
+ (f' (cycle={len(state_history) - cycle_start}, '
f'unstable={unstable_text})'
if cycle_start is not None else ' (round limit)'))
known = initial_known | promoted_roots
derived_reasons = {}
break
seen_states[state] = len(state_history)
state_history.append(state)
next_reasons = {}
round_branch_targets = set()
sorted_known = sorted(known)
for index, entry in enumerate(sorted_known):
hard_cap = (sorted_known[index + 1]
if index + 1 < len(sorted_known) else hi)
walk = _walk_overlay_function(
data, load_addr, size, entry, hard_cap, producer_ranges,
allow_cross_producer_calls)
round_branch_targets.update(walk['branch_targets'])
round_branch_targets.update(walk['jump_table_targets'])
for target in walk['direct_jals']:
if (target not in initial_known and
target not in suppressed_derived):
next_reasons[target] = 'DIRECT_JAL_TARGET'
for target in walk['static_indirect_targets']:
if (target not in initial_known and
target not in suppressed_derived):
next_reasons.setdefault(target, 'STATIC_INDIRECT_TARGET')
for target in walk['forward_branch_targets']:
source_reason = (included.get(entry) or
derived_reasons.get(entry) or
('DISPATCH_ROOT'
if entry in promoted_roots else None))
strong_source = source_reason in (
'DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET',
'STATIC_BRANCH_ROOT', 'STATIC_DISCOVERY_ROOT',
'STATIC_DISPATCH_ENTRY', 'TOML_DECLARED_ENTRY',
'DISPATCH_ROOT')
if not strong_source and not _callable_legacy_seed(
data, load_addr, entry):
continue
source_range = next(
((rlo, rhi) for rlo, rhi in producer_ranges
if rlo <= entry < rhi), None)
target_range = next(
((rlo, rhi) for rlo, rhi in producer_ranges
if rlo <= target < rhi), None)
if producer_ranges and source_range != target_range:
continue
producer_hi = target_range[1] if target_range else hi
if plausible_callable_target(
data, load_addr, size, target, producer_hi):
if (target not in initial_known and
target not in suppressed_derived):
next_reasons.setdefault(target, 'STATIC_BRANCH_ROOT')
normalized = initial_known | promoted_roots | set(next_reasons)
roots = sorted(normalized)
for target in sorted(next_reasons):
if target not in normalized:
continue
index = bisect_left(roots, target)
if index == len(roots) or roots[index] != target or index == 0:
continue
owner = roots[index - 1]
hard_cap = roots[index + 1] if index + 1 < len(roots) else hi
owner_walk = _walk_overlay_function(
data, load_addr, size, owner, hard_cap, producer_ranges,
allow_cross_producer_calls)
if target in owner_walk['visited']:
normalized.remove(target)
roots.pop(index)
if normalized == known and next_reasons == derived_reasons:
all_branch_targets = round_branch_targets
break
known = normalized
derived_reasons = next_reasons
if not kernel_window:
break
# Orphan promotion (see DISPATCH_ROOT in INCLUDE_REASONS): a kernel
# dispatch interior that no rooted walk covers is promoted to a
# trusted walk root. Re-enter the walk loop — the new root's walk
# may cover other interiors or discover direct-jal callees.
covered = set()
sorted_known = sorted(known)
for i, entry in enumerate(sorted_known):
hard_cap = sorted_known[i + 1] if i + 1 < len(sorted_known) else hi
walk = _walk_overlay_function(
data, load_addr, size, entry, hard_cap, producer_ranges,
allow_cross_producer_calls)
covered |= walk['visited']
promoted = sorted(a for a, r in included.items()
if r == 'DISPATCH_INTERIOR' and a not in covered
and _is_valid_mips_word(_word_at(data, load_addr, a)))
if not promoted:
break
# Promote ONE orphan per iteration (lowest address first): its walk
# usually covers the remaining orphans, which then stay interiors and
# alias into the new root as host — rather than minting sibling roots
# that split one real function and hard-cap each other.
a = promoted[0]
included[a] = 'DISPATCH_ROOT'
promoted_roots.add(a)
known.add(a)
# Re-walk with the final function set so the branch-target exclusion count
# matches the actual compilation boundaries.
all_branch_targets.clear()
rejected_cross_producer_calls = set()
accepted_cross_producer_calls = set()
final_covered = set()
sorted_known = sorted(known)
for i, entry in enumerate(sorted_known):
hard_cap = sorted_known[i + 1] if i + 1 < len(sorted_known) else hi
walk = _walk_overlay_function(
data, load_addr, size, entry, hard_cap, producer_ranges,
allow_cross_producer_calls)
all_branch_targets.update(walk['branch_targets'])
all_branch_targets.update(walk['jump_table_targets'])
all_branch_targets.update(walk['forward_branch_targets'])
final_covered.update(walk['visited'])
rejected_cross_producer_calls.update(
walk['rejected_cross_producer_calls'])
accepted_cross_producer_calls.update(
walk['accepted_cross_producer_calls'])
# A DISPATCH_INTERIOR is emitted only when the final partition has a real
# CFG host for that exact PC. Reconsider every derived target after all
# later roots have settled; stale ownership falls back to a root (when it
# remains call-proven) or to the interpreter, never a hostless alias.
for addr, reason in sorted(derived_reasons.items()):
if addr in known:
include(addr, reason)
elif addr in final_covered:
included[addr] = 'DISPATCH_INTERIOR'
else:
included.pop(addr, None)
excluded[addr] = 'OBSERVED_PC_ONLY'
for addr, reason in list(included.items()):
if reason == 'DISPATCH_INTERIOR' and addr not in final_covered:
del included[addr]
excluded[addr] = ('OBSERVED_PC_ONLY'
if addr in executed_pcs else 'UNKNOWN')
candidates = {a for a in (executed_pcs | dispatch_entry_pcs |
captured_function_entries | static_discovery_entries |
legacy_seeds | toml_entries)
if region(a)}
for addr in sorted(candidates - set(included)):
if addr in all_branch_targets or addr in jump_table_targets:
excluded[addr] = 'BRANCH_TARGET_ONLY'
elif addr in executed_pcs or addr in legacy_seeds:
excluded[addr] = 'OBSERVED_PC_ONLY'
else:
excluded[addr] = 'UNKNOWN'
bad_included = [(a, r) for a, r in included.items() if r in FATAL_SEED_REASONS]
if bad_included:
details = ', '.join(f'0x{a:08X}:{r}' for a, r in bad_included)
raise RuntimeError(f'fatal seed classification: {details}')
counts = Counter(included.values())
excluded_counts = Counter(excluded.values())
audit = {
'load_addr': load_addr,
'crc32': crc32,
'lo': lo,
'hi': hi,
'executed_pcs': executed_pcs,
'dispatch_entry_pcs': dispatch_entry_pcs,
'static_dispatch_entry_pcs': static_dispatch_entries,
'dispatch_fragment_demands': dispatch_fragment_demands,
'function_entry_pcs': set(included),
'included_reasons': included,
'excluded_reasons': excluded,
'branch_targets_excluded_count': len(all_branch_targets - set(included)),
'counts': counts,
'excluded_counts': excluded_counts,
'producer_ranges': producer_ranges,
'static_alias_ranges': static_alias_ranges,
# Every strong walk root is an exact-entry demand. These roots come from
# decoder/control-flow, callable-boundary, explicit TOML, or dispatch
# proof rather than an arbitrary observed PC. If a shared-region build
# (or a concurrent poorer winner) lacks its F entry, the post-pass can
# safely retry it in the isolated-fragment failure domain.
'static_exact_fragment_demands': {
addr for addr, reason in included.items()
if reason in EXACT_FRAGMENT_REASONS
},
# Only static control-flow/export evidence may nominate an address in a
# sibling byte recipe. The recipient still has to prove an existing
# CRC-valid host and the recompiler must prove a real block leader.
'cross_variant_hosted_demands': {
addr for addr, reason in included.items()
if reason in CROSS_VARIANT_DONOR_REASONS
},
# Alias recipes describe entry labels inside a normal-mode host. They
# need an isolated body only when the stricter shared analysis leaves
# the label outside every current code interval; aliases already inside
# a guarded body remain candidates for the future exact-resume pass.
'static_interval_fragment_demands': {
entry for entry, _lo, _hi in static_alias_ranges
},
'rejected_cross_producer_calls': rejected_cross_producer_calls,
'accepted_cross_producer_calls': accepted_cross_producer_calls,
'static_jump_table_proofs': cap.get('static_jump_table_proofs', []),
'static_call_continuation_pcs': _parse_addr_list(
cap.get('static_call_continuation_pcs', [])),
}
# Interior entries carry the 'interior' marker so the recompiler emits
# them as overlapping aliases, never as walk roots. Promoted kernel
# orphans carry 'dispatch_root' so the recompiler roots them without
# boundary re-verification.
seeds = []
for range_lo, range_hi in producer_ranges:
seeds.append(f'producer_range 0x{range_lo:08X} 0x{range_hi:08X}')
for addr in sorted(accepted_cross_producer_calls):
seeds.append(f'cross_call_allow 0x{addr:08X}')
seeds.extend(seed_line_for_reason(addr, included[addr])
for addr in sorted(included))
return seeds, audit
def print_seed_audit(audit: dict) -> None:
print(f'Overlay {audit["load_addr"]:08X}_{audit["crc32"]:08X}')
print(f'Region: {audit["lo"]:08X}..{audit["hi"] - 1:08X}')
print(f'executed_pcs: {len(audit["executed_pcs"])}')
print(f'dispatch_entry_pcs: {len(audit["dispatch_entry_pcs"])}')
print(f'function_entry_pcs: {len(audit["function_entry_pcs"])}')
print(f'direct_jal_targets_included: {audit["counts"].get("DIRECT_JAL_TARGET", 0)}')
print(f'static_indirect_targets_included: {audit["counts"].get("STATIC_INDIRECT_TARGET", 0)}')
print(f'static_branch_roots_included: '
f'{audit["counts"].get("STATIC_BRANCH_ROOT", 0)}')
print(f'static_discovery_roots_included: '
f'{audit["counts"].get("STATIC_DISCOVERY_ROOT", 0)}')
print(f'function_pointer_targets_included: {audit["counts"].get("FUNCTION_POINTER_TARGET", 0)}')
print(f'toml_entries_included: {audit["counts"].get("TOML_DECLARED_ENTRY", 0)}')
print(f'dispatch_interior_included: {audit["counts"].get("DISPATCH_INTERIOR", 0)}')
print(f'dispatch_roots_promoted: {audit["counts"].get("DISPATCH_ROOT", 0)}')
unhosted_dispatch = (audit.get('dispatch_fragment_demands', set()) &
audit['executed_pcs']) - set(audit['included_reasons'])
print(f'unhosted_executed_dispatch_fragment_demands: {len(unhosted_dispatch)}')
print(f'cross_producer_calls_rejected: '
f'{len(audit["rejected_cross_producer_calls"])}')
print(f'cross_producer_calls_accepted: '
f'{len(audit["accepted_cross_producer_calls"])}')
print(f'branch_targets_excluded: {audit["branch_targets_excluded_count"]}')
print(f'observed_only_excluded: {audit["excluded_counts"].get("OBSERVED_PC_ONLY", 0)}')
print(f'unknown_excluded: {audit["excluded_counts"].get("UNKNOWN", 0)}')
for addr in sorted(audit['included_reasons']):
print(f' {addr:08X} {audit["included_reasons"][addr]}')
for addr in sorted(audit['excluded_reasons']):
reason = audit['excluded_reasons'][addr]
if reason in ('BRANCH_TARGET_ONLY', 'OBSERVED_PC_ONLY', 'UNKNOWN'):
recovery = ('; isolated fragment demand retained'
if addr in unhosted_dispatch else '')
print(f' {addr:08X} excluded: {reason}{recovery}')
def walk_root_seed_entries(seeds: list[str]) -> set[int]:
"""Exact F entries demanded by recompiler walk-root seed records."""
entries = set()
for seed in seeds:
parts = seed.split()
if not parts:
continue
if (parts[0] not in ('call_root', 'dispatch_root') and
not parts[0].startswith('0x')):
continue
try:
entries.add(_canonical_guest_addr(int(parts[-1], 16)))
except ValueError:
pass
return entries
def _game_text_range(toml_doc: dict) -> tuple[int, int]:
game = toml_doc.get('game', {})
load = game.get('load_address')
size = game.get('text_size')
if load is None or size is None:
return (0, 0)
lo = _parse_addr(load) & 0x1FFFFFFF
hi = lo + _parse_addr(size)
return lo, hi
def _classify_target(addr: int, load_addr: int, size: int,
game_text: tuple[int, int]) -> str:
phys = addr & 0x1FFFFFFF
ov_lo = load_addr & 0x1FFFFFFF
ov_hi = ov_lo + size
if ov_lo <= phys < ov_hi:
return 'INSIDE_OVERLAY'
game_lo, game_hi = game_text
if game_hi > game_lo and game_lo <= phys < game_hi:
return 'MAIN_EXE'
if 0x1FC00000 <= phys < 0x1FC80000 or 0xBFC00000 <= addr < 0xBFC80000:
return 'BIOS'
return 'UNKNOWN_BAD'
def audit_generated_c(src: str, load_addr: int, size: int,
crc32: int, toml_doc: dict) -> dict:
defs = {int(x, 16) for x in re.findall(
r'^void func_([0-9A-Fa-f]{8})\(CPUState\* cpu\)$', src, re.MULTILINE)}
decls = {int(x, 16) for x in re.findall(
r'^void func_([0-9A-Fa-f]{8})\(CPUState\* cpu\);$', src, re.MULTILINE)}
direct_calls = [int(x, 16) for x in re.findall(
r'\bfunc_([0-9A-Fa-f]{8})\(cpu\)', src)]
literal_cba = [int(x, 16) for x in re.findall(
r'\bcall_by_address\(cpu,\s*0x([0-9A-Fa-f]{8})u?\)', src)]
unsupported_todo_addrs = {int(x, 16) for x in re.findall(
r'TODO:[^\n]*?0x([0-9A-Fa-f]{8}):', src)}
game_text = _game_text_range(toml_doc)
ov_lo = load_addr & 0x1FFFFFFF
ov_hi = ov_lo + size
def in_overlay(addr: int) -> bool:
phys = addr & 0x1FFFFFFF
return ov_lo <= phys < ov_hi
unknown_bad = set()
missing_direct = set()
direct_outside = set()
for addr in direct_calls:
if not in_overlay(addr):
direct_outside.add(addr)
unknown_bad.add(addr)
elif addr not in defs:
missing_direct.add(addr)
unknown_bad.add(addr)
cba_classes = Counter()
for addr in literal_cba:
cls = _classify_target(addr, load_addr, size, game_text)
cba_classes[cls] += 1
if cls == 'UNKNOWN_BAD':
unknown_bad.add(addr)
decl_without_def = {a for a in decls if in_overlay(a) and a not in defs}
called_decl_without_def = decl_without_def & set(direct_calls)
if called_decl_without_def:
unknown_bad.update(called_decl_without_def)
report = {
'defs': defs,
'decls': decls,
'direct_calls': direct_calls,
'literal_cba': literal_cba,
'direct_inside': sum(1 for a in direct_calls if in_overlay(a)),
'external_cba': sum(1 for a in literal_cba if not in_overlay(a)),
'bios_calls': cba_classes.get('BIOS', 0),
'syscall_uses': len(re.findall(r'\bpsx_syscall\s*\(', src)),
'unknown_dispatch_uses': len(re.findall(r'\bpsx_unknown_dispatch\s*\(', src)),
'unsupported_todo_addrs': unsupported_todo_addrs,
'unknown_bad': unknown_bad,
'direct_outside': direct_outside,
'missing_direct': missing_direct,
'decl_without_def': decl_without_def,
'called_decl_without_def': called_decl_without_def,
'cba_classes': cba_classes,
}
return report
def print_generated_c_audit(load_addr: int, size: int, crc32: int,
report: dict) -> None:
print(f'Overlay {load_addr:08X}_{crc32:08X}')
print(f'Region: {load_addr:08X}..{load_addr + size - 1:08X}')
print(f'Function definitions: {len(report["defs"])}')
print(f'Forward declarations: {len(report["decls"])}')
print(f'Direct calls inside overlay: {report["direct_inside"]}')
print(f'External calls via call_by_address: {report["external_cba"]}')
print(f'BIOS calls: {report["bios_calls"]}')
print(f'Syscall uses: {report["syscall_uses"]}')
print(f'Unsupported instruction TODOs: {len(report["unsupported_todo_addrs"])}')
print(f'Unknown/bad targets: {len(report["unknown_bad"])}')
for addr in sorted(report['unknown_bad']):
print(f' 0x{addr:08X} UNKNOWN_BAD')
for addr in sorted(report['unsupported_todo_addrs'])[:20]:
print(f' 0x{addr:08X} UNSUPPORTED_INSTRUCTION')
if len(report['unsupported_todo_addrs']) > 20:
print(f' ... {len(report["unsupported_todo_addrs"]) - 20} more unsupported instructions')
# ---------------------------------------------------------------------------
# Post-process generated C for DLL compilation
# ---------------------------------------------------------------------------
# The overlay dispatch shim / link contract is the SINGLE SOURCE OF TRUTH for
# the symbols every compiled shard links against (dispatch forwarders, GTE/WS
# hooks, cycle callbacks). It lives beside the ABI header it must agree with,
# at <runtime_include>/overlay_dispatch_preamble.c.inc, so cmake can fold its
# content into the overlay cache tag (codegen_hash_sources.cmake) -- a change to
# the link contract then auto-invalidates the cache instead of silently
# breaking every shard compile against a moved-on runtime. Loaded once by
# load_dispatch_preamble(); patch_generated_c() reads this global.
DISPATCH_PREAMBLE = None
PREAMBLE_INC_NAME = "overlay_dispatch_preamble.c.inc"
def load_dispatch_preamble(runtime_include: str) -> str:
"""Load the shard dispatch-shim preamble from <runtime_include>/<inc> and
cache it in the module global. Missing is a HARD error: a shard built
without the shim links against nothing, and silently skipping it is exactly
the invisible-failure class this tooling exists to eliminate."""
global DISPATCH_PREAMBLE
if DISPATCH_PREAMBLE is not None:
return DISPATCH_PREAMBLE
path = os.path.join(runtime_include, PREAMBLE_INC_NAME)
try:
with open(path, encoding="utf-8") as f:
DISPATCH_PREAMBLE = f.read()
except OSError as e:
raise SystemExit(
f"FATAL: cannot read overlay dispatch preamble {path}: {e}\n"
f" This file is the shard link contract and MUST ship in "
f"runtime/include. Without it no shard can be compiled.")
return DISPATCH_PREAMBLE
def patch_generated_c(src: str, load_addr: int, size: int) -> str:
"""
Post-process psxrecomp-game's _full.c output for standalone DLL compilation:
1. Prepend the dispatch function-pointer preamble (before any includes).
2. Remove forward declarations for functions outside the overlay range —
they'd be unresolved externals in the DLL.
3. Replace direct calls to out-of-range func_XXXXXXXX(cpu) with
call_by_address(cpu, 0xXXXXXXXXu) so they go through the dispatch ptr.
4. Export all func_XXXXXXXX symbols so the runtime can enumerate them.
"""
ov_lo = load_addr & 0x1FFFFFFF
ov_hi = ov_lo + size
def in_overlay(addr: int) -> bool:
phys = addr & 0x1FFFFFFF
return ov_lo <= phys < ov_hi
# 1. Insert preamble AFTER the last #include line (so CPUState is complete)
last_inc = -1
for m in re.finditer(r'^#include\s+[<"].*[>"]\s*$', src, re.MULTILINE):
last_inc = m.end()
if last_inc == -1:
src = DISPATCH_PREAMBLE + src
else:
src = src[:last_inc] + '\n' + DISPATCH_PREAMBLE + src[last_inc:]
# 2. Remove out-of-range forward declarations
def drop_extern(m):
addr = int(m.group(1), 16)
return '' if not in_overlay(addr) else m.group(0)
src = re.sub(r'^void func_([0-9A-Fa-f]{8})\(CPUState\* cpu\);\n',
drop_extern, src, flags=re.MULTILINE)
# 3. Replace out-of-range direct calls with call_by_address
def fix_call(m):
addr = int(m.group(1), 16)
if in_overlay(addr):
return m.group(0)
return f'call_by_address(cpu, 0x{addr:08X}u)'
src = re.sub(r'\bfunc_([0-9A-Fa-f]{8})\(cpu\)',
fix_call, src)
# 4. Add dllexport to every in-overlay func_XXXXXXXX definition.
# psxrecomp-game emits "void func_XXXXXXXX(CPUState* cpu)" on one line
# with "{" on the NEXT line — match the signature line alone (no ";" = definition).
def add_export(m):
addr = int(m.group(1), 16)
if not in_overlay(addr):
return m.group(0)
return (
'#ifdef _WIN32\n__declspec(dllexport)\n#else\n'
'__attribute__((visibility("default")))\n#endif\n'
+ m.group(0)
)
src = re.sub(r'^void func_([0-9A-Fa-f]{8})\(CPUState\* cpu\)$',
add_export, src, flags=re.MULTILINE)
return src
# ---------------------------------------------------------------------------
# Static (B-2) post-processing
# ---------------------------------------------------------------------------
STATIC_PREAMBLE = """\
/* ---- Static overlay (B-2): psx_runtime.h already provides call_by_address. */
"""
def patch_generated_c_static(src: str, load_addr: int, size: int) -> tuple:
"""
Post-process psxrecomp-game's _full.c output for static binary compilation.
Returns (patched_src, sorted_list_of_in_overlay_virt_addrs).
Differences from DLL path:
- No overlay_api.h / OverlayCallbacks / overlay_init
- Callbacks are direct extern calls, not function pointers
- No __declspec(dllexport) — all functions have normal external linkage
- Returns function address list so caller can build the switch dispatch
"""
ov_lo = load_addr & 0x1FFFFFFF
ov_hi = ov_lo + size
def in_overlay(addr: int) -> bool:
return ov_lo <= (addr & 0x1FFFFFFF) < ov_hi
# 1. Insert preamble after last #include
last_inc = -1
for m in re.finditer(r'^#include\s+[<"].*[>"]\s*$', src, re.MULTILINE):
last_inc = m.end()
if last_inc == -1:
src = STATIC_PREAMBLE + src
else:
src = src[:last_inc] + '\n' + STATIC_PREAMBLE + src[last_inc:]
# 2. Remove out-of-range forward declarations
def drop_extern(m):
addr = int(m.group(1), 16)
return '' if not in_overlay(addr) else m.group(0)
src = re.sub(r'^void func_([0-9A-Fa-f]{8})\(CPUState\* cpu\);\n',
drop_extern, src, flags=re.MULTILINE)
# 3. Replace out-of-range direct calls with call_by_address
def fix_call(m):
addr = int(m.group(1), 16)
if in_overlay(addr):
return m.group(0)
return f'call_by_address(cpu, 0x{addr:08X}u)'
src = re.sub(r'\bfunc_([0-9A-Fa-f]{8})\(cpu\)', fix_call, src)
# 4. Collect in-overlay function definition addresses (no export annotation
# needed), spelled in the image's segment (§5.7).
func_virt_addrs = []
image_seg = segment_of(load_addr)
def collect_fn(m):
addr = int(m.group(1), 16)
if in_overlay(addr):
func_virt_addrs.append(image_seg | (addr & PHYS_MASK))
return m.group(0)
src = re.sub(r'^void func_([0-9A-Fa-f]{8})\(CPUState\* cpu\)$',
collect_fn, src, flags=re.MULTILINE)
return src, sorted(func_virt_addrs)
def namespace_generated_static(src: str, namespace: str,
func_virt_addrs: list) -> tuple:
"""Give one generated overlay image private C symbols.
Static mode combines several independently-generated C files into one
translation unit. The recompiler deliberately gives each file the same
helper names, and different overlay images may define functions at the same
guest address. Namespace the four common helpers, every in-image function,
and every alias-body helper. Return the guest-entry -> C-symbol map used by
the content-validated dispatcher.
"""
func_set = set(func_virt_addrs)
for helper in ('psx_lwl', 'psx_lwr', 'psx_swl', 'psx_swr'):
src = re.sub(rf'\b{helper}\b', f'{namespace}_{helper}', src)
def rename_func(m):
addr = _canonical_guest_addr(int(m.group(1), 16))
if addr not in func_set:
return m.group(0)
return f'{namespace}_func_{addr:08X}'
src = re.sub(r'\bfunc_([0-9A-Fa-f]{8})\b', rename_func, src)
def rename_alias(m):
addr = _canonical_guest_addr(int(m.group(1), 16))
if addr not in func_set:
return m.group(0)
return f'{namespace}_alias_body_{addr:08X}'
src = re.sub(r'\bpsx_alias_body_([0-9A-Fa-f]{8})\b', rename_alias, src)
symbols = {va: f'{namespace}_func_{va:08X}' for va in func_set}
return src, symbols
def parse_cps_continuation_owners(src: str) -> dict:
"""Return compiled block entry -> owning function for generated CPS C.
Runtime captures can prove a block label as a dispatch entry even when an
earlier capture never needed that label in the host's ``cpu->pc`` resume
switch. Static mode can add the missing switch arm and synthesize a wrapper
as long as the host's exact compiled code ranges match live RAM.
"""
definition_re = re.compile(
r'^void func_([0-9A-Fa-f]{8})\(CPUState\* cpu\)\n\{',
re.MULTILINE)
definitions = list(definition_re.finditer(src))
owners = {}
for index, match in enumerate(definitions):
host = _canonical_guest_addr(int(match.group(1), 16))
end = definitions[index + 1].start() if index + 1 < len(definitions) else len(src)
body = src[match.end():end]
for block in re.finditer(r'^block_([0-9A-Fa-f]{8}):', body,
re.MULTILINE):
entry = _canonical_guest_addr(int(block.group(1), 16))
owners.setdefault(entry, host)
return owners
def add_cps_resume_case(src: str, host_symbol: str,
host: int, entry: int) -> tuple:
"""Make ``entry`` a legal ``cpu->pc`` resume point in one native host."""
definition = re.search(
rf'^void {re.escape(host_symbol)}\(CPUState\* cpu\)\n\{{',
src, re.MULTILINE)
if not definition:
return src, False
next_definition = re.search(r'^void [A-Za-z_][A-Za-z0-9_]*\(CPUState\* cpu\)',
src[definition.end():], re.MULTILINE)
end = (definition.end() + next_definition.start()
if next_definition else len(src))
segment = src[definition.start():end]
if not re.search(rf'^block_{entry:08X}:', segment, re.MULTILINE):
return src, False
if f'case 0x{entry:08X}u: goto block_{entry:08X};' in segment:
return src, True
hook = segment.find('debug_server_log_call_entry')
prologue = segment[:hook] if hook >= 0 else ''
default_match = re.search(r'^\s+default:', prologue, re.MULTILINE)
if 'if (cpu->pc != 0u)' in prologue and default_match:
insert = definition.start() + default_match.start()
indent = re.match(r'\s*', prologue[default_match.start():]).group(0)
arm = f'{indent}case 0x{entry:08X}u: goto block_{entry:08X};\n'
return src[:insert] + arm + src[insert:], True
prologue_text = (
'\n if (cpu->pc != 0u) {\n'
' uint32_t _cont = cpu->pc; cpu->pc = 0;\n'
' switch (_cont) {\n'
f' case 0x{entry:08X}u: goto block_{entry:08X};\n'
f' case 0x{host:08X}u: break; /* entry at prologue */\n'
f' default: cpu->pc = _cont; psx_native_bad_entry(cpu, '
f'0x{host:08X}u, _cont); return;\n'
' }\n'
' }')
return src[:definition.end()] + prologue_text + src[definition.end():], True
_CPS_SIGNATURE_RE = re.compile(r'^void ([A-Za-z_][A-Za-z0-9_]*)\(CPUState\* cpu\)',
re.MULTILINE)
def add_cps_resume_cases(src: str, requests: list) -> tuple:
"""Batch form of add_cps_resume_case().
requests: [(host_symbol, host, [entry, ...]), ...]. Returns
(src, {host_symbol: set(entries made legal)}).
Per entry this applies exactly the rule add_cps_resume_case() applies --
the block label must exist in the host's own segment, an existing arm is
left alone, otherwise the arm goes before the resume switch's ``default:``
(or a fresh switch is created after the opening brace). The difference is
cost: the per-entry function re-scans and rebuilds the WHOLE part source
(tens of MB) for every entry, which made the post-pass quadratic -- 7,386
wrappers on a 19 MB part took ~7 minutes. Here every function signature is
located once, each host's segment is edited in isolation, and the source
is rebuilt once. Output is byte-identical to the per-entry form.
"""
sigs = list(_CPS_SIGNATURE_RE.finditer(src))
def_index = {}
for i, m in enumerate(sigs):
# A definition is the signature followed by "\n{"; the first one wins,
# like re.search() did.
if src.startswith('\n{', m.end()) and m.group(1) not in def_index:
def_index[m.group(1)] = i
edits = []
ok_by_symbol = {}
for host_symbol, host, entries in requests:
i = def_index.get(host_symbol)
if i is None:
continue
m = sigs[i]
start = m.start()
def_len = (m.end() + 2) - start # through the opening brace
end = sigs[i + 1].start() if i + 1 < len(sigs) else len(src)
seg = src[start:end]
labels = set(re.findall(r'^block_[0-9A-Fa-f]{8}:', seg, re.MULTILINE))
ok = set()
changed = False
for entry in entries:
if f'block_{entry:08X}:' not in labels:
continue
if f'case 0x{entry:08X}u: goto block_{entry:08X};' in seg:
ok.add(entry)
continue
hook = seg.find('debug_server_log_call_entry')
prologue = seg[:hook] if hook >= 0 else ''
default_match = re.search(r'^\s+default:', prologue, re.MULTILINE)
if 'if (cpu->pc != 0u)' in prologue and default_match:
insert = default_match.start()
indent = re.match(r'\s*', prologue[default_match.start():]).group(0)
arm = f'{indent}case 0x{entry:08X}u: goto block_{entry:08X};\n'
seg = seg[:insert] + arm + seg[insert:]
else:
prologue_text = (
'\n if (cpu->pc != 0u) {\n'
' uint32_t _cont = cpu->pc; cpu->pc = 0;\n'
' switch (_cont) {\n'
f' case 0x{entry:08X}u: goto block_{entry:08X};\n'
f' case 0x{host:08X}u: break; /* entry at prologue */\n'
f' default: cpu->pc = _cont; psx_native_bad_entry(cpu, '
f'0x{host:08X}u, _cont); return;\n'
' }\n'
' }')
seg = seg[:def_len] + prologue_text + seg[def_len:]
ok.add(entry)
changed = True
ok_by_symbol[host_symbol] = ok
if changed:
edits.append((start, end, seg))
if edits:
edits.sort()
pieces = []
pos = 0
for start, end, seg in edits:
pieces.append(src[pos:start])
pieces.append(seg)
pos = end
pieces.append(src[pos:])
src = ''.join(pieces)
return src, ok_by_symbol
def generate_overlay_dispatch(variants: list) -> str:
"""Generate byte-validated dispatch for all static overlay variants."""
unique = []
seen = set()
for variant in variants:
# The lookup keys on the exact PC, and each body is compiled for the
# segment of its PC (§5.7): a KUSEG, KSEG0 or KSEG1 entry of the same
# bytes is its own row. PC 0 is the hash table's empty marker.
if (segment_of(variant['addr']) not in SEGMENT_NAMES or
variant['addr'] == 0):
raise ValueError(f"static overlay entry 0x{variant['addr']:08X} "
"is not a KUSEG, KSEG0 or KSEG1 code PC")
ranges = tuple((lo & 0x1FFFFFFF, length)
for lo, length in variant['ranges'])
key = (variant['addr'], variant['crc'], ranges)
if key in seen:
continue
seen.add(key)
item = dict(variant)
item['ranges'] = ranges
unique.append(item)
unique.sort(key=lambda v: (v['addr'], v['crc'], v['ranges'], v['symbol']))
by_addr = {}
for index, variant in enumerate(unique):
variant['range_symbol'] = f'psx_ov_static_ranges_{index:05d}'
by_addr.setdefault(variant['addr'], []).append(variant)
lines = [
'',
'/* Auto-generated, content-validated overlay dispatch -- do not edit. */',
'extern int psx_overlay_static_code_matches(const uint32_t *lo_len_pairs,',
' uint32_t count,',
' uint32_t expected_crc);',
'static uint64_t psx_ov_static_checks = 0;',
'static uint64_t psx_ov_static_hits = 0;',
'static uint64_t psx_ov_static_variant_misses = 0;',
'static uint64_t psx_ov_static_address_misses = 0;',
'',
]
for variant in unique:
flat = []
for lo, length in variant['ranges']:
flat.extend((f'0x{lo:08X}u', f'0x{length:X}u'))
lines.append(
f'static const uint32_t {variant["range_symbol"]}[] = '
'{ ' + ', '.join(flat) + ' };')
# ---- O(1) address lookup ------------------------------------------------
# The dispatcher is consulted on EVERY interpreted entry, and the
# overwhelming majority of those calls are for addresses with no compiled
# entry at all. Measured on BoF3 during a battle-transition stall:
# ~264,000 calls/sec, of which ~1,160 out of every 1,161 found nothing.
# A `switch` over tens of thousands of sparse cases compiles to a binary
# search tree, so each of those fruitless calls walked ~log2(N) compares
# through a jump table far larger than cache -- which is why frame rate
# tracked compiled-case COUNT rather than variant chain depth, and why
# adding bands cost frame rate on screens where no overlay code ran.
#
# Replace it with a compile-time open-addressed hash table. The miss path
# -- the hot one -- becomes: hash, one load, compare, return. Two parallel
# uint32 arrays keep a miss inside a single 128 KB table: the address array
# answers "is this mine?" without touching the entry or variant arrays at
# all. Load factor is held at <= 0.5 to bound probe length.
entry_addrs = sorted(by_addr)
n_entries = len(entry_addrs)
def hash_slot(key: int, mask: int) -> int:
"""MUST stay bit-identical to psx_ov_hash_slot() emitted below."""
h = (key >> 2) & 0xFFFFFFFF
h = (h * 0x9E3779B1) & 0xFFFFFFFF
h ^= h >> 15
return h & mask
bits = 4
while (1 << bits) < max(n_entries, 1) * 2:
bits += 1
size = 1 << bits
mask = size - 1
table_addr = [0] * size
table_idx = [0] * size
max_probe = 0
for index, addr in enumerate(entry_addrs):
slot = hash_slot(addr, mask)
probe = 0
while table_addr[slot] != 0:
assert table_addr[slot] != addr, f'duplicate entry 0x{addr:08X}'
slot = (slot + 1) & mask
probe += 1
table_addr[slot] = addr
table_idx[slot] = index
max_probe = max(max_probe, probe)
# Flat variant array; each entry owns a contiguous run of it.
flat_variants = []
entry_rows = []
for addr in entry_addrs:
entry_rows.append((addr, len(flat_variants), len(by_addr[addr])))
flat_variants.extend(by_addr[addr])
def emit_table(name, values, per_line=8):
out = [f'static const uint32_t {name}[{size}] = {{']
for i in range(0, size, per_line):
chunk = ', '.join(f'0x{v:08X}u' for v in values[i:i + per_line])
out.append(f' {chunk},')
out.append('};')
return out
lines += [
'',
'typedef void (*PsxOvFn)(CPUState *cpu);',
'',
'typedef struct {',
' const uint32_t *ranges;',
' uint32_t count;',
' uint32_t crc;',
' PsxOvFn fn;',
'} PsxOvVariant;',
'',
'typedef struct {',
' uint32_t addr;',
' uint32_t first; /* index into psx_ov_variants */',
' uint32_t n; /* occupants compiled at this address */',
'} PsxOvEntry;',
'',
]
if flat_variants:
lines.append(f'static const PsxOvVariant '
f'psx_ov_variants[{len(flat_variants)}] = {{')
for variant in flat_variants:
lines.append(
f' {{ {variant["range_symbol"]}, '
f'{len(variant["ranges"])}u, 0x{variant["crc"]:08X}u, '
f'{variant["symbol"]} }},')
lines.append('};')
lines.append('')
lines.append(f'static const PsxOvEntry psx_ov_entries[{n_entries}] = {{')
for addr, first, count in entry_rows:
lines.append(f' {{ 0x{addr:08X}u, {first}u, {count}u }},')
lines.append('};')
lines.append('')
lines += emit_table('psx_ov_hash_addr', table_addr)
lines.append('')
lines += emit_table('psx_ov_hash_idx', table_idx)
lines.append('')
# Resident-occupant memo. On hardware the game CD-reads a file to a
# fixed address and jal's straight into it -- identity is implicit in
# control flow and costs nothing. We rediscover it by walking a band's
# occupants and CRC-gating each. On a deep band that walk dominated:
# the memory-card screen measured 41.25 checks per hit, i.e. ~41 failed
# gates before the resident one, 4.4 M failed checks per 2 s.
#
# Remember which occupant last satisfied each address and try it first.
# Correctness is unchanged because the memo only reorders candidates --
# every call still passes psx_overlay_static_code_matches(), so a band
# that swapped occupants fails the memo and falls into the full walk.
# The memo is a hint, never an authority.
lines.append(f'static uint16_t psx_ov_last_hit[{n_entries}];')
# Per-entry hit counters answer "did THIS image's variant run?" (for
# example a mod engine in a RAM mirror versus a menu overlay) through
# overlay_static_entries, where the global psx_ov_static_hits cannot.
lines.append(f'static uint64_t psx_ov_entry_hits[{n_entries}];')
lines.append('')
present = sorted({segment_of(v['addr']) for v in unique} or {KSEG0})
if present == [KSEG0]:
segment_gate = [
' /* Entries are the KSEG0 PCs their bodies were compiled for',
' * (PSX_OVERLAY_CODE_SEGMENT, overlay_loader.h). A KUSEG or KSEG1',
' * alias of the same bytes is a different architectural PC and',
' * stays in the interpreter. */',
' if ((addr & 0xE0000000u) != 0x80000000u) {',
]
else:
segment_gate = [
' /* Entries are the exact PCs their bodies were compiled for, in',
' * their own segment (docs/SEGMENT_AWARE_CODE.md §5.7). A PC in a',
' * segment no body was compiled for stays in the interpreter. */',
' if (' + ' &&\n '.join(
f'(addr & 0xE0000000u) != 0x{seg:08X}u' for seg in present) + ') {',
]
lines += [
f'/* {n_entries} dispatch addresses, {len(flat_variants)} variants, '
f'{size}-slot table (load {n_entries / size:.2f}), '
f'max probe {max_probe}. */',
f'#define PSX_OV_HASH_MASK 0x{mask:X}u',
'',
'static inline uint32_t psx_ov_hash_slot(uint32_t key) {',
' uint32_t h = key >> 2; /* entries are word-aligned */',
' h *= 0x9E3779B1u;',
' h ^= h >> 15;',
' return h & PSX_OV_HASH_MASK;',
'}',
'',
'void psx_overlay_static_get_stats(uint64_t *checks, uint64_t *hits,',
' uint64_t *variant_misses,',
' uint64_t *address_misses) {',
' if (checks) *checks = psx_ov_static_checks;',
' if (hits) *hits = psx_ov_static_hits;',
' if (variant_misses) *variant_misses = psx_ov_static_variant_misses;',
' if (address_misses) *address_misses = psx_ov_static_address_misses;',
'}',
'',
'/* Entry index of the last successful find (single-threaded dispatch). */',
'static uint32_t psx_ov_found_entry = 0;',
'',
'static const PsxOvVariant *psx_overlay_static_find_variant(uint32_t addr) {',
] + segment_gate + [
' psx_ov_static_address_misses++;',
' return 0;',
' }',
' const uint32_t key = addr;',
]
if not flat_variants:
lines += [
' (void)key;',
' (void)psx_ov_found_entry;',
' psx_ov_static_address_misses++;',
' return 0;',
'}',
'',
'int psx_overlay_static_can_dispatch(uint32_t addr) {',
' return psx_overlay_static_find_variant(addr) != 0;',
'}',
'',
'int psx_overlay_dispatch(CPUState *cpu, uint32_t addr) {',
' (void)cpu;',
' if (psx_overlay_static_find_variant(addr) == 0) return 0;',
' return 0;',
'}',
'',
'uint32_t psx_overlay_static_entry_count(void) { return 0u; }',
'',
'int psx_overlay_static_entry_stats(uint32_t index, uint32_t *addr,',
' uint32_t *variants, uint64_t *hits) {',
' (void)index; (void)addr; (void)variants; (void)hits;',
' return 0;',
'}',
'',
]
return '\n'.join(lines)
lines += [
' uint32_t slot = psx_ov_hash_slot(key);',
' for (;;) {',
' uint32_t slot_addr = psx_ov_hash_addr[slot];',
' if (slot_addr == 0u) {',
' /* Empty slot: this address is compiled nowhere. This is',
' * the overwhelmingly common case -- keep it one load. */',
' psx_ov_static_address_misses++;',
' return 0;',
' }',
' if (slot_addr == key) break;',
' slot = (slot + 1u) & PSX_OV_HASH_MASK;',
' }',
' {',
' const uint32_t ei = psx_ov_hash_idx[slot];',
' const PsxOvEntry *e = &psx_ov_entries[ei];',
' const PsxOvVariant *base = &psx_ov_variants[e->first];',
' const uint32_t n = e->n;',
' uint32_t memo = psx_ov_last_hit[ei];',
' uint32_t i;',
' if (memo >= n) memo = 0u;',
' /* Try the occupant that satisfied this address last time. On a',
' * deep band this is the difference between one CRC gate and',
' * walking every occupant. It is only a hint: the gate below',
' * still decides, so a swapped band simply falls through. */',
' if (n > 1u) {',
' const PsxOvVariant *v = base + memo;',
' psx_ov_static_checks++;',
' if (psx_overlay_static_code_matches(v->ranges, v->count,',
' v->crc)) {',
' psx_ov_found_entry = ei;',
' return v;',
' }',
' psx_ov_static_variant_misses++;',
' }',
' for (i = 0; i < n; i++) {',
' const PsxOvVariant *v = base + i;',
' if (n > 1u && i == memo) continue; /* already tried */',
' psx_ov_static_checks++;',
' if (psx_overlay_static_code_matches(v->ranges, v->count,',
' v->crc)) {',
' psx_ov_last_hit[ei] = (uint16_t)i;',
' psx_ov_found_entry = ei;',
' return v;',
' }',
' psx_ov_static_variant_misses++;',
' }',
' }',
' /* Address is ours but no occupant is resident -> interpreter. */',
' return 0;',
'}',
'',
'int psx_overlay_static_can_dispatch(uint32_t addr) {',
' return psx_overlay_static_find_variant(addr) != 0;',
'}',
'',
'int psx_overlay_dispatch(CPUState *cpu, uint32_t addr) {',
' const PsxOvVariant *v = psx_overlay_static_find_variant(addr);',
' if (!v) return 0;',
' psx_ov_static_hits++;',
' psx_ov_entry_hits[psx_ov_found_entry]++;',
' v->fn(cpu);',
' return 1;',
'}',
'',
f'uint32_t psx_overlay_static_entry_count(void) {{ return {n_entries}u; }}',
'',
'/* Entry `index` (sorted by address): its address, occupant count and',
' * dispatches served. 0 past the last entry. */',
'int psx_overlay_static_entry_stats(uint32_t index, uint32_t *addr,',
' uint32_t *variants, uint64_t *hits) {',
f' if (index >= {n_entries}u) return 0;',
' if (addr) *addr = psx_ov_entries[index].addr;',
' if (variants) *variants = psx_ov_entries[index].n;',
' if (hits) *hits = psx_ov_entry_hits[index];',
' return 1;',
'}',
'',
]
return '\n'.join(lines)
# ---------------------------------------------------------------------------
# DLL compilation
# ---------------------------------------------------------------------------
def parse_overlay_func_ids(src_path: str, data: bytes, load_addr: int,
size: int) -> list:
"""Parse the recompiler's _full.ranges manifest and return the list of
in-overlay function identities as (ev, code_crc, ranges) tuples, where
ev = virtual entry, code_crc = AUTHORITATIVE hash of the captured code-range
bytes (the exact bytes the recompiler compiled from), and ranges is the
coalesced [(lo, len), ...] code-range list.
This (entry, code_crc) pair is the per-function IDENTITY: the loader marks a
compiled entry callable iff live RAM matches code_crc, and overlay-cache v2
keys build/dedup decisions on this set rather than on the whole-region CRC.
binascii.crc32 (zlib, poly 0xEDB88320, init/final 0xFFFFFFFF) is bit-identical
to the runtime's crc32_compute, and `data` is the raw little-endian RAM image,
so the offline hash matches the runtime's hash of live RAM byte-for-byte."""
ov_lo = load_addr & 0x1FFFFFFF
ov_hi = ov_lo + size
def in_ov(a: int) -> bool:
return ov_lo <= (a & 0x1FFFFFFF) < ov_hi
# Parse the recompiler manifest into [(entry, [(lo, len), ...]), ...].
funcs: list[tuple[int, list[tuple[int, int]]]] = []
cur = None
with open(src_path) as f:
for line in f:
s = line.split()
if not s:
continue
if s[0] == 'F':
try:
addr = int(s[1], 16)
except (IndexError, ValueError):
cur = None
continue
if in_ov(addr):
cur = (addr, [])
funcs.append(cur)
else:
cur = None
elif s[0] == 'R' and cur is not None:
try:
lo, length = int(s[1], 16), int(s[2], 16)
except (IndexError, ValueError):
continue
cur[1].append((lo, length))
out = []
for entry, ranges in funcs:
if not ranges:
continue
crc = 0
ok = True
for lo, length in ranges:
off = (lo & 0x1FFFFFFF) - ov_lo
if off < 0 or off + length > len(data):
ok = False # range outside captured bytes — can't hash reliably
break
crc = binascii.crc32(data[off:off + length], crc)
if not ok:
continue
# The entry VA in the image's segment (§5.1, §5.7).
ev = (entry & PHYS_MASK) | segment_of(load_addr)
out.append((ev, crc & 0xFFFFFFFF, ranges))
return out
def _mips_control_kind(instr: int) -> int:
"""1=supported delayed transfer, 0=ordinary, -1=reserved/unsupported."""
op = (instr >> 26) & 0x3F
if op == 0:
return int((instr & 0x3F) in (0x08, 0x09)) # JR/JALR
if op == 0x01:
return 1 if ((instr >> 16) & 0x1F) in (0x00, 0x01, 0x10, 0x11) else -1
if op in (0x02, 0x03) or 0x04 <= op <= 0x07:
return 1
if 0x14 <= op <= 0x17:
return -1
if 0x10 <= op <= 0x13 and ((instr >> 21) & 0x1F) == 0x08:
return -1
return 0
def audit_func_id_delay_slots(func_ids: list, data: bytes,
load_addr: int) -> list:
"""Return unsafe emitted dependency errors for exact function identities."""
errors = []
base = load_addr & 0x1FFFFFFF
seg = segment_of(load_addr)
for entry, _crc, ranges in func_ids:
if len(ranges) > 16:
errors.append((entry, None,
f'{len(ranges)} ranges exceed loader limit 16'))
continue
normalized = []
malformed = False
for lo, length in ranges:
phys = lo & 0x1FFFFFFF
off = phys - base
if (phys & 3) or (length & 3) or length < 4 or \
off < 0 or off + length > len(data):
errors.append((entry, phys | seg,
'malformed/out-of-capture code range'))
malformed = True
break
normalized.append((phys, phys + length))
if malformed:
continue
def contains_word(addr: int) -> bool:
return any(lo <= addr and addr + 4 <= hi
for lo, hi in normalized)
controls = set()
for lo, hi in normalized:
for pc in range(lo, hi, 4):
off = pc - base
instr = int.from_bytes(data[off:off + 4], 'little')
kind = _mips_control_kind(instr)
if kind < 0:
errors.append((entry, pc | seg,
'reserved/unsupported branch encoding'))
elif kind > 0:
controls.add(pc)
if not contains_word(pc + 4):
errors.append((entry, pc | seg,
f'delay slot 0x{(pc + 4) | seg:08X} '
'is not identity-hashed'))
delay_pcs = {pc + 4 for pc in controls}
nested = sorted(controls & delay_pcs)
for pc in nested:
errors.append((entry, pc | seg,
'control transfer in a delay slot'))
return errors
def overlay_ranges_text(func_ids: list, pair_id: int | None = None,
provenance: str | None = None) -> str:
"""Serialize one loader manifest.
``P`` is an optional DLL/manifest publication identity. Old runtimes
ignore the unfamiliar record, while pair-aware runtimes require a matching
``overlay_pair_id`` export whenever it is present.
"""
if provenance is not None and provenance not in \
NON_AUTHORITY_MANIFEST_PROVENANCES:
raise ValueError(f'unsupported overlay manifest provenance: {provenance}')
out_lines = ['# psxrecomp overlay code-range manifest v2 (entry+code_crc)\n']
if provenance is not None:
out_lines.append(f'# psxrecomp overlay provenance {provenance}\n')
# F entries are full VAs in the shard's segment (§5.7). A KUSEG or KSEG1
# shard says so with S before its first F; a KSEG0 manifest keeps the
# pre-§5.7 text exactly (no S means KSEG0).
segments = {segment_of(ev) for ev, _crc, _ranges in func_ids}
if len(segments) > 1:
raise ValueError('one manifest names entries in several segments: ' +
', '.join(f'0x{seg:08X}' for seg in sorted(segments)))
seg = segments.pop() if segments else KSEG0
if seg not in SEGMENT_NAMES:
raise ValueError(f'manifest entries in segment 0x{seg:08X}, '
'which maps no RAM')
if seg != KSEG0:
out_lines.append(f'S {seg:08X}\n')
if pair_id is not None:
out_lines.append(f'P {pair_id & 0xFFFFFFFFFFFFFFFF:016X}\n')
for ev, crc, ranges in func_ids:
out_lines.append(f'F {ev:08X} {crc & 0xFFFFFFFF:08X}\n')
for lo, length in ranges:
# Byte extents, KSEG0-spelled in every segment's manifest.
out_lines.append(
f'R {(lo & PHYS_MASK) | KSEG0:08X} {length:X}\n')
return ''.join(out_lines)
def overlay_pair_id(src: str, func_ids: list,
provenance: str | None = None) -> int:
"""Bind a newly compiled DLL to the exact C and range manifest it uses."""
digest = hashlib.sha256()
digest.update(b'psxrecomp overlay pair v1\0')
digest.update(src.encode('utf-8'))
digest.update(b'\0')
digest.update(overlay_ranges_text(
func_ids, provenance=provenance).encode('ascii'))
return int.from_bytes(digest.digest()[:8], 'big')
def add_overlay_pair_export(src: str, pair_id: int) -> str:
"""Add the optional v2-pair binding export without changing the shard ABI."""
return src + f'''\n
PSX_OVERLAY_EXPORT uint64_t overlay_pair_id(void) {{
return UINT64_C(0x{pair_id:016X});
}}
'''
def write_overlay_ranges_from(func_ids: list, out_path: str,
pair_id: int | None = None,
provenance: str | None = None) -> int:
"""Write the {phys}_{key}.ranges manifest (v2) from a func-id list produced by
parse_overlay_func_ids. Returns the number of functions written.
Manifest v2 line format:
F <entry_hex> <code_crc_hex> one per function
R <lo_hex> <len_hex> one per coalesced code range"""
with open(out_path, 'w') as f:
f.write(overlay_ranges_text(func_ids, pair_id, provenance))
return len(func_ids)
def write_overlay_ranges(src_path: str, out_path: str,
data: bytes, load_addr: int, size: int) -> int:
"""Back-compat wrapper: parse the recompiler manifest and write the v2 .ranges.
See parse_overlay_func_ids / write_overlay_ranges_from."""
return write_overlay_ranges_from(
parse_overlay_func_ids(src_path, data, load_addr, size), out_path)
def load_region_coverage(cache_dir: str, phys_addr: int,
expected_abi: int | None = None) -> set:
"""Set of (ev, code_crc) function identities already provided by built DLLs
for this region_start. The loader content-matches per function across ALL
DLLs sharing a region_start, so a function is "covered" as soon as ANY
existing .ranges for that region_start lists it with a matching code_crc.
overlay-cache v2 uses this to skip the (expensive) gcc compile when a capture
would add no new function identity (the volatile-data redundant-build case)."""
covered = set()
prefix = f'{phys_addr:08X}_'
try:
names = os.listdir(cache_dir)
except OSError:
return covered
for name in names:
if not (name.startswith(prefix) and name.endswith('.ranges')):
continue
dll_path = os.path.join(
cache_dir, os.path.splitext(name)[0] + overlay_ext())
covered.update(
(entry, code_crc)
for entry, code_crc, _ranges in load_shard_func_ids(
dll_path, expected_abi))
return covered
def _addr_in_func_ids(addr: int, func_ids: list) -> bool:
"""True if addr falls inside any code range of the given func-id list."""
a = addr & 0x1FFFFFFF
for _ev, _crc, ranges in func_ids:
for lo, length in ranges:
lo &= 0x1FFFFFFF
if lo <= a < lo + length:
return True
return False
def load_region_entry_set(cache_dir: str, phys_addr: int,
expected_abi: int | None = None) -> set:
"""Set of phys-normalized F-line ENTRY addresses provided by ALL built DLLs
(region + fragment) for this region_start. This — not range coverage — is
the dispatchability test: native code is enterable ONLY at F entries, so a
dispatch-proven PC inside a compiled range but absent from every manifest's
F set still runs its whole chain on the interpreter (the 0x80106D7C class,
2026-07-06: 80% of Tomba2 attract interp residue was two range-covered,
entry-less interior PCs that the range-based orphan test refused to
fragment, while the region compile skipped as 'already covered')."""
return {
entry & 0x1FFFFFFF
for entry, _crc in load_region_coverage(
cache_dir, phys_addr, expected_abi)
}
def parse_runtime_shard_manifest(manifest: str,
require_pair: bool = True,
expected_segment: int | None = None
) -> tuple[int | None, list]:
"""Parse only identities representable by the runtime manifest contract.
F entries are full VAs in the manifest's segment: ``S <segment>`` before
the first F, KSEG0 when absent (§5.7). ``expected_segment`` is the segment
of the directory the shard lives in; the runtime refuses a manifest that
names another, and so does this."""
def hex_field(token: str, max_digits: int) -> int:
if not re.fullmatch(rf'[0-9A-Fa-f]{{1,{max_digits}}}', token):
raise ValueError(f'invalid manifest hex field: {token!r}')
return int(token, 16)
pair_ids = []
funcs = []
current = None
segment = None
if '\0' in manifest:
return None, []
# Runtime reads physical records with fgets(), so only LF terminates a
# record. A trailing CR in CRLF is ordinary whitespace; a bare CR between
# directives is not a second record and must therefore fail identically.
for line in manifest.split('\n'):
if len(line) > MANIFEST_PHYSICAL_LINE_MAX:
return None, []
cr = line.find('\r')
record_width = len(line) if cr < 0 else cr
if record_width > MANIFEST_LINE_MAX:
return None, []
stripped = line.strip(MANIFEST_ASCII_WHITESPACE)
parts = (re.split(r'[ \t\r\v\f]+', stripped)
if stripped else [])
if not parts:
continue
if parts[0] == 'S':
if len(parts) != 2 or segment is not None or funcs:
return None, []
try:
segment = hex_field(parts[1], 8)
except ValueError:
return None, []
if segment not in SEGMENT_NAMES:
return None, []
elif parts[0] == 'P':
if len(parts) != 2:
return None, []
try:
pair_ids.append(hex_field(parts[1], 16))
except ValueError:
return None, []
elif parts[0] == 'F':
if len(parts) != 3:
return None, []
try:
current = [hex_field(parts[1], 8), hex_field(parts[2], 8), []]
except ValueError:
return None, []
if segment_of(current[0]) != (KSEG0 if segment is None else segment):
return None, []
funcs.append(current)
elif parts[0] == 'R' and current is not None:
if len(parts) != 3:
return None, []
try:
current[2].append((hex_field(parts[1], 8),
hex_field(parts[2], 8)))
except ValueError:
return None, []
if len(pair_ids) > 1 or (require_pair and len(pair_ids) != 1) or not funcs:
return None, []
if (expected_segment is not None and
(KSEG0 if segment is None else segment) != expected_segment):
return None, []
entries = [entry & 0x1FFFFFFF for entry, _crc, _ranges in funcs]
if len(set(entries)) != len(entries):
return None, []
for entry, _crc, ranges in funcs:
entry_phys = entry & 0x1FFFFFFF
if (not 1 <= len(ranges) <= 16 or
any(length <= 0 for _start, length in ranges) or
any((start & 0x1FFFFFFF) >= PSX_RAM_SIZE or
length > PSX_RAM_SIZE - (start & 0x1FFFFFFF)
for start, length in ranges) or
not any((start & 0x1FFFFFFF) <= entry_phys <
(start & 0x1FFFFFFF) + length
for start, length in ranges)):
return None, []
return (pair_ids[0] if pair_ids else None,
[tuple(func) for func in funcs])
def load_shard_func_ids(dll_path: str,
expected_abi: int | None = None,
include_supplemental: bool = True) -> list:
"""Read one runtime-valid shard manifest as ``(entry, crc, ranges)``.
A nonempty DLL and manifest are not enough: the pair-aware runtime rejects
a manifest whose ``P`` identity does not match the DLL export. Discovery
must apply the same validity rule before allowing a cached shard to
suppress a needed fragment.
"""
ranges_path = os.path.splitext(dll_path)[0] + '.ranges'
try:
with _shard_pair_lock(dll_path):
_recover_shard_pair_locked(dll_path)
if (not os.path.isfile(dll_path) or
os.path.getsize(dll_path) == 0 or
not os.path.isfile(ranges_path) or
os.path.getsize(ranges_path) == 0):
return []
with open(ranges_path, encoding='ascii', newline='') as f:
manifest = f.read()
if (not include_supplemental and
manifest_declares_supplemental_provenance(manifest)):
return []
pair_id, valid_funcs = parse_runtime_shard_manifest(
manifest, require_pair=False,
expected_segment=cache_dir_segment(os.path.dirname(dll_path)))
if (not valid_funcs or not _dll_runtime_exports_match(
dll_path, expected_abi, pair_id,
{entry for entry, _crc, _ranges in valid_funcs})):
return []
except (OSError, UnicodeError):
return []
return valid_funcs
def load_shard_entry_set(dll_path: str,
expected_abi: int | None = None) -> set:
"""Phys-normalized F entries from one runtime-valid shard pair."""
return {
entry & 0x1FFFFFFF
for entry, _crc, _ranges in load_shard_func_ids(dll_path, expected_abi)
}
def load_shard_code_ranges(dll_path: str,
expected_abi: int | None = None) -> list:
"""Phys-normalized guarded R intervals from one runtime-valid shard pair."""
return [
(start & 0x1FFFFFFF, (start & 0x1FFFFFFF) + length)
for _entry, _crc, ranges in load_shard_func_ids(dll_path, expected_abi)
for start, length in ranges if length > 0
]
def load_region_current_variant_coverage(cache_dir: str, phys_addr: int,
data: bytes, load_addr: int,
size: int,
expected_abi: int | None = None
) -> tuple[set, list]:
"""Return exact entries/ranges whose manifest CRC matches ``data``.
Shards for different byte variants share a region-start namespace. An F
entry from another variant cannot satisfy the current capture: the runtime
will reject it by code CRC. Recompute each cached function identity over
the current bytes and expose only identities the runtime can actually call.
"""
entries = set()
ranges_out = []
prefix = f'{phys_addr:08X}_'
try:
names = os.listdir(cache_dir)
except OSError:
return entries, ranges_out
for name in names:
if not (name.startswith(prefix) and name.endswith('.ranges')):
continue
dll_path = os.path.join(
cache_dir, os.path.splitext(name)[0] + overlay_ext())
shard_entries, shard_ranges = current_variant_func_id_coverage(
load_shard_func_ids(dll_path, expected_abi), data, load_addr, size)
entries.update(shard_entries)
ranges_out.extend(shard_ranges)
return entries, ranges_out
def current_variant_func_ids(func_ids: list, data: bytes,
load_addr: int, size: int) -> list:
"""Retain complete identities whose guarded bytes match this image."""
valid_ids = []
ov_lo = load_addr & 0x1FFFFFFF
ov_hi = ov_lo + size
for entry, expected_crc, ranges in func_ids:
crc = 0
valid = True
for start, length in ranges:
lo = start & 0x1FFFFFFF
if length <= 0 or lo < ov_lo or lo + length > ov_hi:
valid = False
break
crc = binascii.crc32(data[lo - ov_lo:lo - ov_lo + length], crc)
if valid and (crc & 0xFFFFFFFF) == (expected_crc & 0xFFFFFFFF):
candidate = [(entry, expected_crc, ranges)]
if not audit_func_id_delay_slots(candidate, data, load_addr):
valid_ids.append((entry, expected_crc, ranges))
return valid_ids
def load_region_current_variant_func_ids(cache_dir: str, phys_addr: int,
data: bytes, load_addr: int,
size: int,
expected_abi: int | None = None,
include_supplemental: bool = True
) -> list:
"""Complete runtime-valid identities matching one captured byte recipe."""
out = []
prefix = f'{phys_addr:08X}_'
try:
names = os.listdir(cache_dir)
except OSError:
return out
for name in names:
if not (name.startswith(prefix) and name.endswith('.ranges')):
continue
dll_path = os.path.join(
cache_dir, os.path.splitext(name)[0] + overlay_ext())
out.extend(current_variant_func_ids(
load_shard_func_ids(
dll_path, expected_abi,
include_supplemental=include_supplemental),
data, load_addr, size))
return out
def current_variant_func_id_coverage(func_ids: list, data: bytes,
load_addr: int,
size: int) -> tuple[set, list]:
"""Filter parsed identities to those callable for these captured bytes."""
entries = set()
ranges_out = []
for entry, _expected_crc, ranges in current_variant_func_ids(
func_ids, data, load_addr, size):
entries.add(entry & 0x1FFFFFFF)
ranges_out.extend(
((start & 0x1FFFFFFF), (start & 0x1FFFFFFF) + length)
for start, length in ranges)
return entries, ranges_out
# ---------------------------------------------------------------------------
# Doomed-interior fail memo
# ---------------------------------------------------------------------------
# An executed orphan interior whose bytes (in THIS captured image) are data, not
# code, fails the generated-C audit EVERY time — but the fragment pass re-attempts
# it on every autocompile cycle, making the recompiler walk thousands of words of
# data and emit tens of thousands of lines of C only to reject it (measured on
# Vigilante 8: 91 interiors, ~16k walked words each, per cycle). The memo records
# which (region_phys, interior_pc, region_crc) fragments have already failed so
# later cycles SKIP them instead of re-walking. Correctness is preserved:
# - The key includes region_crc, so a DIFFERENT captured variant (different
# bytes at the same PC) is NOT memoized — it re-attempts and can succeed.
# - The memo file lives inside the cg<N>_<hash> cache dir, so ANY codegen/
# contract/header change (which bumps the hash -> fresh dir -> fresh memo)
# re-attempts everything (e.g. the psx_rfe_mark_escape contract fix).
# So a memoized skip only ever elides a build that is deterministically doomed.
# Printed when a capture contributes no walk-root seed. The wording is
# deliberate: the region is NOT declared data-only -- its executed dispatch
# demands are still served by the isolated-fragment pass, which runs
# separately. Named so the two emit sites and the test cannot drift apart.
# They did: 4683e923 reworded both prints and left
# test_compile_overlays_static_split asserting the old string.
NO_SHARED_WALK_ROOT_SEEDS_SKIP = (
'SKIP: no shared walk-root seeds; checking fragments separately')
INTERIOR_FAIL_MEMO = 'interior_fail_memo.txt'
def _interior_fail_key(phys_addr: int, interior: int, data: bytes,
load_addr: int, size: int, producer_ranges,
cross_call_allow, expected_abi: int, cps: bool,
toml_doc: dict, guard_bytes: int = 0) -> str:
"""Stable identity of every input to deterministic fragment generation."""
recipe = {
# v3 adds guard_bytes: the analysis bound is an input to generation, so
# a memo recorded under the old (guard-word-as-code) analysis must not
# suppress a retry under the corrected one.
'schema': 'psxrecomp fragment fail key v3',
'phys_addr': phys_addr,
'interior': interior & 0x1FFFFFFF,
'load_addr': load_addr,
'size': size,
'guard_bytes': guard_bytes,
'data_sha256': hashlib.sha256(data).hexdigest(),
'producer_ranges': [list(pair) for pair in producer_ranges],
'cross_call_allow': sorted(set(cross_call_allow)),
'overlay_abi': expected_abi,
'cps': bool(cps),
'toml': toml_doc,
}
digest = hashlib.sha256(json.dumps(
recipe, sort_keys=True, separators=(',', ':'),
default=str).encode('utf-8')).hexdigest()[:24]
return f'{phys_addr:08X}_{interior & 0x1FFFFFFF:08X}_{digest}'
def interior_failure_is_deterministic(reason: str) -> bool:
"""Only guest-byte/codegen audit verdicts belong in the persistent memo."""
return (reason.startswith(('generated-c-audit:', 'requested-entry-audit:',
'guest-walk-audit:')) or
unsupported_guest_branch_rejection(reason))
def unsupported_guest_branch_rejection(reason: str) -> bool:
"""An invalid guest branch is data/unsupported code, not a tool failure.
Other delay-slot identity failures remain fatal: missing guarded delay
words or a changed instruction identity can indicate an emitter defect.
"""
return (reason.startswith('delay-slot-identity:') and
reason.endswith(': reserved/unsupported branch encoding'))
def reconcile_empty_primary_scans(pending, cache_dir, expected_abi, stats):
"""Accept an empty primary only if guarded fragments serve every root.
Each capture view is checked against its own segment's shards (§5.7)."""
for label, physical, load, size, data, roots in pending:
seg = segment_of(load)
current, _ = load_region_current_variant_coverage(
segment_cache_dir(cache_dir, seg), physical, data, load, size,
expected_abi)
served = {(entry & PHYS_MASK) | seg for entry in current}
if roots and roots <= served:
print(f' primary scan recovered by guarded fragments: {label}')
stats.add_skip()
else:
stats.add_fail(label, 'no_ranges',
'primary roots remain without guarded native coverage')
def optional_static_fragment_rejection(entry: int, job: dict,
reason: str) -> bool:
"""Whether an unplayed speculative candidate failed safely.
Publishing nothing is the correct outcome for a static-only candidate that
fails a deterministic byte/codegen audit. Execution evidence or an
explicit operator demand turns the same rejection into a real coverage
failure and therefore remains fatal.
"""
return (entry in job.get('static_demands', set()) and
entry not in job.get('executed', set()) and
entry not in job.get('forced', set()) and
interior_failure_is_deterministic(reason))
def interior_fail_memo_action(entry: int, job: dict, force: bool) -> str:
"""Choose ``retry``, ``fail``, or ``skip`` for a deterministic miss.
Required executed/forced demands remain loud without repeating an
expensive deterministic data-as-code walk. ``--force`` explicitly retries
it; an unplayed speculative demand remains a safe skip.
"""
if force:
return 'retry'
if (entry in job.get('executed', set()) or
entry in job.get('forced', set())):
return 'fail'
return 'skip'
def load_interior_fail_memo(cache_dir: str) -> dict:
memo = {}
try:
with open(os.path.join(cache_dir, INTERIOR_FAIL_MEMO)) as f:
for ln in f:
ln = ln.strip()
if ln and not ln.startswith('#'):
key, _sep, reason = ln.partition('#')
memo[key.strip()] = (reason.strip() or
'known deterministic failure')
except OSError:
pass
return memo
def append_interior_fail_memo(cache_dir: str, key: str, reason: str) -> None:
try:
os.makedirs(cache_dir, exist_ok=True)
with open(os.path.join(cache_dir, INTERIOR_FAIL_MEMO), 'a') as f:
f.write(f'{key} # {(reason or "").strip()[:120]}\n')
except OSError:
pass # best-effort; a memo write failure must never break the compile
def served_static_variant_keys(parts) -> set:
"""(entry, image_crc) pairs the built static parts can dispatch.
Coverage is per byte-variant: a part serves an address only for the image
it was compiled from, because the dispatcher's CRC gate rejects that
identity against any other resident bytes."""
return {(variant['addr'], part['image_crc'])
for part in parts for variant in part['variants']}
def select_static_fragment_demands(requested_keys, parts) -> list:
"""Requested (entry, image_crc) demands no built part serves, sorted.
Keyed by (entry, image crc) and NEVER by the bare address. With address
keys, the first variant at an address -- from any capture of the band --
marked that address served for every other capture, and the per-entry
source map kept only the last capture's bytes, so in a band with N
occupants exactly one occupant ever received isolated fragments
(BoF3 SCENARIO band, 2026-09-05: 20 sections, SCENA16 resident, all 53 of
its observed entries interpreted while every spanning piece had been
compiled from SCENA19/04/06 bytes and could never validate)."""
return sorted(set(requested_keys) - served_static_variant_keys(parts))
_STATIC_IMAGE_SOURCES: dict = {}
def _init_static_fragment_worker(image_sources: dict) -> None:
"""Pool initializer: the image table travels to each worker ONCE.
A demand is (entry, image); thousands of demands share a few hundred
images, so pickling the bytes per job would move the whole capture set
through the pipe once per demand. Workers look images up by key."""
global _STATIC_IMAGE_SOURCES
_STATIC_IMAGE_SOURCES = image_sources
def static_fragment_job(entry: int, image_key, args):
"""Worker for the static isolated-fragment pass: one (entry, image).
Module-level and self-contained so it runs in a ProcessPoolExecutor.
Returns (part or None, reason)."""
data, load_addr, size, phys_addr, guard_bytes = \
_STATIC_IMAGE_SOURCES[image_key]
with image_segment(segment_of(load_addr)):
return generate_interior_fragment_static(
entry, data, load_addr, size, phys_addr, args,
guard_bytes=guard_bytes)
def generate_interior_fragment_static(interior: int, data: bytes,
load_addr: int, size: int,
phys_addr: int, args, *,
guard_bytes: int):
"""Generate one isolated, exact-range-gated static interior shard.
Returns (part, reason): part is None on failure and reason then says
why. Reasons starting with 'generated-c-audit:', 'requested-entry-audit:'
or 'guest-walk-audit:' are deterministic verdicts on these exact bytes
(the interior is data in this image) and are safe to memoize; anything
else (recompiler exit, missing output) is not."""
with tempfile.TemporaryDirectory() as tmp:
psx = os.path.join(tmp, 'frag.psx')
with open(psx, 'wb') as f:
f.write(make_psxexe(load_addr, interior, data,
guard_bytes=guard_bytes))
seeds_path = os.path.join(tmp, 'seeds.txt')
with open(seeds_path, 'w') as f:
f.write(f'dispatch_root 0x{interior:08X}\n')
out_dir_tmp = os.path.join(tmp, 'out')
os.makedirs(out_dir_tmp)
cmd = [args.recompiler, psx, '--seeds', seeds_path,
'--out-dir', out_dir_tmp, '--overlay',
'--ws-config', os.path.abspath(args.game_toml)]
cmd += recompiler_project_root_args(args)
sub_env = dict(os.environ)
if args.cps:
sub_env['PSX_CPS'] = '1'
result = subprocess.run(
cmd, capture_output=True, text=True,
cwd=os.path.dirname(os.path.abspath(args.game_toml)), env=sub_env)
if result.returncode != 0:
# A walk from this interior that runs off the image (a branch at
# the last word, its delay slot past the end) is the recompiler's
# own verdict that these bytes are not code from here: it throws
# rather than auditing. Deterministic for the image, so memoizable
# alongside the audit rejections.
text = (result.stderr or '') + (result.stdout or '')
walk = [ln.strip() for ln in text.splitlines()
if 'outside the input image' in ln or
'cannot emit control flow' in ln]
if walk:
return None, 'guest-walk-audit: ' + walk[0][:160]
return None, f'recompiler: exit {result.returncode}'
full_c = ranges_src = None
for filename in os.listdir(out_dir_tmp):
if filename.endswith('_full.c'):
full_c = os.path.join(out_dir_tmp, filename)
elif filename.endswith('_full.ranges'):
ranges_src = os.path.join(out_dir_tmp, filename)
if not full_c or not ranges_src:
return None, 'no-output: recompiler emitted no _full.c/_full.ranges'
with open(full_c) as f:
src, func_addrs = patch_generated_c_static(
f.read(), load_addr, size)
image_crc = binascii.crc32(data) & 0xFFFFFFFF
audit = audit_generated_c(src, load_addr, size, image_crc, {})
if audit['unknown_bad'] or audit['unsupported_todo_addrs']:
return None, (f'generated-c-audit: {len(audit["unknown_bad"])} '
f'unknown_bad, '
f'{len(audit["unsupported_todo_addrs"])} unsupported')
func_ids = parse_overlay_func_ids(ranges_src, data, load_addr, size)
ids_by_addr = {}
for ev, code_crc, ranges in func_ids:
ids_by_addr.setdefault(ev, []).append((code_crc, ranges))
if set(func_addrs) - set(ids_by_addr):
return None, 'static-ranges: dispatchable function(s) lack exact ranges'
entry = (interior & PHYS_MASK) | segment_of(load_addr)
if entry not in ids_by_addr or entry not in set(func_addrs):
return None, 'requested-entry-audit: fragment omitted its own entry'
namespace = (f'ov_frag_{phys_addr:08X}_{image_crc:08X}_'
f'{entry:08X}')
continuation_owners = parse_cps_continuation_owners(src)
src, symbols = namespace_generated_static(src, namespace, func_addrs)
variants = []
for ev in sorted(func_addrs):
for code_crc, ranges in ids_by_addr[ev]:
variants.append({
'addr': ev,
'symbol': symbols[ev],
'crc': code_crc,
'ranges': ranges,
})
return {
'src': src,
'variants': variants,
'namespace': namespace,
'image_crc': image_crc,
'func_addrs': set(func_addrs),
'symbols': symbols,
'ids_by_addr': ids_by_addr,
'continuation_owners': continuation_owners,
}, 'ok'
def validate_overlay_func_ids(func_ids: list,
label: str = 'shard') -> str | None:
"""Validate identities against the runtime's fixed manifest contract."""
entries = [entry & 0x1FFFFFFF for entry, _crc, _ranges in func_ids]
if len(set(entries)) != len(entries):
return f'{label} contains duplicate function entries'
for entry, _crc, ranges in func_ids:
if not 1 <= len(ranges) <= 16:
return (f'{label} entry 0x{entry:08X} has {len(ranges)} ranges '
'(runtime supports 1..16)')
if any(length <= 0 for _lo, length in ranges):
return f'{label} entry 0x{entry:08X} has an empty guarded range'
if not _addr_in_func_ids(entry, [(entry, 0, ranges)]):
return f'{label} entry 0x{entry:08X} is outside its guarded ranges'
return None
def validate_fragment_requested_ids(requested: set[int], frag_ids: list,
generated_defs: set[int]) -> str | None:
"""Return a fail-closed reason when a fragment omitted a requested entry.
A non-empty manifest is not sufficient: the recompiler can emit some other
reachable function while rejecting/demoting an exact requested PC. The
loader can also retain at most 16 ranges for one function. Refuse to
publish unless every requested entry has one unambiguous, runtime-
representable identity whose guarded bytes include the entry itself.
"""
for requested_entry in sorted(requested):
entry = _canonical_guest_addr(requested_entry)
if entry not in generated_defs:
return f'requested entry 0x{entry:08X} missing from generated C'
matches = [
(ev, crc, ranges) for ev, crc, ranges in frag_ids if ev == entry
]
if len(matches) != 1:
return (f'requested entry 0x{entry:08X} has {len(matches)} '
'manifest identities (expected exactly 1)')
contract_error = validate_overlay_func_ids(frag_ids, 'fragment')
if contract_error:
return contract_error
for requested_entry in sorted(requested):
entry = _canonical_guest_addr(requested_entry)
matches = [
(ev, crc, ranges) for ev, crc, ranges in frag_ids if ev == entry
]
if not _addr_in_func_ids(entry, matches):
return f'requested entry 0x{entry:08X} is outside its guarded ranges'
return None
def validate_interior_fragment_ids(interior: int, frag_ids: list,
generated_defs: set[int]) -> str | None:
"""Compatibility wrapper for a singleton isolated fragment."""
return validate_fragment_requested_ids(
{interior}, frag_ids, generated_defs)
def validate_hosted_fragment_ids(requested: set[int], host_specs: dict,
frag_ids: list,
generated_defs: set[int],
allow_missing_targets: bool = False
) -> str | None:
"""Require every emitted host/alias to equal its selected cached owner.
The recompiler's host-range block boundary is only a syntactic landing-pad
check. Safety comes from this publication gate: the original host and
every added alias must reproduce the already runtime-validated owner CRC
and complete guarded-range geometry exactly.
"""
targets = {_canonical_guest_addr(entry) for entry in requested}
specs = {
_canonical_guest_addr(entry): spec
for entry, spec in host_specs.items()
}
if set(specs) != targets:
return 'hosted target/spec set mismatch'
expected_by_entry = {}
hosts = set()
for target, spec in sorted(specs.items()):
host = _canonical_guest_addr(spec['host'])
expected = _normalized_func_identity(spec['owner_identity'])
if expected[0] != host:
return f'hosted target 0x{target:08X} owner entry differs from host'
if target == host:
return f'hosted target 0x{target:08X} equals its host'
hosts.add(host)
prior = expected_by_entry.setdefault(host, expected[1:])
if prior != expected[1:]:
return f'hosted host 0x{host:08X} has conflicting identities'
expected_by_entry[target] = expected[1:]
if targets & hosts:
return 'hosted target is also a host in the same shard'
required = targets | hosts
contract_error = validate_overlay_func_ids(frag_ids, 'hosted fragment')
if contract_error:
return contract_error
if len(frag_ids) > HOSTED_EMITTED_IDENTITY_CAP:
return (f'hosted shard emitted {len(frag_ids)} identities '
f'(cap {HOSTED_EMITTED_IDENTITY_CAP})')
intervals = sorted(
(_canonical_guest_addr(start),
_canonical_guest_addr(start) + length)
for _entry, _crc, ranges in frag_ids
for start, length in ranges)
guarded_bytes = 0
merged_end = None
for start, end in intervals:
if merged_end is None or start > merged_end:
guarded_bytes += end - start
merged_end = end
elif end > merged_end:
guarded_bytes += end - merged_end
merged_end = end
if guarded_bytes > HOSTED_UNIQUE_GUARDED_BYTE_CAP:
return (f'hosted shard guards {guarded_bytes} unique bytes '
f'(cap {HOSTED_UNIQUE_GUARDED_BYTE_CAP})')
by_entry = {}
for func_id in frag_ids:
identity = _normalized_func_identity(func_id)
by_entry.setdefault(identity[0], []).append(identity)
for entry in sorted(required):
matches = by_entry.get(entry, [])
emitted = entry in generated_defs
if (allow_missing_targets and entry in targets and
not matches and not emitted):
continue
if not emitted:
return f'requested entry 0x{entry:08X} missing from generated C'
if len(matches) != 1:
return (f'hosted entry 0x{entry:08X} has {len(matches)} '
'normalized identities (expected exactly 1)')
if matches[0][1:] != expected_by_entry[entry]:
return (f'hosted entry 0x{entry:08X} changed selected owner '
'CRC/range identity')
return None
def hosted_fragment_served_targets(requested: set[int], frag_ids: list,
generated_defs: set[int]) -> set[int]:
"""Targets actually emitted after the C++ block-leader proof."""
manifest_entries = {
_canonical_guest_addr(entry) for entry, _crc, _ranges in frag_ids
}
return ({_canonical_guest_addr(entry) for entry in requested} &
manifest_entries &
{_canonical_guest_addr(entry) for entry in generated_defs})
def make_interior_fragment_job(phys_addr: int, load_addr: int, size: int,
data: bytes, seed_audit: dict,
forced_interiors: set[int],
capture: dict | None = None) -> dict | None:
"""Build one explicit fragment job without mutating the shared seed set."""
interiors = {
a for a, reason in seed_audit['included_reasons'].items()
if reason == 'DISPATCH_INTERIOR'
}
dispatch_roots = {
a for a, reason in seed_audit['included_reasons'].items()
if reason in ('DISPATCH_ENTRY', 'STATIC_DISPATCH_ENTRY',
'DISPATCH_ROOT')
}
executed = set(seed_audit.get('executed_pcs', set()))
# A final DISPATCH_INTERIOR needs a discovered host, but a guarded isolated
# fragment does not. Do not rebuild live demand solely from surviving
# shared seeds. Retain the existing execution gate: arbitrary seeds,
# static-only interiors, and merely observed instructions are not enough.
observed_dispatch = (set(seed_audit.get('dispatch_fragment_demands', set())) &
executed)
static_exact_demands = set(
seed_audit.get('static_exact_fragment_demands', set()))
hosted_donor_demands = set(
seed_audit.get('cross_variant_hosted_demands', set()))
static_interval_demands = set(
seed_audit.get('static_interval_fragment_demands', set()))
static_demands = static_exact_demands | static_interval_demands
region_hi = phys_addr + size
# --force-interior names a PC, so it forces the view of its own segment.
forced = {
a for a in forced_interiors
if phys_addr <= (a & 0x1FFFFFFF) < region_hi and
segment_of(a) == segment_of(load_addr)
}
if not (((interiors or dispatch_roots) and executed) or observed_dispatch or
static_demands or forced):
return None
return {
'phys_addr': phys_addr,
'load_addr': load_addr,
'size': size,
'data': data,
# Trailing delay-slot guard words in `data` (readable, not
# discoverable). Stated by the capture writer where the field exists,
# reconstructed from the page-run format otherwise.
'guard_bytes': capture_guard_bytes(
capture, size, f'region 0x{load_addr:08X}'),
'candidates': (interiors | dispatch_roots | observed_dispatch |
static_demands | forced),
'executed': executed,
'static_demands': static_demands,
'static_exact_demands': static_exact_demands,
'hosted_donor_demands': hosted_donor_demands,
'included_reasons': {
addr: {reason}
for addr, reason in seed_audit['included_reasons'].items()
},
'static_interval_demands': static_interval_demands,
'forced': forced,
'producer_ranges': tuple(seed_audit['producer_ranges']),
'cross_call_allow': tuple(seed_audit['accepted_cross_producer_calls']),
'resident_cap': ({
'producer': BIOS_RESIDENT_PRODUCER,
'bios_sha256': capture.get('bios_sha256', ''),
'producer_name': capture.get('producer_name', 'BIOS resident code'),
} if capture and capture.get('producer') == BIOS_RESIDENT_PRODUCER
else None),
}
def merge_fragment_jobs_by_recipe(jobs: list[dict],
invocation_recipe=()) -> list[dict]:
"""Union evidence that can safely share one byte-variant fragment recipe.
Repeated captures of identical bytes commonly carry different observed or
static roots. Batching each capture independently would still create many
supplemental DLLs, so merge only jobs whose complete recompiler recipe is
identical. Invocation-wide ABI/flavor/CPS/TOML inputs are constant and are
therefore intentionally not repeated in this key.
"""
grouped = {}
set_fields = (
'candidates', 'executed', 'static_demands',
'static_exact_demands', 'hosted_donor_demands',
'static_interval_demands', 'forced',
)
for job in jobs:
key = (
job['phys_addr'], hashlib.sha256(job['data']).digest(),
job['load_addr'], job['size'],
tuple(sorted(job['producer_ranges'])),
tuple(sorted(set(job['cross_call_allow']))),
json.dumps(job.get('resident_cap'), sort_keys=True),
tuple(invocation_recipe),
)
merged = grouped.get(key)
if merged is None:
merged = dict(job)
for field in set_fields:
merged[field] = set(job[field])
merged['included_reasons'] = {
addr: set(reasons)
for addr, reasons in job.get('included_reasons', {}).items()
}
grouped[key] = merged
continue
for field in set_fields:
merged[field].update(job[field])
for addr, reasons in job.get('included_reasons', {}).items():
merged.setdefault('included_reasons', {}).setdefault(
addr, set()).update(reasons)
if merged.get('resident_cap') is None and job.get('resident_cap'):
merged['resident_cap'] = job['resident_cap']
return [grouped[key] for key in sorted(grouped)]
def _normalized_func_identity(func_id) -> tuple:
entry, code_crc, ranges = func_id
return (_canonical_guest_addr(entry), code_crc & 0xFFFFFFFF,
tuple((_canonical_guest_addr(start), length)
for start, length in ranges))
def _producer_for_job_addr(job: dict, addr: int):
"""Return a stable producer token, treating a simple image as one producer."""
phys = addr & 0x1FFFFFFF
ranges = job.get('producer_ranges', ())
if not ranges:
lo = job['load_addr'] & 0x1FFFFFFF
hi = lo + job['size']
return ('image', lo, hi) if lo <= phys < hi else None
for index, (start, end) in enumerate(ranges):
lo = start & 0x1FFFFFFF
hi = lo + (end - start)
if lo <= phys < hi:
return ('producer', index, lo, hi)
return None
def _preferred_host_reason(reasons) -> str | None:
"""Choose one existing marker deterministically without widening trust."""
priority = (
'DIRECT_JAL_TARGET', 'STATIC_INDIRECT_TARGET',
'STATIC_BRANCH_ROOT', 'STATIC_DISCOVERY_ROOT',
'STATIC_DISPATCH_ENTRY', 'DISPATCH_ENTRY', 'FUNCTION_POINTER_TARGET',
'TOML_DECLARED_ENTRY',
)
reason_set = set(reasons) & HOSTED_OWNER_REASONS
return next((reason for reason in priority if reason in reason_set), None)
def select_hosted_interior_demands(job: dict, donor_recipes: dict,
current_func_ids: list,
audit_out: dict | None = None) -> dict:
"""Map sibling-proven targets to unique CRC-valid local host identities.
This is deliberately fail-closed. Aliases and multi-range functions are
not eligible owners: an owner must have one exact range beginning at its F
entry, carry local root evidence, share the target's producer, and be the
sole distinct identity containing that target.
"""
donor_items = sorted(donor_recipes.items())
nomination_dropped = max(0, len(donor_items) - HOSTED_NOMINATION_CAP)
donor_items = donor_items[:HOSTED_NOMINATION_CAP]
lo = job['load_addr'] & 0x1FFFFFFF
hi = lo + job['size']
local_reasons = {
_canonical_guest_addr(addr): set(reasons)
for addr, reasons in job.get('included_reasons', {}).items()
}
identities = {
_normalized_func_identity(func_id)
for func_id in current_func_ids
}
rejected = Counter()
owner_intervals = []
for identity in identities:
host, _crc, ranges = identity
reason = _preferred_host_reason(local_reasons.get(host, ()))
if reason is None:
rejected['owner_identity_no_local_root'] += 1
continue
if len(ranges) != 1:
rejected['owner_identity_alias_or_multirange'] += 1
continue
range_start, length = ranges[0]
if range_start != host or length <= 0:
rejected['owner_identity_not_rooted_range'] += 1
continue
producer = _producer_for_job_addr(job, host)
if producer is None:
rejected['owner_identity_no_producer'] += 1
continue
range_end = range_start + length
if (range_end & 0x1FFFFFFF) > producer[-1]:
rejected['owner_identity_crosses_producer_end'] += 1
continue
owner_intervals.append(
(range_start, range_end, producer, identity, reason))
owner_intervals.sort(key=lambda item: (item[0], item[1], item[3]))
selected = {}
active_intervals = []
next_interval = 0
for raw_target, donors in donor_items:
target = _canonical_guest_addr(raw_target)
if not donors:
rejected['target_no_donor_recipe'] += 1
continue
target_phys = target & 0x1FFFFFFF
if (target & 3) != 0 or not (lo <= target_phys < hi):
rejected['target_outside'] += 1
continue
target_producer = _producer_for_job_addr(job, target)
if target_producer is None:
rejected['target_no_producer'] += 1
continue
while (next_interval < len(owner_intervals) and
owner_intervals[next_interval][0] < target):
active_intervals.append(owner_intervals[next_interval])
next_interval += 1
active_intervals = [
interval for interval in active_intervals
if interval[1] > target
]
candidates = [
(identity, reason)
for _start, _end, producer, identity, reason in active_intervals
if producer == target_producer
]
# Multiple byte/geometry owners make runtime load order semantically
# relevant. Duplicate copies of the identical identity collapse in
# the set above; any remaining ambiguity is rejected.
if len(candidates) != 1:
rejected['target_no_unique_owner'] += 1
continue
owner_identity, host_reason = candidates[0]
selected[target] = {
'host': owner_identity[0],
'owner_identity': owner_identity,
'host_reason': host_reason,
}
bounded = {}
selected_hosts = set()
for target, spec in sorted(selected.items()):
host = _canonical_guest_addr(spec['host'])
if len(bounded) >= HOSTED_SELECTED_ALIAS_CAP:
rejected['selected_alias_cap_dropped'] += 1
continue
if host not in selected_hosts and len(
selected_hosts) >= HOSTED_SELECTED_HOST_CAP:
rejected['selected_host_cap_dropped'] += 1
continue
bounded[target] = spec
selected_hosts.add(host)
selected = bounded
if nomination_dropped:
rejected['nomination_cap_dropped'] += nomination_dropped
if audit_out is not None:
audit_out.clear()
audit_out.update(rejected)
audit_out['donor_targets'] = len(donor_recipes)
audit_out['current_identities'] = len(identities)
audit_out['selected'] = len(selected)
return selected
def organic_root_entries(func_ids) -> set[int]:
"""Return entries whose identity is exactly one range rooted at entry.
Hosted aliases reuse their owner's range, so their F entry must never gain
sibling-donor authority merely because the alias survived CRC validation.
Requiring the same geometry as hosted-owner selection also rejects a later
island in a multi-range identity from becoming a recursive donor.
"""
roots = set()
for func_id in func_ids:
entry, _crc, ranges = _normalized_func_identity(func_id)
if len(ranges) != 1:
continue
start, length = ranges[0]
if start == entry and length > 0:
roots.add(entry)
return roots
def recipient_sibling_proofs(donor_entries_by_recipe: dict[int, tuple],
recipient_index: int,
authority_phys_entries: set,
limit: int = HOSTED_NOMINATION_CAP
) -> tuple[dict, bool]:
"""Return the fixed first entries proven by another byte recipe.
Each recipe's donor list is sorted once. A heap merge excludes the
recipient before applying the nomination limit. Authority coverage is also
removed before the cap because it is reconstructed deterministically on
every invocation. Supplemental coverage deliberately remains in the
returned window through *all* later selection caps (alias and host as well
as nomination); ``still_needed`` removes it only at compile time. Filtering
it here would let an unchanged rerun page into a host that was dropped by a
later cap. Duplicate addresses collapse.
"""
streams = [
entries for index, entries in sorted(donor_entries_by_recipe.items())
if index != recipient_index and entries
]
if not streams or limit <= 0:
return {}, bool(streams)
nominated = []
previous = None
have_previous = False
for entry in heapq.merge(*streams):
if have_previous and entry == previous:
continue
previous = entry
have_previous = True
if (entry & 0x1FFFFFFF) in authority_phys_entries:
continue
if len(nominated) >= limit:
return ({entry: True for entry in nominated}, True)
nominated.append(entry)
return ({entry: True for entry in nominated}, False)
def select_fragment_orphans(candidates: set[int], executed: set[int],
forced: set[int], static_exact: set[int],
static_interval: set[int],
current_variant_entries: set[int],
current_variant_ranges: list) -> list[int]:
"""Select exact demands that the relevant current artifacts cannot serve."""
def current_region_contains(addr):
phys = addr & 0x1FFFFFFF
return any(start <= phys < end for start, end in current_variant_ranges)
return sorted(
addr for addr in candidates
# Every exact demand is byte-variant-specific. A same-address F from a
# different region variant cannot satisfy it because the runtime's CRC
# guard will reject that identity against the current bytes.
if (addr in forced or
(addr in static_exact and
(addr & 0x1FFFFFFF) not in current_variant_entries) or
(addr in static_interval and not current_region_contains(addr)) or
(addr in executed and
(addr & 0x1FFFFFFF) not in current_variant_entries)))
def select_batchable_strong_roots(static_exact: set[int], executed: set[int],
forced: set[int], static_interval: set[int],
current_variant_entries: set[int]) -> list[int]:
"""Strong exact roots safe to share a supplemental failure domain."""
batchable, _isolated = partition_strong_root_demands(
static_exact, executed, forced, static_interval,
current_variant_entries)
return batchable
def partition_strong_root_demands(static_exact: set[int], executed: set[int],
forced: set[int], static_interval: set[int],
current_variant_entries: set[int]
) -> tuple[list[int], list[int]]:
"""Partition every missing exact root without deferring live overlaps."""
isolated = executed | forced | static_interval
missing = {
entry for entry in static_exact
if (entry & 0x1FFFFFFF) not in current_variant_entries
}
return (sorted(missing - isolated), sorted(missing & isolated))
def compile_batched_fragment_roots(entries, compile_one, on_success,
on_singleton_failure,
still_needed=lambda _entry: True,
should_bisect=lambda _status: True,
on_batch_failure=None) -> int:
"""Compile one root batch, bisecting only when the batch fails closed.
``compile_one`` returns the normal ``(func_ids, status)`` pair. Successful
batches stay grouped; a rejected interaction is recursively partitioned so
one bad root cannot discard good roots. Only singleton failures are exposed
to memo/accounting policy.
"""
roots = sorted(entry for entry in set(entries) if still_needed(entry))
if not roots:
return 0
frag_ids, status = compile_one(roots)
if frag_ids:
on_success(roots, frag_ids, status)
return 1
if len(roots) == 1:
on_singleton_failure(roots[0], status)
return 0
if not should_bisect(status):
if on_batch_failure is not None:
on_batch_failure(roots, status)
else:
for root in roots:
on_singleton_failure(root, status)
return 0
middle = len(roots) // 2
return (compile_batched_fragment_roots(
roots[:middle], compile_one, on_success,
on_singleton_failure, still_needed, should_bisect,
on_batch_failure) +
compile_batched_fragment_roots(
roots[middle:], compile_one, on_success,
on_singleton_failure, still_needed, should_bisect,
on_batch_failure))
def compile_batched_hosted_aliases(hosted: dict, compile_one, on_success,
on_singleton_failure,
still_needed=lambda _entry: True,
should_bisect=lambda _status: True,
on_batch_failure=None,
attempt_cap=HOSTED_COMPILE_ATTEMPT_CAP,
stop_requested=lambda: False) -> int:
"""Compile bounded host groups, splitting hosts before their targets.
Multiple host roots can repartition one another. Exact identity validation
makes that interaction fail closed; host-group-first bisection then removes
the conflicting root without needlessly exploding every target belonging
to otherwise compatible hosts.
"""
by_host = {}
for raw_target, spec in sorted(hosted.items()):
target = _canonical_guest_addr(raw_target)
host = _canonical_guest_addr(spec['host'])
key = (host, _normalized_func_identity(spec['owner_identity']),
spec['host_reason'])
by_host.setdefault(key, []).append(target)
# A group larger than the per-shard alias bound is split deterministically;
# each piece still carries exactly one owner identity.
groups = []
for key, targets in sorted(by_host.items()):
unique_targets = sorted(set(targets))
for offset in range(0, len(unique_targets), HOSTED_BATCH_ALIAS_CAP):
groups.append((key, unique_targets[
offset:offset + HOSTED_BATCH_ALIAS_CAP]))
chunks = []
chunk = []
alias_count = 0
for group in groups:
group_aliases = len(group[1])
if chunk and (len(chunk) >= HOSTED_BATCH_HOST_CAP or
alias_count + group_aliases > HOSTED_BATCH_ALIAS_CAP or
chunk[-1][0][0] == group[0][0]):
chunks.append(chunk)
chunk = []
alias_count = 0
chunk.append(group)
alias_count += group_aliases
if chunk:
chunks.append(chunk)
attempts = 0
def run(group_batch):
nonlocal attempts
if stop_requested():
return 0
filtered = []
for key, targets in group_batch:
needed = [entry for entry in targets if still_needed(entry)]
if needed:
filtered.append((key, needed))
if not filtered:
return 0
specs = {}
for (host, owner_identity, host_reason), targets in filtered:
for target in targets:
specs[target] = {
'host': host,
'owner_identity': owner_identity,
'host_reason': host_reason,
}
entries = sorted(specs)
if attempts >= attempt_cap:
status = (f'hosted-entry-audit: compile attempt cap '
f'{attempt_cap} reached')
if on_batch_failure is not None:
on_batch_failure(entries, status)
return 0
attempts += 1
frag_ids, status = compile_one(specs)
if frag_ids:
served = on_success(entries, frag_ids, status)
if served is None:
return 1
served = ({_canonical_guest_addr(entry) for entry in served} &
set(entries))
if not served:
status = ('hosted-entry-audit: successful batch reported no '
'requested targets')
if on_batch_failure is not None:
on_batch_failure(entries, status)
else:
for entry in entries:
on_singleton_failure(entry, status)
return 1
missing_groups = []
for key, targets in filtered:
missing = [entry for entry in targets if entry not in served]
if missing:
missing_groups.append((key, missing))
if not missing_groups:
return 1
# The recompiler may prove only a subset of a multi-target hosted
# request. Retry the deterministic remainder now; otherwise a later
# invocation would shrink the batch and accidentally page into new
# successes. Every successful retry removes at least one target and
# the global attempt cap still bounds adversarial inputs.
return 1 + run(missing_groups)
if len(filtered) > 1 and should_bisect(status):
middle = len(filtered) // 2
return run(filtered[:middle]) + run(filtered[middle:])
key, targets = filtered[0]
if len(targets) > 1 and should_bisect(status):
middle = len(targets) // 2
return (run([(key, targets[:middle])]) +
run([(key, targets[middle:])]))
if len(targets) == 1:
on_singleton_failure(targets[0], status)
elif on_batch_failure is not None:
on_batch_failure(targets, status)
else:
for target in targets:
on_singleton_failure(target, status)
return 0
built = 0
for chunk_groups in chunks:
if stop_requested():
break
built += run(chunk_groups)
return built
def hosted_entries_still_missing(hosted: dict, current_phys_entries,
rejected_entries=()) -> list[int]:
"""Return nominated aliases that still need capacity accounting."""
covered = {int(entry) & 0x1FFFFFFF for entry in current_phys_entries}
rejected = {_canonical_guest_addr(entry) for entry in rejected_entries}
return sorted({
_canonical_guest_addr(entry)
for entry in hosted
if ((int(entry) & 0x1FFFFFFF) not in covered and
_canonical_guest_addr(entry) not in rejected)
})
def fragment_capacity_allows_entry(saturated: bool, repair_all: bool,
forced_entries, entry: int) -> bool:
"""Whether capacity policy permits one fragment compile attempt.
A saturated namespace cannot admit ordinary growth. ``--force`` still
has to reach the transactional publisher because replacing the same shard
can be capacity-neutral. ``--force-interior`` grants that exception only
to the explicitly named entry.
"""
return (not saturated or repair_all or
_canonical_guest_addr(entry) in {
_canonical_guest_addr(forced)
for forced in forced_entries
})
def fragment_capacity_fastpath_eligible(saturated: bool, repair_all: bool,
jobs) -> bool:
"""Whether an already-full invocation may bypass fragment owner analysis."""
return (saturated and not repair_all and
not any(job.get('forced') for job in jobs))
def fragment_batch_failure_is_partitionable(reason: str) -> bool:
"""Whether retrying smaller root sets can change this failure verdict."""
if reason.startswith('candidate-capacity: full'):
return False
if unsupported_guest_branch_rejection(reason):
return True
return reason.startswith((
'generated-c-audit:', 'requested-entry-audit:',
'hosted-entry-audit:',
'candidate-capacity:',
'no-generated-output', 'no-func-ids',
'fragment cache-key collision',
))
def compile_fragment_batch(requested_entries, data: bytes, load_addr: int,
size: int, phys_addr: int, cache_dir: str,
args, sub_env: dict, toml_doc: dict,
producer_ranges=(), cross_call_allow=(),
hosted_owners: dict | None = None,
manifest_provenance: str | None = None,
*, guard_bytes: int):
"""Compile an ISOLATED interior-entry 'island' fragment that ENTERS at an
executed orphan DISPATCH_INTERIOR PC (a host that static analysis never
discovered, e.g. an FMV driver reached via a computed jump) and covers the
recompiler's worklist-discovered reachable CFG from that PC — a single
`dispatch_root` seed; out-of-island branches become dispatcher exits.
Emitted as its OWN <region>_<key>.dll so a fragment that fails the
generated-C audit is dropped ALONE and never poisons the region's trusted DLL
(separate failure domain — the invariant the earlier host-recovery attempt
violated). Safe because: dispatch_root is exempt from the boundary re-check
that would reject a mid-function start; the recompiler translates mid-function
code literally (regs from CPU state, no synthesized stack frame — the caller
already set up the frame); and the loader's per-function live-RAM CRC gate
rejects any fragment whose bytes don't match. Returns (frag_ids, status):
status 'built' or 'cached' on success (frag_ids is the func-id list), or
(None, reason) on failure/skip so the caller can tally WHY a fragment
dropped instead of the old silent None."""
requested = sorted({
(entry & PHYS_MASK) | segment_of(load_addr)
for entry in requested_entries
})
if not requested:
return None, 'no-requested-entries'
if hosted_owners is not None:
hosted_specs = {
_canonical_guest_addr(entry): spec
for entry, spec in hosted_owners.items()
if _canonical_guest_addr(entry) in requested
}
if set(hosted_specs) != set(requested):
return None, 'hosted-entry-audit: target/spec set mismatch'
hosts = sorted({_canonical_guest_addr(spec['host'])
for spec in hosted_specs.values()})
if set(hosts) & set(requested):
return None, 'hosted-entry-audit: target is also a host'
first_entry = hosts[0]
else:
hosted_specs = None
first_entry = requested[0]
with tempfile.TemporaryDirectory() as tmp:
psx = os.path.join(tmp, 'frag.psx')
with open(psx, 'wb') as f:
f.write(make_psxexe(load_addr, first_entry, data,
guard_bytes=guard_bytes))
seeds_path = os.path.join(tmp, 'seeds.txt')
with open(seeds_path, 'w') as f:
for range_lo, range_hi in producer_ranges:
f.write(f'producer_range 0x{range_lo:08X} 0x{range_hi:08X}\n')
for target in sorted(set(cross_call_allow)):
f.write(f'cross_call_allow 0x{target:08X}\n')
if hosted_specs is None:
for entry in requested:
f.write(f'dispatch_root 0x{entry:08X}\n')
else:
host_markers = {}
for entry, spec in sorted(hosted_specs.items()):
host = _canonical_guest_addr(spec['host'])
marker = seed_line_for_reason(host, spec['host_reason'])
prior = host_markers.setdefault(host, marker)
if prior != marker:
return None, (f'hosted-entry-audit: host '
f'0x{host:08X} has conflicting markers')
for host, marker in sorted(host_markers.items()):
f.write(marker + '\n')
for entry in requested:
host = _canonical_guest_addr(hosted_specs[entry]['host'])
f.write(f'hosted_interior 0x{entry:08X} '
f'0x{host:08X}\n')
out_dir_tmp = os.path.join(tmp, 'out')
os.makedirs(out_dir_tmp)
cmd = [args.recompiler, psx, '--seeds', seeds_path,
'--out-dir', out_dir_tmp, '--overlay',
'--ws-config', os.path.abspath(args.game_toml)]
cmd += recompiler_project_root_args(args)
r = subprocess.run(cmd, capture_output=True, text=True,
cwd=os.path.dirname(os.path.abspath(args.game_toml)),
env=sub_env)
if r.returncode != 0:
return None, f'recompiler-error: {(r.stderr or r.stdout or "").strip()}'
full_c = ranges_src = None
for fn in os.listdir(out_dir_tmp):
if fn.endswith('_full.c'):
full_c = os.path.join(out_dir_tmp, fn)
elif fn.endswith('_full.ranges'):
ranges_src = os.path.join(out_dir_tmp, fn)
if not full_c or not ranges_src:
return None, 'no-generated-output (_full.c/_full.ranges missing)'
with open(full_c) as f:
src = patch_generated_c(f.read(), load_addr, size)
c_audit = audit_generated_c(src, load_addr, size,
binascii.crc32(data) & 0xFFFFFFFF, toml_doc)
if c_audit['unknown_bad'] or c_audit['unsupported_todo_addrs']:
# RETAIN the failed fragment's C for triage. Region shards already
# keep their _patched.c on audit failure; interior fragments used to
# return here BEFORE the (success-only) retain below, so the exact
# shards you most want to inspect — WHY did this orphan interior fail
# to lower — vanished with the temp dir. Write it out with a header
# summarizing the audit verdict so a failure is drillable, not just
# counted. (compile_overlays.py is not in the codegen hash: no reshard.)
try:
os.makedirs(cache_dir, exist_ok=True)
ub = ', '.join(f'0x{a:08X}' for a in sorted(c_audit['unknown_bad']))
us = ', '.join(f'0x{a:08X}'
for a in sorted(c_audit['unsupported_todo_addrs']))
header = (
f'/* FRAGMENT AUDIT FAILED — retained for triage (not compiled).\n'
f' * requested ({len(requested)}): '
f'{", ".join(f"0x{entry:08X}" for entry in requested)}\n'
f' * region phys : 0x{phys_addr:08X}\n'
f' * unknown_bad ({len(c_audit["unknown_bad"])}): {ub}\n'
f' * unsupported ({len(c_audit["unsupported_todo_addrs"])}): {us}\n'
f' */\n')
request_key = binascii.crc32(b''.join(
struct.pack('<I', entry) for entry in requested)) & 0xFFFFFFFF
failed_c = os.path.join(
cache_dir,
f'{phys_addr:08X}_{request_key:08X}_fragment_FAILED.c')
with open(failed_c, 'w') as f:
f.write(header + src)
except OSError:
pass # retention is best-effort; never mask the real failure
return None, (f'generated-c-audit: '
f'{len(c_audit["unknown_bad"])} unknown_bad, '
f'{len(c_audit["unsupported_todo_addrs"])} unsupported')
frag_ids = parse_overlay_func_ids(ranges_src, data, load_addr, size)
if not frag_ids:
return None, 'no-func-ids (empty ranges manifest)'
delay_errors = audit_func_id_delay_slots(frag_ids, data, load_addr)
if delay_errors:
entry, pc, reason = delay_errors[0]
where = f' at 0x{pc:08X}' if pc is not None else ''
return None, (f'delay-slot-identity: func 0x{entry:08X}'
f'{where}: {reason}')
if hosted_specs is None:
identity_error = validate_fragment_requested_ids(
set(requested), frag_ids, c_audit['defs'])
identity_error_prefix = 'requested-entry-audit'
else:
identity_error = validate_hosted_fragment_ids(
set(requested), hosted_specs, frag_ids, c_audit['defs'],
allow_missing_targets=True)
identity_error_prefix = 'hosted-entry-audit'
if identity_error:
return None, f'{identity_error_prefix}: {identity_error}'
if hosted_specs is not None:
served = hosted_fragment_served_targets(
set(requested), frag_ids, c_audit['defs'])
if not served:
return None, ('hosted-no-target: no requested target passed '
'the recipient block-leader proof')
if hosted_specs is not None:
if manifest_provenance not in (
None, HOSTED_MANIFEST_PROVENANCE):
return None, ('hosted-entry-audit: conflicting manifest '
'provenance')
manifest_provenance = HOSTED_MANIFEST_PROVENANCE
pair_id = overlay_pair_id(src, frag_ids, manifest_provenance)
src = add_overlay_pair_export(src, pair_id)
# Key the fragment DLL by its func-identity SET (dedup like a region
# bundle); the loader keys DLLs by the region_start filename prefix and
# content-matches each function, so a fragment is just another DLL for
# this region_start.
key = fragment_shard_key(frag_ids, manifest_provenance)
dll_path = os.path.join(cache_dir, f'{phys_addr:08X}_{key:08X}{overlay_ext()}')
if not args.force:
cache_status = cached_shard_manifest_status(
dll_path, overlay_abi_tag(args.runtime_include, args.flavor),
overlay_ranges_text(
frag_ids, pair_id, manifest_provenance), pair_id)
if cache_status == 'match':
return frag_ids, 'cached' # exact identity already built
if cache_status == 'mismatch':
return None, ('fragment cache-key collision/stale pair: existing '
'manifest does not match generated identity')
patched_c = os.path.join(tmp, 'frag_patched.c')
with open(patched_c, 'w') as f:
f.write(src)
# Keep the exact generated fragment beside other retained overlay
# sources. Orphan interiors are the hardest shards to audit when a
# native/interpreter differential finds a timing or device mismatch;
# deleting their only C representation with the temp directory made
# the responsible lowering impossible to inspect after compilation.
retained_c = os.path.join(cache_dir, f'{key:08X}_fragment_patched.c')
with open(retained_c, 'w') as f:
f.write(src)
include_dirs = [args.runtime_include]
recomp_root = os.path.dirname(os.path.dirname(args.recompiler))
p = os.path.join(recomp_root, 'lib/fmt/include')
if os.path.isdir(p):
include_dirs.append(p)
# Honor the SAME compiler the region path uses (args.compiler / args.tcc).
# Previously this hardcoded gcc, so on a tcc-only player box (no gcc on
# PATH) EVERY interior-fragment shard silently failed to build — the
# FMV-driver / orphan-interior class ran interpreted forever. The cache
# dir already namespaces by args.compiler, so only the invocation was wrong.
publication = {}
if not compile_dll(patched_c, dll_path, include_dirs,
gcc=args.gcc, flavor=args.flavor,
compiler=args.compiler, tcc=args.tcc,
func_ids=frag_ids, pair_id=pair_id,
preserve_existing_pair=not args.force,
expected_existing_abi=overlay_abi_tag(
args.runtime_include, args.flavor),
publication_result=publication,
candidate_cap=overlay_candidate_cap(
args.runtime_include),
manifest_provenance=manifest_provenance):
if publication.get('capacity_error'):
capacity_error = publication['capacity_error']
match = re.search(
r'(\d+) existing -> \d+ projected .* > (\d+);',
capacity_error)
if match and int(match.group(1)) >= int(match.group(2)):
return None, 'candidate-capacity: full; ' + capacity_error
return None, 'candidate-capacity: ' + capacity_error
return None, 'compile-error (see COMPILE ERROR above)'
if not publication.get('published', True):
# Another process won the content-key publication race. It is a
# cache hit only when the complete pair is byte-for-byte the same
# manifest identity; a 32-bit filename-key collision must fail
# closed instead of warming coverage from our discarded staging.
cache_status = cached_shard_manifest_status(
dll_path, overlay_abi_tag(args.runtime_include, args.flavor),
overlay_ranges_text(
frag_ids, pair_id, manifest_provenance), pair_id)
if cache_status != 'match':
return None, ('fragment cache-key collision/concurrent pair: '
'preserved manifest does not match generated identity')
return frag_ids, 'cached'
return frag_ids, 'built'
def compile_interior_fragment(interior: int, data: bytes, load_addr: int,
size: int, phys_addr: int, cache_dir: str,
args, sub_env: dict, toml_doc: dict,
producer_ranges=(), cross_call_allow=(),
*, guard_bytes: int):
"""Compile one isolated uncertain interior entry.
Executed, operator-forced, and interval-only aliases deliberately use this
singleton wrapper so one speculative entry cannot poison trusted roots.
Play-free strong roots use ``compile_fragment_batch`` instead.
"""
return compile_fragment_batch(
{interior}, data, load_addr, size, phys_addr, cache_dir, args, sub_env,
toml_doc, producer_ranges, cross_call_allow,
manifest_provenance=ORPHAN_MANIFEST_PROVENANCE,
guard_bytes=guard_bytes)
def fragment_shard_key(func_ids: list,
provenance: str | None = None) -> int:
"""Filename identity shared by fragment publication and sidecars."""
crc = 0
if provenance is not None:
crc = binascii.crc32(
b'psxrecomp fragment provenance\0' +
provenance.encode('ascii') + b'\0')
return binascii.crc32(b''.join(
struct.pack('<II', entry, code_crc)
for entry, code_crc, _ranges in sorted(func_ids)), crc) & 0xFFFFFFFF
def _toolchain_env(gcc: str):
"""Build an environment that guarantees gcc's toolchain dir is on PATH.
A mingw gcc invoked by absolute path (as the runtime's cmd.exe autocompile
spawn does, and as any launch without mingw64/bin on PATH does) fails
SILENTLY — exit 1, empty stderr — because cc1/as/ld/collect2 and the runtime
DLLs (libwinpthread-1, libgcc_s_seh-1, libstdc++-6) are resolved via PATH and
can't be found, so gcc never actually compiles. Auto-detect the toolchain
bin dir and prepend it. Detection order: $PSX_MINGW_BIN override, the dir of
the given --gcc path, `which gcc`, then common install locations. Returns
(env, resolved_gcc_dir_or_None)."""
import shutil
env = os.environ.copy()
cands = []
if os.environ.get('PSX_MINGW_BIN'):
cands.append(os.environ['PSX_MINGW_BIN'])
if ('/' in gcc) or ('\\' in gcc):
cands.append(os.path.dirname(os.path.abspath(gcc)))
w = shutil.which(gcc)
if w:
cands.append(os.path.dirname(w))
cands += [r'C:\msys64\mingw64\bin', r'C:\mingw64\bin', '/usr/bin']
for d in cands:
if d and (os.path.isfile(os.path.join(d, 'gcc.exe')) or
os.path.isfile(os.path.join(d, 'gcc'))):
# Prepend in the RUNNING INTERPRETER'S path flavor. Under an
# MSYS-flavored python (os.sep == '/'), PATH is a ':'-separated
# POSIX list; splicing a Windows 'C:/...' entry in (its drive
# colon reads as a separator) mangles the whole child PATH, so
# cc1/as/ld/collect2 lose their DLLs and gcc dies exit-1 with
# EMPTY stderr — the 2026-07-15 in-game shard-compile root cause
# (runtime cmd.exe /C python resolved to devkitPro's MSYS python).
# Convert X:[/\]... -> /x/... before prepending; verified live:
# POSIX-prepend rc 0, Windows-prepend rc 1.
pd = d
if os.sep == '/':
m = re.match(r'^([A-Za-z]):[\\/](.*)$', d)
if m:
pd = '/' + m.group(1).lower() + '/' + m.group(2).replace('\\', '/')
env['PATH'] = pd + os.pathsep + env.get('PATH', '')
return env, d
return env, None
# ---- TinyCC (tcc) overlay compile — toolchain-free user fallback -----------
# tcc has its own built-in linker (no ld/collect2) and bundles its own headers,
# so it is self-contained. The only friction is that tcc 0.9.27 does not skip a
# UTF-8 BOM, and the recompiler's runtime headers carry one — so we feed tcc a
# BOM-stripped copy of the include dirs (used ONLY for tcc's -I; the codegen hash
# still derives from the real headers, so the cache dir matches the runtime).
_TCC_BOMFREE_INC = {} # orig include dir -> bom-stripped temp dir (memoized)
_TCC_COMPAT_PREFIX = b'''\
#ifdef __TINYC__
/* TinyCC 0.9.27 has no GCC __builtin_ctz. Keep this shim local to the
* disposable generated source so runtime headers and the codegen hash stay
* byte-identical across compilers. */
static inline unsigned psx_tcc_ctz(unsigned value) {
unsigned bit = 0u;
while (((value >> bit) & 1u) == 0u) ++bit;
return bit;
}
#define __builtin_ctz(value) psx_tcc_ctz((unsigned)(value))
#endif
'''
def _bom_free_incdir(d: str) -> str:
d = os.path.abspath(d)
if d in _TCC_BOMFREE_INC:
return _TCC_BOMFREE_INC[d]
out = tempfile.mkdtemp(prefix='tcc_inc_')
for name in os.listdir(d):
# Headers, and the .c.inc fragments a BIOS module's glue.c includes by
# name (overlay_dispatch_preamble.c.inc, psx_bios_module_glue.c.inc).
# tcc sees only this copy, so a file left out here is "not found".
if not name.endswith(('.h', '.inc')):
continue
with open(os.path.join(d, name), 'rb') as f:
data = f.read()
if data[:3] == b'\xef\xbb\xbf':
data = data[3:]
with open(os.path.join(out, name), 'wb') as f:
f.write(data)
_TCC_BOMFREE_INC[d] = out
return out
def _tcc_bundle_args(tcc: str) -> list:
"""The Linux release bundle (tools/release_stage.py stage_linux_tcc) is a
source-built tcc whose compiled-in paths point at the packaging machine:
point it at its own directory (-B: tcc's headers and libtcc1.a), give it
the bundled musl libc headers, and link no crt/libc of the player's (a
SteamOS player has none): libc symbols resolve against the game's glibc
when the shard is loaded. Any other tcc (Windows bundle, a system tcc)
gets no extra arguments."""
d = os.path.dirname(os.path.abspath(tcc))
libc_inc = os.path.join(d, 'libc-include')
libtcc1 = os.path.join(d, 'libtcc1.a')
if not (os.path.isdir(libc_inc) and os.path.isfile(libtcc1)):
return []
# -nostdinc: only the bundle's headers (tcc's, then musl's); a Linux box
# with libc6-dev installed must not change what a shard compiles against.
return ['-B' + d, '-nostdinc', '-nostdlib', '-isystem', os.path.join(d, 'include'),
'-isystem', libc_inc, libtcc1]
def _compile_dll_tcc(c_path: str, out_dll: str, include_dirs, flavor: int,
tcc: str) -> bool:
# Strip a UTF-8 BOM off the overlay C itself (tcc 0.9.27 chokes on it) and
# provide the one GCC builtin used by the shared cycle-model header.
with open(c_path, 'rb') as f:
data = f.read()
if data[:3] == b'\xef\xbb\xbf':
data = data[3:]
if not data.startswith(_TCC_COMPAT_PREFIX):
data = _TCC_COMPAT_PREFIX + data
with open(c_path, 'wb') as f:
f.write(data)
cmd = [tcc, '-shared',
'-DPSX_OVERLAY_DLL_BUILD',
# Keep TCC semantically identical to the GCC overlay path. Omitting
# these left debug_server_cyc_observe unresolved in release shards
# and disabled guest-cycle charging in every successfully linked
# TCC shard.
'-DPSX_NO_DEBUG_TOOLS',
'-DPSX_ENABLE_BLOCK_CYCLES=1',
f'-DPSX_OVERLAY_FLAVOR={int(flavor)}',
# PGXP flavor bit (overlay_api.h PSX_OVERLAY_FLAVOR_PGXP): arm the
# PGXP_*() hook macros the emitter writes into every overlay C.
*(['-DPSX_PGXP=1'] if int(flavor) & 2 else []),
native_path(c_path), '-o', native_path(out_dll)]
for d in include_dirs:
cmd.append('-I' + native_path(_bom_free_incdir(d)))
cmd += _tcc_bundle_args(tcc)
print(f' compile (tcc): {" ".join(cmd)}')
r = subprocess.run(cmd, capture_output=True, text=True)
if r.returncode != 0:
print(f' TCC COMPILE ERROR (exit {r.returncode}):\n{r.stderr or r.stdout}')
return False
return True
def _compile_dll_direct(c_path: str, out_dll: str, include_dirs: list[str],
gcc: str = 'gcc', flavor: int = 0,
compiler: str = 'gcc', tcc: str = 'tcc') -> bool:
import platform
# Absolute paths (interpreter flavor) for EVERY path the native compiler
# touches — see native_path's docstring for the two rules and the
# 2026-07-15 MSYS-python incident they encode.
c_path = native_path(c_path)
out_dll = native_path(out_dll)
if compiler == 'tcc':
return _compile_dll_tcc(c_path, out_dll, include_dirs, flavor, tcc)
env, tc_dir = _toolchain_env(gcc)
includes = [f'-I{native_path(d)}' for d in include_dirs]
# On Windows, DLLs use PE relocations — -fPIC triggers GCC CRT init
# that conflicts with the host process. Use -shared without -fPIC.
pic_flag = [] if is_windows() else ['-fPIC']
cmd = [
gcc, '-shared', *pic_flag, *target_arch_flags('gcc'), '-O2',
'-DPSX_OVERLAY_DLL_BUILD',
# Overlays mirror the runtime's no-debug-tools build: the emitter guards
# debug_server_cyc_observe (and friends) behind PSX_NO_DEBUG_TOOLS, and the
# shipped/native runtime is built without debug tools, so define it here too or
# the overlay emits calls to symbols the runtime doesn't have (undefined ref).
'-DPSX_NO_DEBUG_TOOLS',
# CYCLE MODEL UNIFICATION (Tomba2 logo Timer1 fork): compiled-overlay code
# MUST charge guest cycles exactly like the dirty-RAM interpreter and the
# BIOS (both built with PSX_ENABLE_BLOCK_CYCLES=1). Without this flag every
# psx_advance_cycles() in overlay code is #ifdef'd out, so a function run as
# a native overlay charges ~0 cycles while the same function run via the
# interp charges per-instruction -> timer-sensitive code (e.g. a Timer1
# debounce) reads different values per backend and the game forks.
'-DPSX_ENABLE_BLOCK_CYCLES=1',
# Codegen-flavor tag baked into overlay_abi() (base=0). The loader
# rejects DLLs whose flavor differs from the runtime's, so a widescreen
# cache and a base cache can never cross-contaminate even if they share
# a directory (they key by guest-bytes CRC, which is flavor-blind).
f'-DPSX_OVERLAY_FLAVOR={int(flavor)}',
# PGXP flavor bit (overlay_api.h PSX_OVERLAY_FLAVOR_PGXP): arm the
# PGXP_*() hook macros the emitter writes into every overlay C.
*(['-DPSX_PGXP=1'] if int(flavor) & 2 else []),
c_path,
'-o', out_dll,
*includes,
'-lm',
]
if tc_dir is None:
print(' WARNING: no gcc toolchain dir found (PSX_MINGW_BIN / --gcc dir / '
'PATH / common locations); compile will likely fail silently.')
print(f' compile: {" ".join(cmd)}')
r = subprocess.run(cmd, capture_output=True, text=True, env=env)
if r.returncode != 0:
msg = (r.stderr or r.stdout or '').strip()
if not msg:
# No compiler output at all = the compiler never really ran
# (spawn/DLL-load failure: foreign-MSYS runtime mix, bad toolchain
# dir, or non-native paths). Say so instead of printing nothing —
# an empty COMPILE ERROR is undiagnosable from the runtime's tail.
msg = ('(no compiler output — gcc likely failed to launch: '
f'toolchain dir={tc_dir!r}, check PATH/DLL mix and that '
'all paths above are OS-native)')
print(f' COMPILE ERROR (exit {r.returncode}):\n{msg}')
return False
return True
def compile_dll(c_path: str, out_dll: str, include_dirs: list[str],
gcc: str = 'gcc', flavor: int = 0,
compiler: str = 'gcc', tcc: str = 'tcc',
func_ids: list | None = None,
pair_id: int | None = None,
preserve_existing_pair: bool = False,
expected_existing_abi: int | None = None,
publication_result: dict | None = None,
candidate_cap: int | None = None,
manifest_provenance: str | None = None) -> bool:
"""Publish a shard only after all of its artifacts are complete.
GCC and tcc write their output incrementally. If the process is interrupted
while targeting the final cache name, the next run sees that partial file
and used to skip it as if it were complete. Compile to a unique sibling.
When ``func_ids`` is supplied, stage the range manifest too and publish the
two as one recoverable transaction. Moving the old DLL out of its loader-
visible name first means every interruption point contains either a complete
old pair, a complete new pair, or no loadable pair -- never mixed halves.
The pair ID additionally lets a pair-aware loader reject the only filesystem
TOCTOU left: an old manifest parsed immediately before the DLL rename.
"""
final_out = os.path.abspath(out_dll)
out_dir = os.path.dirname(final_out)
os.makedirs(out_dir, exist_ok=True)
if func_ids is not None and (not func_ids or pair_id is None):
print(' COMPILE ERROR: paired shard requires non-empty ranges and pair ID')
return False
if func_ids is not None and candidate_cap is not None:
try:
capacity_error = preflight_shard_candidate_capacity(
final_out, func_ids, pair_id, manifest_provenance,
expected_existing_abi, candidate_cap)
except Exception as exc:
# A failed optimization must not replace the authoritative locked
# projection after linking. Compile normally and let publication
# either admit or reject the staged pair.
print(f' capacity preflight unavailable: {exc}')
capacity_error = None
if capacity_error is not None:
if publication_result is not None:
publication_result['capacity_error'] = capacity_error
print(f' COMPILE SKIP (preflight): {capacity_error}')
return False
suffix = os.path.splitext(final_out)[1] or overlay_ext()
fd, staged = tempfile.mkstemp(
prefix=f'.{os.path.basename(final_out)}.tmp.', suffix=suffix,
dir=out_dir)
os.close(fd)
os.unlink(staged) # let the native compiler create its output normally
staged_ranges = os.path.splitext(staged)[0] + '.ranges'
try:
if not _compile_dll_direct(
c_path, staged, include_dirs, gcc=gcc, flavor=flavor,
compiler=compiler, tcc=tcc):
return False
if not os.path.isfile(staged) or os.path.getsize(staged) == 0:
print(' COMPILE ERROR: compiler reported success but emitted no DLL')
return False
if func_ids is None:
os.replace(staged, final_out)
if publication_result is not None:
publication_result['published'] = True
return True
write_overlay_ranges_from(
func_ids, staged_ranges, pair_id, manifest_provenance)
if not os.path.isfile(staged_ranges) or os.path.getsize(staged_ranges) == 0:
print(' COMPILE ERROR: range manifest staging produced no output')
return False
try:
published = publish_shard_pair(
staged, staged_ranges, final_out,
preserve_existing=preserve_existing_pair,
expected_existing_abi=expected_existing_abi,
candidate_cap=candidate_cap)
if publication_result is not None:
publication_result['published'] = published
except ShardCandidateCapacityError as exc:
if publication_result is not None:
publication_result['capacity_error'] = str(exc)
print(f' COMPILE SKIP: {exc}')
return False
except Exception as exc:
print(f' COMPILE ERROR: could not publish DLL/ranges pair: {exc}')
try:
recover_shard_pair(final_out)
except Exception as recover_exc:
print(f' COMPILE ERROR: shard-pair recovery also failed: '
f'{recover_exc}')
return False
return True
finally:
# TinyCC emits an export-definition sidecar beside its requested DLL.
# It inherits the unique staged basename and must not accumulate in the
# content cache after either success or failure.
staged_sidecars = [os.path.splitext(staged)[0] + '.def']
for temporary in [staged, staged_ranges, *staged_sidecars]:
try:
os.unlink(temporary)
except FileNotFoundError:
pass
def _shard_pair_paths(dll_path: str) -> tuple[str, str, str, str]:
dll_path = os.path.abspath(dll_path)
ranges_path = os.path.splitext(dll_path)[0] + '.ranges'
return (dll_path, ranges_path,
dll_path + '.pair-lock', dll_path + '.pair-txn.json')
def _unlink_if_present(path: str) -> None:
try:
os.unlink(path)
except FileNotFoundError:
pass
def _best_effort_unlink(path: str) -> None:
"""Remove transaction debris when possible (loaded Windows DLLs may refuse)."""
try:
os.unlink(path)
except OSError:
pass
def _file_sha256(path: str) -> str | None:
try:
digest = hashlib.sha256()
with open(path, 'rb') as source:
for chunk in iter(lambda: source.read(1024 * 1024), b''):
digest.update(chunk)
return digest.hexdigest()
except OSError:
return None
def _write_json_atomic(path: str, value: dict) -> None:
directory = os.path.dirname(path)
fd, staged = tempfile.mkstemp(prefix=f'.{os.path.basename(path)}.',
suffix='.tmp', dir=directory)
try:
with os.fdopen(fd, 'w', encoding='utf-8') as out:
json.dump(value, out, sort_keys=True)
out.flush()
os.fsync(out.fileno())
os.replace(staged, path)
finally:
_best_effort_unlink(staged)
@contextmanager
def _exclusive_file_lock(lock_path: str, timeout: float | None = 120.0):
"""Kernel-owned cross-process lock on one permanent byte-range file.
The lock file is intentionally permanent. OS byte-range/advisory locks are
released by the kernel when a process dies; deleting the file would create
an inode/handle split that permits two simultaneous owners.
"""
lock_path = os.path.abspath(lock_path)
os.makedirs(os.path.dirname(lock_path), exist_ok=True)
lock_file = open(lock_path, 'a+b')
if os.path.getsize(lock_path) == 0:
lock_file.write(b'\0')
lock_file.flush()
deadline = (time.monotonic() + timeout if timeout is not None else None)
acquired = False
try:
while not acquired:
try:
lock_file.seek(0)
if os.name == 'nt':
import msvcrt
msvcrt.locking(lock_file.fileno(), msvcrt.LK_NBLCK, 1)
else:
import fcntl
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX |
fcntl.LOCK_NB)
acquired = True
except OSError:
if deadline is not None and time.monotonic() >= deadline:
raise TimeoutError(
f'timed out waiting for shard-pair lock {lock_path}')
time.sleep(0.025)
yield
finally:
if acquired:
lock_file.seek(0)
if os.name == 'nt':
import msvcrt
msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
lock_file.close()
@contextmanager
def _shard_pair_lock(dll_path: str, timeout: float = 120.0):
"""Cross-process lock for one canonical DLL/ranges filename pair."""
_dll, _ranges, lock_path, _journal = _shard_pair_paths(dll_path)
with _exclusive_file_lock(lock_path, timeout):
yield
def _candidate_capacity_namespace(dll_path: str) -> tuple[str, list[str]]:
"""Return the shared lock and compiler-tier dirs for one cache namespace.
Normal caches are ``GAME/compiler/arch/cgN_hash_gcCONFIG`` (legacy tests
may still use ``cgN_hash``). The runtime candidate table is process-global,
so serialize/count every compiler tier with the same GAME/arch/cg leaf.
Tests and hand-built paths fall back to their containing directory.
The KUSEG and KSEG1 shards of a leaf live in its seg-kuseg/ and seg-kseg1/
subdirectories (§5.7) and take slots from the same process-global table,
so a shard there belongs to its leaf's namespace, and every namespace
lists each leaf's segment directories after the leaf.
"""
leaf = os.path.dirname(os.path.abspath(dll_path))
if cache_dir_segment(leaf) != KSEG0:
leaf = os.path.dirname(leaf)
cg_name = os.path.basename(leaf)
arch_dir = os.path.dirname(leaf)
compiler_dir = os.path.dirname(arch_dir)
game_dir = os.path.dirname(compiler_dir)
compiler_name = os.path.basename(compiler_dir)
if (re.fullmatch(r'cg\d+_[0-9A-Fa-f]{8}(?:_gc[0-9A-Fa-f]{8})?', cg_name) and
compiler_name in ('gcc', 'tcc')):
arch_name = os.path.basename(arch_dir)
tier_dirs = [
directory
for tier in ('gcc', 'tcc')
for directory in _with_segment_dirs(
os.path.join(game_dir, tier, arch_name, cg_name))
]
return (os.path.join(game_dir, '.overlay-candidate-capacity.lock'),
tier_dirs)
return (os.path.join(leaf, '.overlay-candidate-capacity.lock'),
_with_segment_dirs(leaf))
def _with_segment_dirs(leaf: str) -> list[str]:
"""A cache leaf, then its per-segment subdirectories (§5.7)."""
return [leaf] + [os.path.join(leaf, SEGMENT_CACHE_SUBDIRS[seg])
for seg in (KUSEG, KSEG1)]
def _segment_leaf(directory: str) -> str:
"""The cache leaf a (possibly per-segment) shard directory belongs to."""
directory = os.path.abspath(directory)
if cache_dir_segment(directory) != KSEG0:
return os.path.dirname(directory)
return directory
def _segment_basename(path: str) -> tuple[int, str]:
"""A shard's identity within one compiler tier: its segment and name.
The same {phys}_{crc} name is a different shard in each segment."""
return (cache_dir_segment(os.path.dirname(os.path.abspath(path))),
os.path.normcase(os.path.basename(path)))
def cache_candidate_capacity_snapshot(
dll_path: str, expected_abi: int | None) -> tuple[int, dict]:
"""Read one recovered candidate namespace under its publication lock."""
capacity_lock, cache_dirs = _candidate_capacity_namespace(dll_path)
with _exclusive_file_lock(capacity_lock):
recover_candidate_namespace(cache_dirs)
return cache_candidate_inventory(cache_dirs, expected_abi)
def cache_candidate_capacity_full(dll_path: str, expected_abi: int | None,
candidate_cap: int) -> tuple[bool, int]:
"""Read one serialized namespace inventory and report a saturated cache."""
total, counts = cache_candidate_capacity_snapshot(dll_path, expected_abi)
return (total >= candidate_cap or
getattr(counts, 'raw_total', sum(counts.values())) >=
2 * candidate_cap or
getattr(counts, 'range_link_total', 0) >=
2 * candidate_cap * 8 or
getattr(counts, 'file_total', len(counts)) >=
RUNTIME_CACHE_FILE_CAP), total
def candidate_capacity_saturated_in_tier(total: int, counts: dict,
candidate_cap: int,
cache_dir: str) -> bool:
"""Whether one active tier has reached any physical/runtime capacity."""
active = os.path.normcase(_segment_leaf(cache_dir))
saturated = (total >= candidate_cap or
getattr(counts, 'raw_total', sum(counts.values())) >=
2 * candidate_cap or
getattr(counts, 'range_link_total', 0) >=
2 * candidate_cap * 8 or
getattr(counts, 'file_total', len(counts)) >=
RUNTIME_CACHE_FILE_CAP)
return (saturated and bool(counts) and all(
os.path.normcase(_segment_leaf(os.path.dirname(path))) == active
for path in counts
))
class CandidateInventory(dict):
"""Per-selected-path counts plus the metadata needed for safe projection.
Keep the historical ``path -> F count`` mapping API: several fixed-point
callers use its keys to decide which compiler tier owns the inventory. The
extra fields distinguish the deduplicated process-lifetime candidate cost
from physical F-row, range-link, and file costs, none of which may dedup.
"""
def __init__(self):
super().__init__()
self.identities = {}
self.raw_counts = {}
self.range_link_counts = {}
self.raw_total = 0
self.range_link_total = 0
self.file_total = 0
RUNTIME_CACHE_FILE_CAP = 4096
def _runtime_manifest_range_link_count(funcs: list) -> int:
"""Mirror LazyMan's unique 4 KiB RAM-page links for parsed F records."""
page_count = PSX_RAM_SIZE // 4096
total = 0
for _entry, _crc, ranges in funcs:
seen_pages = set()
for start, length in ranges:
lo = start & 0x1FFFFFFF
hi = lo + length - 1
p0, p1 = lo >> 12, hi >> 12
if p0 >= page_count:
continue
p1 = min(p1, page_count - 1)
seen_pages.update(range(p0, p1 + 1))
total += len(seen_pages)
return total
def _normalized_runtime_manifest_identity(manifest: str, pair_id: int | None,
funcs: list):
"""Return a conservative whole-pair identity, or ``None`` if ambiguous.
Runtime address aliases are normalized, but F and R ordering is retained:
range order participates in the guarded CRC calculation. Compiler-owned
provenance is part of the identity. Unknown/malformed provenance and
legacy manifests without P remain unique rather than being guessed equal.
"""
if pair_id is None:
return None
provenance = manifest_provenance(manifest)
if (manifest_declares_supplemental_provenance(manifest) and
provenance is None):
return None
normalized_funcs = tuple(
(entry & 0x1FFFFFFF, crc,
tuple((start & 0x1FFFFFFF, length) for start, length in ranges))
for entry, crc, ranges in funcs
)
return pair_id, provenance, normalized_funcs
def _candidate_group_identity(cache_dir: str, manifest_identity):
if manifest_identity is None:
return None
return os.path.normcase(os.path.abspath(cache_dir)), manifest_identity
_candidate_validation_cache_lock = threading.Lock()
_candidate_abi_cache = {}
_candidate_record_cache = {}
_candidate_rejection_witness_cache = {}
def _candidate_file_fingerprint(path: str):
"""Return the replacement-sensitive identity of one cache artifact.
Cache publication uses atomic replacement, so a new pair receives a new
file identity even when its size and timestamps happen to match. Keep the
timestamps as defense for in-place test/repair writes. Callers still scan
the namespace and perform the authoritative capacity projection every time;
this cache only avoids repeatedly mapping unchanged DLLs and reparsing their
unchanged manifests near the candidate ceiling.
"""
try:
stat = os.stat(path)
except OSError:
return None
return (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns,
getattr(stat, 'st_ctime_ns', None))
def _candidate_cached_abi_matches(dll: str, expected_abi: int) -> bool:
path = os.path.normcase(os.path.abspath(dll))
fingerprint = (
expected_abi, _dll_abi_matches,
_candidate_file_fingerprint(dll),
)
with _candidate_validation_cache_lock:
cached = _candidate_abi_cache.get(path)
if cached is not None and cached[0] == fingerprint:
return cached[1]
matched = _dll_abi_matches(dll, expected_abi)
# Do not memoize across an uncooperative in-place writer. Cooperating cache
# publishers are already excluded by the namespace lock held by callers.
if matched and fingerprint[2] == _candidate_file_fingerprint(dll):
with _candidate_validation_cache_lock:
_candidate_abi_cache[path] = (fingerprint, matched)
return matched
def _load_candidate_inventory_record(dll: str,
expected_abi: int | None):
"""Return valid rows, identity, raw F count, and raw range-link count."""
ranges_path = os.path.splitext(dll)[0] + '.ranges'
path = os.path.normcase(os.path.abspath(dll))
def fingerprint():
return (
expected_abi, _dll_runtime_exports_match,
_candidate_file_fingerprint(dll),
_candidate_file_fingerprint(ranges_path),
)
# A positive cached record is immutable under the cooperating publisher's
# atomic-replacement contract. Double-stat it before reuse; a changed or
# transaction-hidden artifact falls through to locked recovery below.
observed = fingerprint()
with _candidate_validation_cache_lock:
cached = _candidate_record_cache.get(path)
if (cached is not None and cached[0] == observed and
observed == fingerprint()):
return cached[1]
try:
with _shard_pair_lock(dll):
_recover_shard_pair_locked(dll)
if (not os.path.isfile(dll) or os.path.getsize(dll) == 0 or
not os.path.isfile(ranges_path) or
os.path.getsize(ranges_path) == 0):
return [], None, 0, 0
observed = fingerprint()
with _candidate_validation_cache_lock:
cached = _candidate_record_cache.get(path)
if cached is not None and cached[0] == observed:
return cached[1]
with open(ranges_path, encoding='ascii', newline='') as source:
manifest = source.read()
pair_id, funcs = parse_runtime_shard_manifest(
manifest, require_pair=False,
expected_segment=cache_dir_segment(os.path.dirname(dll)))
raw_count = len(funcs)
range_link_count = _runtime_manifest_range_link_count(funcs)
if (not funcs or not _dll_runtime_exports_match(
dll, expected_abi, pair_id,
{entry for entry, _crc, _ranges in funcs})):
record = ([], None, raw_count, range_link_count)
else:
identity = _normalized_runtime_manifest_identity(
manifest, pair_id, funcs)
record = (
funcs,
_candidate_group_identity(os.path.dirname(dll), identity),
raw_count, range_link_count)
stable = observed == fingerprint()
# Boolean DLL validation currently conflates deterministic export
# mismatch with transient LoadLibrary failure. Cache successes only
# so a one-off loader failure cannot suppress a pair indefinitely.
if stable and record[0]:
with _candidate_validation_cache_lock:
_candidate_record_cache[path] = (observed, record)
return record
except (OSError, UnicodeError):
return [], None, 0, 0
def cache_candidate_inventory(cache_dirs: list[str],
expected_abi: int | None) -> tuple[int, dict]:
"""Inventory runtime-selected pairs with conservative whole-pair dedup.
Only a non-legacy P identity with the exact normalized manifest semantics
and provenance may share candidate cost, and only inside one compiler-tier
leaf. Raw manifest rows, per-F page links, and files remain undeduped.
"""
ext = overlay_ext()
canonical = re.compile(
rf'^[0-9A-Fa-f]{{8}}_[0-9A-Fa-f]{{8}}{re.escape(ext)}$')
counts = CandidateInventory()
seen_basenames = set()
ordered_dirs = []
for path in cache_dirs:
absolute = os.path.abspath(path)
if absolute not in ordered_dirs:
ordered_dirs.append(absolute)
for cache_dir in ordered_dirs:
try:
names = sorted(os.listdir(cache_dir))
except OSError:
continue
for name in names:
if not canonical.fullmatch(name):
continue
dll = os.path.abspath(os.path.join(cache_dir, name))
if (expected_abi is not None and
not _candidate_cached_abi_matches(dll, expected_abi)):
# Runtime's ABI preflight removes this path from the index
# before scanning the next compiler tier, so it cannot shadow
# a valid lower-tier basename.
continue
basename = _segment_basename(dll)
if basename in seen_basenames:
continue
# Runtime scans gcc before tcc and suppresses a lower-tier path by
# basename even when the selected higher-tier pair later fails ABI
# validation. Mirror that exact candidate population here.
seen_basenames.add(basename)
funcs, identity, raw_count, range_link_count = \
_load_candidate_inventory_record(dll, expected_abi)
counts[dll] = len(funcs)
counts.identities[dll] = identity
counts.raw_counts[dll] = raw_count
counts.range_link_counts[dll] = range_link_count
counts.raw_total = sum(counts.raw_counts.values())
counts.range_link_total = sum(counts.range_link_counts.values())
counts.file_total = len(counts)
seen_identities = set()
total = 0
for path, count in counts.items():
identity = counts.identities[path]
if identity is None or identity not in seen_identities:
total += count
if identity is not None:
seen_identities.add(identity)
return total, counts
def recover_candidate_namespace(cache_dirs: list[str]) -> None:
"""Recover transaction-hidden canonical pairs before capacity inventory."""
ext = overlay_ext()
journal_suffix = ext + '.pair-txn.json'
canonical = re.compile(
rf'^[0-9A-Fa-f]{{8}}_[0-9A-Fa-f]{{8}}{re.escape(ext)}$')
for cache_dir in cache_dirs:
try:
journals = sorted(
name for name in os.listdir(cache_dir)
if name.endswith(journal_suffix))
except OSError:
continue
for name in journals:
dll_name = name[:-len('.pair-txn.json')]
if canonical.fullmatch(dll_name):
recover_shard_pair(os.path.join(cache_dir, dll_name))
def _projected_cache_capacity_usage(total: int, counts: dict,
cache_dirs: list[str], final_dll: str,
staged_count: int, staged_identity=None,
staged_range_links: int = 0
) -> tuple[int, int, int, int]:
"""Model candidate, lazy-F, range-link, and file usage."""
final_dll = os.path.abspath(final_dll)
final_dir = os.path.normcase(os.path.dirname(final_dll))
ranks = {
os.path.normcase(os.path.abspath(path)): index
for index, path in enumerate(cache_dirs)
}
final_rank = ranks.get(final_dir, 0)
basename = _segment_basename(final_dll)
selected_path = next((
path for path in counts
if _segment_basename(path) == basename
), None)
records = [(path, count,
getattr(counts, 'identities', {}).get(path),
getattr(counts, 'raw_counts', {}).get(path, count),
getattr(counts, 'range_link_counts', {}).get(path, 0))
for path, count in counts.items()]
if selected_path is not None:
selected_rank = ranks.get(
os.path.normcase(os.path.dirname(selected_path)), 0)
if final_rank > selected_rank:
# A lower-tier publication is invisible while a same-basename
# higher tier exists.
return (total, getattr(counts, 'raw_total', sum(counts.values())),
getattr(counts, 'range_link_total', 0),
getattr(counts, 'file_total', len(counts)))
records = [record for record in records if record[0] != selected_path]
group_identity = _candidate_group_identity(final_dir, staged_identity)
records.append((final_dll, staged_count, group_identity, staged_count,
staged_range_links))
projected = 0
seen_identities = set()
for _path, count, identity, _raw_count, _range_links in records:
if identity is None or identity not in seen_identities:
projected += count
if identity is not None:
seen_identities.add(identity)
return (projected,
sum(raw_count for _path, _count, _identity, raw_count, _links
in records),
sum(links for _path, _count, _identity, _raw, links in records),
len(records))
def projected_cache_candidate_usage(total: int, counts: dict,
cache_dirs: list[str], final_dll: str,
staged_count: int,
staged_identity=None) -> tuple[int, int, int]:
"""Compatibility API for candidate, lazy-F, and file usage."""
projected, raw, _range_links, files = _projected_cache_capacity_usage(
total, counts, cache_dirs, final_dll, staged_count, staged_identity)
return projected, raw, files
def projected_cache_candidate_count(total: int, counts: dict,
cache_dirs: list[str], final_dll: str,
staged_count: int,
staged_identity=None) -> int:
"""Compatibility wrapper returning process-lifetime candidate usage."""
return projected_cache_candidate_usage(
total, counts, cache_dirs, final_dll, staged_count,
staged_identity)[0]
def _candidate_capacity_projection_error(
total: int, counts: dict, projected: int, projected_raw: int,
projected_range_links: int, projected_files: int,
staged_count: int, candidate_cap: int) -> str | None:
"""Return the same fail-closed capacity verdict before or after linking."""
raw_total = getattr(counts, 'raw_total', sum(counts.values()))
range_link_total = getattr(counts, 'range_link_total', 0)
file_total = getattr(counts, 'file_total', len(counts))
if projected > candidate_cap and projected > total:
return ('overlay candidate capacity would be exceeded: '
f'{total} existing -> {projected} projected with '
f'{staged_count} staged > {candidate_cap}; canonical '
'pair preserved')
if projected_raw > 2 * candidate_cap and projected_raw > raw_total:
return ('overlay lazy-manifest capacity would be exceeded: '
f'{raw_total} existing -> {projected_raw} projected > '
f'{2 * candidate_cap}; canonical pair preserved')
if (projected_range_links > 2 * candidate_cap * 8 and
projected_range_links > range_link_total):
return ('overlay lazy range-link capacity would be exceeded: '
f'{range_link_total} existing -> '
f'{projected_range_links} projected > '
f'{2 * candidate_cap * 8}; canonical pair preserved')
if (projected_files > RUNTIME_CACHE_FILE_CAP and
projected_files > file_total):
return ('overlay cache file capacity would be exceeded: '
f'{file_total} existing -> {projected_files} projected > '
f'{RUNTIME_CACHE_FILE_CAP}; canonical pair preserved')
return None
def preflight_shard_candidate_capacity(
final_dll: str, func_ids: list, pair_id: int,
manifest_provenance: str | None, expected_abi: int | None,
candidate_cap: int) -> str | None:
"""Reject an impossible publication before invoking the native linker.
The locked publication check remains authoritative. This early check is
deliberately one-sided: a concurrent deletion after the snapshot can only
defer safe coverage until the next invocation, while an admitted shard is
still reprojected under the namespace lock after it has been compiled.
"""
final_dll = os.path.abspath(final_dll)
capacity_lock, cache_dirs = _candidate_capacity_namespace(final_dll)
manifest = overlay_ranges_text(
func_ids, pair_id, manifest_provenance)
staged_identity = _normalized_runtime_manifest_identity(
manifest, pair_id, func_ids)
staged_range_links = _runtime_manifest_range_link_count(func_ids)
with _exclusive_file_lock(capacity_lock):
cache_dirs_key = tuple(
os.path.normcase(os.path.abspath(path)) for path in cache_dirs)
witness_key = (
os.path.normcase(os.path.abspath(capacity_lock)), cache_dirs_key,
expected_abi, candidate_cap, _dll_abi_matches,
_dll_runtime_exports_match,
)
with _candidate_validation_cache_lock:
witness = _candidate_rejection_witness_cache.get(witness_key)
if witness is not None:
witness_total, witness_counts = witness
witness_usage = _projected_cache_capacity_usage(
witness_total, witness_counts, cache_dirs, final_dll,
len(func_ids), staged_identity, staged_range_links)
witness_error = _candidate_capacity_projection_error(
witness_total, witness_counts, *witness_usage,
len(func_ids), candidate_cap)
if witness_error is not None:
# Reuse only a proof of impossibility. Added pairs make the
# bound stronger; deletion/replacement can merely defer safe
# optional coverage until the next command. An apparent
# admission always falls through to a fresh locked inventory.
return witness_error
recover_candidate_namespace(cache_dirs)
total, counts = cache_candidate_inventory(cache_dirs, expected_abi)
with _candidate_validation_cache_lock:
_candidate_rejection_witness_cache[witness_key] = (total, counts)
usage = _projected_cache_capacity_usage(
total, counts, cache_dirs, final_dll, len(func_ids),
staged_identity, staged_range_links)
return _candidate_capacity_projection_error(
total, counts, *usage, len(func_ids), candidate_cap)
def _cleanup_old_pair_debris(dll_path: str) -> None:
for path in glob.glob(os.path.abspath(dll_path) + '.pair-txn.*.old-*'):
_best_effort_unlink(path)
def _recover_shard_pair_locked(dll_path: str) -> None:
dll, ranges, _lock, journal_path = _shard_pair_paths(dll_path)
if not os.path.isfile(journal_path):
_cleanup_old_pair_debris(dll)
return
try:
with open(journal_path, encoding='utf-8') as source:
txn = json.load(source)
except (OSError, ValueError) as exc:
raise RuntimeError(f'cannot recover shard transaction {journal_path}: {exc}')
required = ('backup_dll', 'backup_ranges', 'new_dll_sha256',
'new_ranges_sha256', 'old_dll_sha256', 'old_ranges_sha256')
if txn.get('schema') != 'psxrecomp shard pair transaction v1' or any(
key not in txn for key in required):
raise RuntimeError(f'invalid shard transaction journal {journal_path}')
backup_dll = txn['backup_dll']
backup_ranges = txn['backup_ranges']
token = txn.get('token', '')
if (not re.fullmatch(r'[0-9a-f]{32}', token) or
backup_dll != f'{dll}.pair-txn.{token}.old-dll' or
backup_ranges != f'{dll}.pair-txn.{token}.old-ranges'):
raise RuntimeError(f'invalid shard transaction ownership in {journal_path}')
new_complete = (_file_sha256(dll) == txn['new_dll_sha256'] and
_file_sha256(ranges) == txn['new_ranges_sha256'])
if new_complete:
# The DLL rename is the commit point. A crash before a journal phase
# update must not roll back a new DLL already paired with its exact
# manifest. Remove the journal first; old loaded-DLL backups are merely
# deferred debris and may remain locked until process exit on Windows.
_unlink_if_present(journal_path)
_best_effort_unlink(backup_dll)
_best_effort_unlink(backup_ranges)
return
def restore(path: str, backup: str, expected_hash: str | None) -> None:
current_hash = _file_sha256(path)
if expected_hash is None:
if current_hash is not None:
_unlink_if_present(path)
return
if current_hash == expected_hash:
return
if _file_sha256(backup) != expected_hash:
raise RuntimeError(
f'cannot recover {path}: neither canonical nor backup is the old file')
if current_hash is not None:
_unlink_if_present(path)
os.replace(backup, path)
# Restore the manifest first; restoring the DLL is the rollback commit.
restore(ranges, backup_ranges, txn['old_ranges_sha256'])
restore(dll, backup_dll, txn['old_dll_sha256'])
_unlink_if_present(journal_path)
_best_effort_unlink(backup_dll)
_best_effort_unlink(backup_ranges)
def recover_shard_pair(dll_path: str) -> None:
"""Recover/finish a dead writer's transaction under its OS-owned lock."""
with _shard_pair_lock(dll_path):
_recover_shard_pair_locked(dll_path)
def _runtime_valid_shard_pair_locked(dll: str, ranges: str,
expected_abi: int | None) -> bool:
"""Mirror the pair-aware loader's minimum acceptance checks."""
if (not os.path.isfile(dll) or os.path.getsize(dll) == 0 or
not os.path.isfile(ranges) or os.path.getsize(ranges) == 0):
return False
try:
with open(ranges, encoding='ascii', newline='') as f:
manifest = f.read()
pair_id, funcs = parse_runtime_shard_manifest(
manifest, require_pair=False,
expected_segment=cache_dir_segment(os.path.dirname(dll)))
except (OSError, UnicodeError):
return False
return (bool(funcs) and _dll_runtime_exports_match(
dll, expected_abi, pair_id,
{entry for entry, _crc, _ranges in funcs}))
def publish_shard_pair(staged_dll: str, staged_ranges: str,
final_dll: str, preserve_existing: bool = False,
expected_existing_abi: int | None = None,
candidate_cap: int | None = None) -> bool:
"""Commit a staged pair, optionally preserving a valid concurrent winner.
Region DLL names are keyed only by whole-region CRC. Two processes can
compile the same bytes with different discovery evidence; the later,
poorer writer must not overwrite a richer pair after that richer process
has used its own result to suppress fragment repair. First valid publisher
wins, while each process's post-pass re-reads the actual winner and emits
any exact-demand fragments it still lacks.
"""
dll, ranges, _lock, journal_path = _shard_pair_paths(final_dll)
token = uuid.uuid4().hex
backup_dll = f'{dll}.pair-txn.{token}.old-dll'
backup_ranges = f'{dll}.pair-txn.{token}.old-ranges'
capacity_lock, cache_dirs = _candidate_capacity_namespace(dll)
capacity_guard = (_exclusive_file_lock(capacity_lock)
if candidate_cap is not None else nullcontext())
with capacity_guard:
projected = None
staged_count = 0
total = 0
if candidate_cap is not None:
try:
with open(staged_ranges, encoding='ascii', newline='') as source:
staged_manifest = source.read()
staged_pair, staged_funcs = (
parse_runtime_shard_manifest(staged_manifest))
except (OSError, UnicodeError) as exc:
raise RuntimeError(
f'cannot read staged range manifest: {exc}') from exc
if not staged_funcs:
raise RuntimeError(
'staged range manifest has no runtime-valid F records')
staged_count = len(staged_funcs)
recover_candidate_namespace(cache_dirs)
total, counts = cache_candidate_inventory(
cache_dirs, expected_existing_abi)
staged_identity = _normalized_runtime_manifest_identity(
staged_manifest, staged_pair, staged_funcs)
staged_range_links = _runtime_manifest_range_link_count(staged_funcs)
(projected, projected_raw, projected_range_links,
projected_files) = _projected_cache_capacity_usage(
total, counts, cache_dirs, dll, staged_count,
staged_identity, staged_range_links)
with _shard_pair_lock(dll):
_recover_shard_pair_locked(dll)
if (preserve_existing and _runtime_valid_shard_pair_locked(
dll, ranges, expected_existing_abi)):
return False
if candidate_cap is not None:
capacity_error = _candidate_capacity_projection_error(
total, counts, projected, projected_raw,
projected_range_links, projected_files, staged_count,
candidate_cap)
if capacity_error is not None:
raise ShardCandidateCapacityError(capacity_error)
txn = {
'schema': 'psxrecomp shard pair transaction v1',
'token': token,
'backup_dll': backup_dll,
'backup_ranges': backup_ranges,
'old_dll_sha256': _file_sha256(dll),
'old_ranges_sha256': _file_sha256(ranges),
'new_dll_sha256': _file_sha256(staged_dll),
'new_ranges_sha256': _file_sha256(staged_ranges),
}
if txn['new_dll_sha256'] is None or txn['new_ranges_sha256'] is None:
raise RuntimeError('staged DLL/ranges disappeared before publication')
_write_json_atomic(journal_path, txn)
try:
if txn['old_dll_sha256'] is not None:
os.replace(dll, backup_dll)
if txn['old_ranges_sha256'] is not None:
os.replace(ranges, backup_ranges)
# The new manifest alone is inert. Publishing the DLL is the commit.
os.replace(staged_ranges, ranges)
os.replace(staged_dll, dll)
_recover_shard_pair_locked(dll) # recognizes committed hashes
except BaseException:
_recover_shard_pair_locked(dll)
raise
return True
def _dll_abi_matches(dll_path: str, expected_abi: int) -> bool:
"""Read overlay_abi from one loaded image, then release it immediately."""
import _ctypes
import ctypes
library = None
try:
# Shards are generated as ordinary C (cdecl). ctypes.WinDLL selects
# stdcall and can corrupt a 32-bit Windows caller's stack; CDLL is the
# correct loader on every supported host (x64 merely masks the error).
library = ctypes.CDLL(dll_path)
abi_fn = library.overlay_abi
abi_fn.argtypes = []
abi_fn.restype = ctypes.c_int
return (abi_fn() & 0xFFFFFFFF) == (expected_abi & 0xFFFFFFFF)
except (OSError, AttributeError):
return False
finally:
if library is not None and library._handle:
handle = library._handle
library._handle = 0
if os.name == 'nt':
_ctypes.FreeLibrary(handle)
else:
_ctypes.dlclose(handle)
def _dll_pair_id_matches(dll_path: str, expected_pair_id: int) -> bool:
"""Read overlay_pair_id from one loaded image, then release it."""
import _ctypes
import ctypes
library = None
try:
library = ctypes.CDLL(dll_path)
pair_fn = library.overlay_pair_id
pair_fn.argtypes = []
pair_fn.restype = ctypes.c_uint64
return pair_fn() == (expected_pair_id & 0xFFFFFFFFFFFFFFFF)
except (OSError, AttributeError):
return False
finally:
if library is not None and library._handle:
handle = library._handle
library._handle = 0
if os.name == 'nt':
_ctypes.FreeLibrary(handle)
else:
_ctypes.dlclose(handle)
def _dll_runtime_exports_match(dll_path: str, expected_abi: int | None,
expected_pair_id: int | None,
entries: set[int]) -> bool:
"""Validate the exports the runtime needs in one library mapping."""
import _ctypes
import ctypes
library = None
try:
library = ctypes.CDLL(dll_path)
abi_fn = library.overlay_abi
abi_fn.argtypes = []
abi_fn.restype = ctypes.c_int
if (expected_abi is not None and
(abi_fn() & 0xFFFFFFFF) != (expected_abi & 0xFFFFFFFF)):
return False
try:
pair_fn = library.overlay_pair_id
except AttributeError:
if expected_pair_id is not None:
return False
else:
if expected_pair_id is None:
return False
pair_fn.argtypes = []
pair_fn.restype = ctypes.c_uint64
if pair_fn() != (expected_pair_id & 0xFFFFFFFFFFFFFFFF):
return False
getattr(library, 'overlay_init')
getattr(library, 'overlay_flush_cycles')
for entry in entries:
# Exports are named by the entry's full VA (§5.1, §5.7).
getattr(library, f'func_{entry:08X}')
return True
except (OSError, AttributeError):
return False
finally:
if library is not None and library._handle:
handle = library._handle
library._handle = 0
if os.name == 'nt':
_ctypes.FreeLibrary(handle)
else:
_ctypes.dlclose(handle)
def compiled_shard_complete(dll_path: str,
expected_abi: int | None = None) -> bool:
"""Whether the pair satisfies the runtime's manifest/export contract."""
return bool(load_shard_func_ids(dll_path, expected_abi))
def shard_pair_files_complete(dll_path: str) -> bool:
"""Filesystem-only completeness used by crash-transaction tests."""
ranges = os.path.splitext(dll_path)[0] + '.ranges'
with _shard_pair_lock(dll_path):
_recover_shard_pair_locked(dll_path)
return (os.path.isfile(dll_path) and
os.path.getsize(dll_path) > 0 and
os.path.isfile(ranges) and
os.path.getsize(ranges) > 0)
def cached_shard_manifest_status(dll_path: str, expected_abi: int | None,
expected_manifest: str,
expected_pair_id: int | None = None) -> str:
"""Return ``match``, ``mismatch``, or ``missing`` under the pair lock."""
ranges = os.path.splitext(dll_path)[0] + '.ranges'
with _shard_pair_lock(dll_path):
_recover_shard_pair_locked(dll_path)
complete = (os.path.isfile(dll_path) and
os.path.getsize(dll_path) > 0 and
os.path.isfile(ranges) and
os.path.getsize(ranges) > 0)
if not complete:
return 'missing'
try:
with open(ranges) as f:
actual_manifest = f.read()
except OSError:
return 'missing'
if expected_pair_id is not None:
if (expected_abi is not None and
not _dll_abi_matches(dll_path, expected_abi)):
# A stale flavor/ABI is replaceable, not an identity collision.
return 'missing'
try:
entries = {
int(parts[1], 16)
for parts in (line.split()
for line in actual_manifest.splitlines())
if len(parts) >= 3 and parts[0] == 'F'
}
except ValueError:
return 'mismatch'
if (not entries or not _dll_runtime_exports_match(
dll_path, None, expected_pair_id, entries)):
return 'mismatch'
elif (expected_abi is not None and
not _dll_abi_matches(dll_path, expected_abi)):
return 'missing'
return 'match' if actual_manifest == expected_manifest else 'mismatch'
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Static mode (--static): per-capture worker + split output writer
# ---------------------------------------------------------------------------
def static_capture_job(cap: dict, args, toml: dict, forced_interiors: set,
static_out: str) -> dict:
"""Recompile ONE capture for static mode. Module-level and self-contained so
it can run in a worker process (ProcessPoolExecutor needs a picklable
callable and picklable arguments: the capture dict, the argparse Namespace,
the parsed game.toml, the forced-interior set).
Nothing here touches shared state. Everything the sequential path used to
write into main()'s accumulators comes back in the result and is merged by
the caller in canonical capture order, so the emitted C is byte-identical
whether --jobs is 1 or 12. Prints go to result['log'] instead of stdout so
parallel workers never interleave.
result: label, outcome ('ok' | 'skip' | 'fail'), fail=(class, detail),
part (the static_parts entry, or None),
requested_entries (set of (entry, image_crc)),
entry_sources ({(entry, image_crc): (data, load_addr, size,
phys_addr, guard_bytes)}), log.
Requested entries are keyed by (entry, image crc), never by the bare
address: the dispatcher validates every variant against the exact bytes
it was compiled from, so a demand is only served by a variant built from
THIS capture's bytes. Two captures of one band that both observed an
address are two demands (see the isolated-fragment pass in main()).
"""
result = {'label': None, 'outcome': None, 'fail': None, 'part': None,
'requested_entries': set(), 'entry_sources': {}, 'log': ''}
log = io.StringIO()
with contextlib.redirect_stdout(log):
try:
_static_capture_job(cap, args, toml, forced_interiors, static_out,
result)
finally:
result['log'] = log.getvalue()
return result
def _static_capture_job(cap, args, toml, forced_interiors, static_out, result):
# Runs in a worker process too, so it sets its image segment itself.
with image_segment(segment_of(int(cap['load_addr'], 16))):
_static_capture_job_in_segment(cap, args, toml, forced_interiors,
static_out, result)
def _static_capture_job_in_segment(cap, args, toml, forced_interiors,
static_out, result):
load_addr = int(cap['load_addr'], 16)
size = int(cap['size'])
data = base64.b64decode(cap['bytes_b64'])
crc32 = binascii.crc32(data) & 0xFFFFFFFF
phys_addr = (load_addr & 0x1FFFFFFF)
_label = f'overlay 0x{load_addr:08X} crc {crc32:08X}'
result['label'] = _label
# Trailing delay-slot guard words appended by the capture writer.
# Declared to the recompiler through the PS-EXE analysis-bound tag so
# they stay readable but are never discovered as code.
guard_bytes = capture_guard_bytes(cap, size, _label)
def fail(cls, detail):
result['outcome'] = 'fail'
result['fail'] = (cls, detail)
for captured_entry in _parse_addr_list(cap.get('dispatch_entry_pcs', [])):
entry = _canonical_guest_addr(captured_entry)
key = (entry, crc32)
result['requested_entries'].add(key)
result['entry_sources'][key] = (
data, load_addr, size, phys_addr, guard_bytes)
# --force-interior is an explicit operator assertion that a live dispatch
# entry was observed even if the retained capture lost its classifier
# provenance. Bind the requested PC to this capture's exact bytes; the
# post-pass builds a content-validated isolated shard for it.
region_hi = phys_addr + size
for forced_entry in forced_interiors:
forced_phys = forced_entry & 0x1FFFFFFF
if (phys_addr <= forced_phys < region_hi and
segment_of(forced_entry) == segment_of(load_addr)):
entry = forced_entry
key = (entry, crc32)
result['requested_entries'].add(key)
result['entry_sources'][key] = (
data, load_addr, size, phys_addr, guard_bytes)
seeds, seed_audit = classify_overlay_seeds(cap, data, load_addr, size,
crc32, toml)
print(f'Overlay load=0x{load_addr:08X} size={size} crc32=0x{crc32:08X}'
+ (f' guard={guard_bytes}B (delay-slot only, not analysed)'
if guard_bytes else ''))
print(f' seeds: {len(seeds)} mode: static -> {static_out}')
print_seed_audit(seed_audit)
root_seeds = [
seed for seed in seeds
if ((seed.split()[0] in ('call_root', 'dispatch_root')) or
seed.split()[0].startswith('0x'))
]
if not root_seeds:
print(f' {NO_SHARED_WALK_ROOT_SEEDS_SKIP}\n')
result['outcome'] = 'skip'
return
with tempfile.TemporaryDirectory() as tmp:
# Fake PS-EXE: the header entry PC becomes a walk root in the
# recompiler, so it must be a walk-root seed, never an interior one.
entry_pc = int(root_seeds[0].split()[-1], 16)
psx_path = os.path.join(tmp, f'overlay_{load_addr:08X}.psx')
with open(psx_path, 'wb') as f:
f.write(make_psxexe(load_addr, entry_pc, data,
guard_bytes=guard_bytes))
seeds_path = os.path.join(tmp, 'seeds.txt')
with open(seeds_path, 'w') as f:
for seed in seeds:
f.write(seed + '\n')
out_dir_tmp = os.path.join(tmp, 'out')
os.makedirs(out_dir_tmp)
# Evidence-scoped discovery (the overlay-compilation contract): compile
# only the proven entry seeds and what they reach, never a whole-byte
# sweep. Same invocation as the DLL path.
cmd = [args.recompiler, psx_path,
'--seeds', seeds_path,
'--out-dir', out_dir_tmp,
'--overlay',
'--ws-config', os.path.abspath(args.game_toml)]
cmd += recompiler_project_root_args(args)
print(f' recompile: {args.recompiler} ...{" [CPS]" if args.cps else ""}')
toml_dir = os.path.dirname(os.path.abspath(args.game_toml))
sub_env = dict(os.environ)
if args.cps:
sub_env['PSX_CPS'] = '1' # §25: emit continuation-passing overlay C
r = subprocess.run(cmd, capture_output=True, text=True,
cwd=toml_dir, env=sub_env)
if r.returncode != 0:
print(f' RECOMPILER ERROR:\n{r.stderr or r.stdout}')
fail('recompiler', r.stderr or r.stdout)
return
stem = os.path.basename(psx_path)
full_c = os.path.join(out_dir_tmp, stem + '_full.c')
if not os.path.exists(full_c):
candidates = [f for f in os.listdir(out_dir_tmp) if f.endswith('_full.c')]
if not candidates:
print(f' ERROR: no _full.c in {out_dir_tmp}')
fail('no_output', 'no _full.c emitted')
return
full_c = os.path.join(out_dir_tmp, candidates[0])
with open(full_c) as f:
src = f.read()
ranges_src = None
for fn in os.listdir(out_dir_tmp):
if fn.endswith('_full.ranges'):
ranges_src = os.path.join(out_dir_tmp, fn)
break
src, func_addrs = patch_generated_c_static(src, load_addr, size)
continuation_owners = parse_cps_continuation_owners(src)
c_audit = audit_generated_c(src, load_addr, size, crc32, toml)
print_generated_c_audit(load_addr, size, crc32, c_audit)
if c_audit['unknown_bad'] or c_audit['unsupported_todo_addrs']:
print(' GENERATED-C AUDIT FAILED\n')
fail('audit', f'{len(c_audit["unknown_bad"])} unknown_bad, '
f'{len(c_audit["unsupported_todo_addrs"])} unsupported')
return
if not ranges_src:
print(' STATIC RANGE AUDIT FAILED: no _full.ranges manifest\n')
fail('no_ranges', 'no _full.ranges manifest')
return
func_ids = parse_overlay_func_ids(ranges_src, data, load_addr, size)
ids_by_addr = {}
for ev, code_crc, ranges in func_ids:
ids_by_addr.setdefault(ev, []).append((code_crc, ranges))
missing = sorted(set(func_addrs) - set(ids_by_addr))
if missing:
sample = ', '.join(f'0x{a:08X}' for a in missing[:8])
print(f' STATIC RANGE AUDIT FAILED: {len(missing)} '
f'dispatchable function(s) lack exact ranges: {sample}\n')
fail('static_ranges', f'{len(missing)} funcs lack exact ranges')
return
# Whole-image identity plus compiled-entry coverage makes the namespace
# deterministic while allowing a later, richer seed capture of
# identical bytes to coexist without symbol clashes.
cov_blob = ','.join(f'{a:08X}' for a in sorted(func_addrs)).encode()
cov_crc = binascii.crc32(cov_blob) & 0xFFFFFFFF
# A KUSEG or KSEG1 image of the same bytes links beside the KSEG0 one
# (§5.7): ov_s0_… and ov_s5_…; KSEG0 keeps its name.
seg_tag = ('' if segment_of(load_addr) == KSEG0
else f's{segment_of(load_addr) >> 29}_')
namespace = f'ov_{seg_tag}{phys_addr:08X}_{crc32:08X}_{cov_crc:08X}'
src, symbols = namespace_generated_static(src, namespace, func_addrs)
variants = []
for ev in sorted(func_addrs):
for code_crc, ranges in ids_by_addr[ev]:
variants.append({
'addr': ev,
'symbol': symbols[ev],
'crc': code_crc,
'ranges': ranges,
})
result['part'] = {
'src': src,
'variants': variants,
'namespace': namespace,
'image_crc': crc32,
'func_addrs': set(func_addrs),
'symbols': symbols,
'ids_by_addr': ids_by_addr,
'continuation_owners': continuation_owners,
}
print(f' recompiled: {len(func_addrs)} functions, '
f'{len(variants)} exact identities\n')
result['outcome'] = 'ok'
STATIC_HEADER = ('/* Auto-generated overlay dispatch -- do not edit.\n'
' * Rebuild: python3 psxrecomp/tools/compile_overlays.py --static ...\n'
' */\n')
_STATIC_PART_RE = re.compile(r'_(\d{4})\.c$')
def static_part_paths(static_out: str) -> list:
"""Existing split-output part files next to static_out, sorted."""
d = os.path.dirname(static_out) or '.'
stem = os.path.splitext(os.path.basename(static_out))[0]
if not os.path.isdir(d):
return []
out = []
for fn in os.listdir(d):
if fn.startswith(stem + '_') and _STATIC_PART_RE.search(fn):
out.append(os.path.join(d, fn))
return sorted(out)
def _write_if_changed(path: str, text: str) -> bool:
"""Write only when the content differs, so an unchanged translation unit
keeps its mtime and the build system does not recompile it."""
try:
with open(path, encoding='utf-8') as f:
if f.read() == text:
return False
except (OSError, UnicodeDecodeError):
pass
with open(path, 'w', encoding='utf-8', newline='\n') as f:
f.write(text)
return True
def write_static_outputs(static_out: str, parts: list, variants: list,
dispatch_src: str, single_file: bool = False) -> list:
"""Emit the static overlay C.
Default (split): `static_out` holds the content-validated dispatcher plus
an extern prototype for every compiled entry, and each part becomes its own
translation unit `<stem>_NNNN.c` beside it. Every part already carries its
own #includes and a private symbol namespace, so it compiles standalone.
The build then parallelizes across parts, and because unchanged parts are
not rewritten, only overlays whose code changed recompile. Stale parts from
a previous, larger run are deleted so a glob never links a leftover.
single_file: the pre-split behaviour, everything in `static_out`.
Returns the list of files that make up the output (written or unchanged).
"""
stale = static_part_paths(static_out)
if single_file:
combined = STATIC_HEADER
for part in parts:
combined += part['src']
combined += dispatch_src
_write_if_changed(static_out, combined)
for p in stale:
os.remove(p)
return [static_out]
d = os.path.dirname(static_out) or '.'
stem = os.path.splitext(os.path.basename(static_out))[0]
# One unit per region part, plus one unit per IMAGE for its isolated
# fragments. Fragments are per (entry, image) and a band with many
# occupants carries thousands of them (BoF3: 20k), so a unit per fragment
# would mean 20k files for the glob and 20k compiler launches. Fragment
# parts are privately namespaced, so concatenating an image's fragments
# is safe; keeping them apart from the region unit means a new fragment
# rewrites only its image's fragment unit, never the region.
units = [] # (label, [parts])
frag_units = {}
for part in parts:
if part['namespace'].startswith('ov_frag_'):
key = part['namespace'].rsplit('_', 1)[0] # ov_frag_<phys>_<crc>
if key not in frag_units:
frag_units[key] = [key + '_fragments', []]
units.append(frag_units[key])
frag_units[key][1].append(part)
else:
units.append([part['namespace'], [part]])
written = []
for index, (label, unit_parts) in enumerate(units):
path = os.path.join(d, f'{stem}_{index:04d}.c')
# No part COUNT in the header: adding or dropping an overlay must leave
# every unchanged unit byte-identical (and therefore not rebuilt).
text = (STATIC_HEADER +
f'/* Static overlay translation unit {index}: {label} */\n' +
''.join(part['src'] for part in unit_parts))
_write_if_changed(path, text)
written.append(path)
keep = set(written)
for p in stale:
if p not in keep:
os.remove(p)
symbols = sorted({variant['symbol'] for variant in variants})
main = STATIC_HEADER
main += '#include "psx_runtime.h"\n\n'
main += (f'/* Dispatcher for {len(written)} overlay translation unit(s) '
f'({stem}_0000.c ...). Every compiled entry has external linkage '
f'in its own unit; declare them here. */\n')
main += ''.join(f'void {sym}(CPUState *cpu);\n' for sym in symbols)
main += dispatch_src
_write_if_changed(static_out, main)
return [static_out] + written
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument('--captures', default=None,
help='overlay_captures.json from the runtime. Optional: the '
'runtime injects PSX_OVERLAY_CAPTURES (the canonical '
'<exe>/overlay_captures.json), which wins. Required only '
'for manual/offline invocation.')
ap.add_argument('--game-toml', required=True,
help='game.toml (for game_id)')
ap.add_argument('--recompiler', required=True,
help='path to psxrecomp-game.exe')
ap.add_argument('--runtime-include', required=True,
help='path to psxrecomp runtime/include dir')
ap.add_argument('--project-root', default=None,
help='root the recompiler resolves the BIOS profile against '
'(bios/SCPH1001.toml, or <framework>/bios/SCPH1001.toml '
'one level down). Defaults to the framework root derived '
'from --runtime-include. Only needed when that '
'derivation is wrong, because we spawn the recompiler '
'with cwd = dirname(game.toml) and a PACKAGED game.toml '
'lives in a directory with no BIOS profile under it.')
ap.add_argument('--out-dir', default='build-dev/cache',
help='cache root dir (default: build-dev/cache)')
ap.add_argument('--gcc', default='gcc',
help='GCC binary (default: gcc)')
ap.add_argument('--compiler', choices=['gcc', 'tcc'], default='gcc',
help='overlay shard compiler: gcc (default; dev/production, '
'best-optimized) or tcc (bundled, toolchain-free user '
'fallback). tcc shards land in the tcc/ cache namespace.')
ap.add_argument('--tcc', default='tcc',
help='TinyCC binary (used when --compiler tcc)')
ap.add_argument('--force', action='store_true',
help='recompile even if output already exists')
ap.add_argument('--only-region', action='append', default=[],
help='compile only captures whose normalized load address '
'matches this value (repeatable; manual validation aid)')
ap.add_argument('--check', action='store_true',
help='PREFLIGHT: build every shard into a throwaway temp dir '
'(implies --force, ignores any injected cache dir, never '
'touches the real cache) and report pass/fail. Answers '
'"would this game\'s shards compile against the CURRENT '
'recompiler + runtime headers?" — exits non-zero if any '
'shard that should build fails. Use after a header/ABI '
'or emitter change to catch silent shard breakage.')
ap.add_argument('--force-interior', action='append', default=[],
help='also compile this virtual/physical PC as an isolated '
'interior fragment (repeatable; diagnostic recovery for '
'an observed dispatch PC whose classifier provenance was '
'lost after a later capture)')
ap.add_argument('--static-single-file', action='store_true',
help='with --static: emit one monolithic overlays_static.c '
'(pre-split behaviour) instead of the dispatcher plus '
'one translation unit per overlay.')
ap.add_argument('--static', action='store_true',
help='B-2 mode: compile into binary (overlays_static.c) instead of DLL')
ap.add_argument('--flavor', type=int, default=0,
help='codegen flavor id baked into overlay_abi() (0=base/master; '
'widescreen build passes 1). The loader rejects DLLs whose '
'flavor != the runtime flavor, keeping caches separate.')
ap.add_argument('--cps', action='store_true',
help='continuation-passing (RECURSION_BUG.md §25): set PSX_CPS '
'when invoking the recompiler so overlay funcs tail-transfer '
'+ carry an entry-switch. Must match the runtime build.')
ap.add_argument('--jobs', type=int,
default=max(1, (os.cpu_count() or 4) - 2),
help='parallel region-group workers (default: cores-2). '
'Captures are grouped by region start; regions are '
'independent (dedup coverage, prior-ranges merge, '
'fragments, and filenames all key on the region), '
'captures within one region stay ordered. 1 = the '
'sequential path. In --static mode each capture is an '
'independent recompile+audit, run in a process pool '
'and merged in capture order (byte-identical output '
'to the sequential path).')
ap.add_argument('--target-os', choices=['auto', 'win', 'linux', 'macos'], default='auto',
help='target operating system for compiled overlay shards (default: auto)')
ap.add_argument('--arch-abi', default=None,
help='"<os>-<arch>" of the runtime that will load the shards '
'(e.g. macos-x64). The spawning runtime injects '
'PSX_OVERLAY_ARCH_ABI, which wins. Default: this '
"interpreter's architecture.")
args = ap.parse_args()
target_os = args.target_os
if target_os == 'auto':
if 'mingw' in args.gcc.lower() or 'w64' in args.gcc.lower() or args.gcc.lower().endswith('.exe'):
target_os = 'win'
else:
target_os = None
set_target_os(target_os)
# Canonicalized to KSEG0 as before (a physical and a KUSEG PC are the
# same number); a forced interior therefore forces KSEG0 views.
forced_interiors = {
(int(v, 0) & 0x1FFFFFFF) | KSEG0
for v in args.force_interior
}
# ---- Framework-injected cache location wins over CLI flags ----------------
# The runtime (autocompile.c) exports PSX_OVERLAY_CACHE_DIR / PSX_OVERLAY_CAPTURES
# set to the loader's CANONICAL <exe>/cache and <exe>/overlay_captures.json. We
# honor those above --out-dir/--captures so the WRITE cache and READ captures
# are always exactly where the loader reads — no per-game config, no drift
# between dev and prod (the read/write-location divergence bug). The CLI flags
# remain the fallback for manual/offline invocation (no env set).
# --check is a preflight: it must NEVER write the real cache, so it ignores
# the injected cache dir and builds into a throwaway temp dir with --force.
_check_tmp = None
if args.check:
args.force = True
_check_tmp = tempfile.mkdtemp(prefix='shardcheck_')
args.out_dir = _check_tmp
print(f'[check] preflight build into throwaway dir: {_check_tmp}')
_env_out = None if args.check else os.environ.get('PSX_OVERLAY_CACHE_DIR')
if _env_out:
if _env_out != args.out_dir:
print(f'[cache] PSX_OVERLAY_CACHE_DIR overrides --out-dir: {_env_out}')
args.out_dir = _env_out
_env_cap = os.environ.get('PSX_OVERLAY_CAPTURES')
if _env_cap:
if _env_cap != args.captures:
print(f'[cache] PSX_OVERLAY_CAPTURES overrides --captures: {_env_cap}')
args.captures = _env_cap
# Flavor is pinned by the spawning runtime the same way as the cache
# locations: --flavor defaults to 0 (play), so an instrumented (flavor 2)
# runtime used to spawn compiles whose shards its loader then rejected —
# every overlay ran interpreted on debug builds, silently.
_env_flavor = os.environ.get('PSX_OVERLAY_FLAVOR')
if _env_flavor:
try:
_fl = int(_env_flavor, 0)
except ValueError:
_fl = None
if _fl is not None and _fl != args.flavor:
print(f'[cache] PSX_OVERLAY_FLAVOR overrides --flavor: {_fl}')
args.flavor = _fl
# Architecture is pinned by the spawning runtime too (overlay_loader.h's
# PSX_OVERLAY_ARCH_ABI): the shards must be the RUNTIME's architecture and
# land in its arch-abi directory, whatever this interpreter or the
# compiler binary happens to be.
_env_arch = os.environ.get('PSX_OVERLAY_ARCH_ABI')
if _env_arch:
if args.arch_abi and _env_arch != args.arch_abi:
print(f'[cache] PSX_OVERLAY_ARCH_ABI overrides --arch-abi: {_env_arch}')
_arch_err = apply_runtime_arch_abi(_env_arch, 'PSX_OVERLAY_ARCH_ABI')
else:
_arch_err = apply_runtime_arch_abi(args.arch_abi, '--arch-abi')
if _arch_err:
ap.error(_arch_err)
if not args.static:
_arch_mismatch = interpreter_arch_mismatch()
if _arch_mismatch:
print(f'ERROR: cannot build overlay shards: {_arch_mismatch}')
sys.exit(2)
if not args.captures:
ap.error('no captures file: set PSX_OVERLAY_CAPTURES (runtime injects it) '
'or pass --captures for manual/offline use')
# Resolve the recompiler exe and runtime-include to absolute paths up front.
# game.toml's overlay_autocompile_cmd uses paths RELATIVE to the project root
# (the runtime spawns this script via cmd.exe with cwd = project root) so the
# shipped config is portable across machines — never bake absolute paths into
# game.toml. But two Windows quirks break relative paths once we hand them off:
# 1. subprocess/CreateProcess does NOT resolve a relative *executable*
# against the child cwd (the recompiler spawn below, cwd=toml_dir), so a
# relative --recompiler fails with WinError 2 and the cache never warms.
# 2. gcc launched from cmd.exe rejects relative forward-slash -I/-c/-o paths
# (already handled inside compile_dll via os.path.abspath).
# Anchoring both here (against this process's cwd = project root, where the
# relative paths are meant to resolve) makes relative config work everywhere.
args.recompiler = os.path.abspath(args.recompiler)
args.runtime_include = os.path.abspath(args.runtime_include)
# Load the shard link-contract preamble from runtime/include up front (hard
# error if absent). Threads/fragments read the module global thereafter.
load_dispatch_preamble(args.runtime_include)
# Read game ID from game.toml (strip BOM if present)
with open(args.game_toml, 'rb') as f:
raw = f.read().lstrip(b'\xef\xbb\xbf') # UTF-8 BOM
toml = tomllib.loads(raw.decode('utf-8'))
game_id = toml.get('game', {}).get('id', 'UNKNOWN')
print(f'Game ID: {game_id}')
# Stale-recompiler-binary guard — BOTH modes (static overlays are just as
# wrong when emitted by a stale binary; they simply have no tag to hide in).
verify_recompiler_matches_tag(args.recompiler, codegen_hash(args.runtime_include))
if args.static:
static_out = os.path.join(args.out_dir, 'overlays_static.c')
if os.path.exists(static_out) and not args.force:
print(f'SKIP: {static_out} already exists (use --force to recompile)')
return
os.makedirs(args.out_dir, exist_ok=True)
else:
# Namespaced + versioned gcc cache:
# <game_id>/gcc/<arch-abi>/cg<N>_<emitter-hash>_gc<config-hash>/
# (SLJIT.md §4 — no comingling; cg<N> = codegen version so a new emitter
# build never reuses a stale DLL, old versions coexist). MUST match
# overlay_loader.c scan_cache_dir(). Pre-1.0: no legacy fallback.
cg = codegen_ver(args.runtime_include)
ch = codegen_hash(args.runtime_include)
gh = overlay_config_hash(args.recompiler, args.game_toml)
cache_dir = os.path.join(args.out_dir, game_id, args.compiler, cache_arch_abi(),
cache_tag(args.runtime_include, args.recompiler,
args.game_toml, args.flavor))
os.makedirs(cache_dir, exist_ok=True)
print(f'Cache dir: {cache_dir} '
f'(codegen ver {cg}, emitter {ch:08x}, config {gh:08x})')
with open(args.captures) as f:
captures = json.load(f)
if args.only_region:
only_regions = {int(value, 0) & 0x1FFFFFFF
for value in args.only_region}
captures = [cap for cap in captures
if (int(cap['load_addr'], 16) & 0x1FFFFFFF) in only_regions]
if not captures:
ap.error('--only-region did not match any capture')
# One view per (capture, segment with entries), §5.7. A capture whose
# segment record is malformed is reported and not compiled.
segment_view_errors = []
views = []
for cap in captures:
try:
views.extend(capture_segment_views(
cap, declared_view_segments(cap, toml, forced_interiors)))
except (KeyError, TypeError, ValueError, RuntimeError) as exc:
segment_view_errors.append(
(f'overlay {cap.get("load_addr", "?")}', str(exc)))
if len(views) != len(captures) or segment_view_errors:
by_seg = Counter(SEGMENT_NAMES[segment_of(int(v['load_addr'], 16))]
for v in views)
print(f'Segment views: {len(captures)} capture(s) -> {len(views)} '
'view(s) (' + ', '.join(f'{n} {name}' for name, n in
sorted(by_seg.items())) + ')')
captures = views
print(f'Captures: {len(captures)} overlay(s) to process\n')
# B-2 static mode: accumulate privately-namespaced generated C plus exact
# per-function identities for the content-validated dispatcher.
static_parts = []
static_requested_entries = set()
static_entry_sources = {}
# overlay-cache v2: per-region_start function-identity coverage, so a capture
# that adds no NEW (entry, code_crc) skips the gcc compile (volatile-data
# redundant-build elimination). Lazily loaded from existing .ranges, kept warm
# in-memory and updated as we build, so repeats within one run also dedup.
region_coverage_cache = {} # phys_addr -> set((ev, code_crc))
cov_lock = threading.Lock() # guards region_coverage_cache + its sets
# Per-region info for the post-loop interior-entry fragment pass (decoupled
# from region-compile success): (phys, load_addr, size, data, interior_pcs,
# executed_pcs). Collected right after classification so it survives a region
# whose own compile is skipped or audit-fails.
interior_frag_jobs = []
pending_empty_primary = []
def _merge_static_result(res):
"""Fold one static_capture_job result into the shared accumulators.
Called in canonical capture order for both the sequential and the
--jobs path, so static_parts / entry sources are order-identical."""
if res['log']:
print(res['log'], end='')
static_requested_entries.update(res['requested_entries'])
static_entry_sources.update(res['entry_sources'])
if res['outcome'] == 'ok':
static_parts.append(res['part'])
stats.add_ok()
elif res['outcome'] == 'skip':
stats.add_skip()
else:
cls, detail = res['fail']
stats.add_fail(res['label'], cls, detail)
# Per-capture body, extracted so the region-parallel driver below can call
# it. All shared state is either read-only closure (args/toml/cache_dir)
# or passed per-region (region_coverage_cache / interior_frag_jobs), so a
# worker owning a region owns every mutable it touches.
def _do_capture(cap, region_coverage_cache, interior_frag_jobs, stats):
# A capture view compiles in its own segment (§5.7): helpers that name
# PCs spell them in it, and its shard lives in that segment's cache
# directory.
with image_segment(segment_of(int(cap['load_addr'], 16))):
_do_capture_in_segment(cap, region_coverage_cache,
interior_frag_jobs, stats)
def _do_capture_in_segment(cap, region_coverage_cache, interior_frag_jobs,
stats):
load_addr = int(cap['load_addr'], 16)
size = int(cap['size'])
data = base64.b64decode(cap['bytes_b64'])
crc32 = binascii.crc32(data) & 0xFFFFFFFF
phys_addr = (load_addr & 0x1FFFFFFF)
_label = f'overlay 0x{load_addr:08X} crc {crc32:08X}'
# Trailing delay-slot guard words appended by the capture writer.
# Declared to the recompiler through the PS-EXE analysis-bound tag so
# they stay readable but are never discovered as code.
guard_bytes = capture_guard_bytes(cap, size, _label)
if args.static:
# Static mode is one self-contained job per capture (see
# static_capture_job); the sequential and --jobs paths share it.
_merge_static_result(static_capture_job(
cap, args, toml, forced_interiors, static_out))
return
seg_cache_dir = segment_cache_dir(cache_dir, segment_of(load_addr))
# Coverage is per segment: each segment's shards are their own.
coverage_key = (segment_of(load_addr), phys_addr)
# Reclassify prior F entry addresses from an identical image. Callable
# entries may become current roots; other entries re-enter as dispatch
# candidates and require a current exact host. Never reuse prior R host
# ranges: later roots can repartition the same bytes incompatibly.
if not args.static:
ranges_name = f'{phys_addr:08X}_{crc32:08X}.ranges'
prior_ranges_path = os.path.join(seg_cache_dir, ranges_name)
if os.path.exists(prior_ranges_path):
prior_entries = []
with open(prior_ranges_path) as pf:
for ln in pf:
parts = ln.split()
if parts and parts[0] == 'F':
try:
prior_entries.append(int(parts[1], 16))
except (IndexError, ValueError):
pass
if prior_entries:
fe = _parse_addr_list(cap.get('function_entry_pcs', []))
de = _parse_addr_list(cap.get('dispatch_entry_pcs', []))
for a in prior_entries:
if _callable_legacy_seed(data, load_addr, a):
fe.add(a)
else:
de.add(a)
cap['function_entry_pcs'] = sorted(fe)
cap['dispatch_entry_pcs'] = sorted(de)
cap.setdefault('schema', 'merged')
print(f' merged {len(prior_entries)} prior-manifest entries '
f'from {prior_ranges_path}')
seeds, seed_audit = classify_overlay_seeds(cap, data, load_addr, size,
crc32, toml)
this_ids = None # region func-ids once recompiled (None if skipped early)
# Record this region's executed dispatch-proven PCs for the decoupled
# fragment pass (runs after the loop, regardless of this region's
# compile outcome). Interiors AND callable dispatch roots both go in:
# a callable root can be starved by the DLL-already-exists skip when
# an OLDER capture of the same image bytes built this region's DLL
# before the PC became dispatch-proven (same filename, stale entry
# set — the 0x80024548 class), and the fragment pass is the demand-
# driven recovery path for exactly that.
if not args.static:
# A manual exact demand must not depend on the capture still
# retaining executed_pcs or on today's classifier assigning the PC
# DISPATCH_INTERIOR. That provenance is precisely what
# --force-interior restores. Keep it isolated from the shared
# region seed set and queue the fragment directly.
fragment_job = make_interior_fragment_job(
phys_addr, load_addr, size, data, seed_audit,
forced_interiors, cap)
if fragment_job is not None:
interior_frag_jobs.append(fragment_job)
if not args.static:
dll_path = os.path.join(seg_cache_dir,
f'{phys_addr:08X}_{crc32:08X}{overlay_ext()}')
print(f'Overlay load=0x{load_addr:08X} size={size} crc32=0x{crc32:08X}'
+ (f' guard={guard_bytes}B (delay-slot only, not analysed)'
if guard_bytes else ''))
print(f' seeds: {len(seeds)} dll: {dll_path}')
print_seed_audit(seed_audit)
demanded_root_entries = walk_root_seed_entries(seeds)
root_seeds = [
seed for seed in seeds
if ((seed.split()[0] in ('call_root', 'dispatch_root')) or
seed.split()[0].startswith('0x'))
]
if not root_seeds:
print(f' {NO_SHARED_WALK_ROOT_SEEDS_SKIP}\n')
stats.add_skip()
return
# Cheap evidence-aware cache gate. A valid existing pair is not enough:
# a later capture can add roots for identical bytes. Skip reanalysis only
# when runtime-callable identities matching THIS byte variant already
# serve every current walk-root demand. The fragment job was queued
# above and still repairs any exact interior/static demand independently.
if not args.static and not args.force:
expected_abi = overlay_abi_tag(
args.runtime_include, args.flavor)
if cap.get('producer') == BIOS_RESIDENT_PRODUCER:
# A resident fast-path needs one canonical preloadable bundle;
# aggregate fragment coverage is insufficient because the
# sidecar names exactly one DLL stem.
current_entries, _current_ranges = (
current_variant_func_id_coverage(
load_shard_func_ids(dll_path, expected_abi),
data, load_addr, size))
else:
current_entries, _current_ranges = (
load_region_current_variant_coverage(
seg_cache_dir, phys_addr, data, load_addr, size,
expected_abi))
if demanded_root_entries <= {
_canonical_guest_addr(entry)
for entry in current_entries}:
reconcile_bios_resident_marker(dll_path, cap, False)
print(f' SKIP: runtime-valid current-byte cache serves all '
f'{len(demanded_root_entries)} walk-root demand(s)\n')
stats.add_skip()
return
with tempfile.TemporaryDirectory() as tmp:
# Write fake PS-EXE. The header entry PC becomes a walk root in the
# recompiler, so it must be a walk-root seed — never an 'interior'
# one. Root seed lines are either '0x...' or 'dispatch_root 0x...'.
entry_pc = int(root_seeds[0].split()[-1], 16)
psx_path = os.path.join(tmp, f'overlay_{load_addr:08X}.psx')
with open(psx_path, 'wb') as f:
f.write(make_psxexe(load_addr, entry_pc, data,
guard_bytes=guard_bytes))
# Write seeds file
seeds_path = os.path.join(tmp, 'seeds.txt')
with open(seeds_path, 'w') as f:
for s in seeds:
f.write(s + '\n')
out_dir_tmp = os.path.join(tmp, 'out')
os.makedirs(out_dir_tmp)
# Run psxrecomp-game in --overlay mode (always, for every overlay
# input). Evidence-scoped discovery: compile only the proven entry
# seeds and the code reachable from them; never whole-byte sweep
# (which decodes embedded data tables as code). Branch/jump-table
# targets stay as in-parent labels, not standalone functions. This
# is the overlay-compilation contract, not a tunable.
cmd = [args.recompiler, psx_path,
'--seeds', seeds_path,
'--out-dir', out_dir_tmp,
'--overlay',
# Forward the [widescreen] site lists so overlay-resident
# emits (backdrop screenX squash, and any sprite-tag/cull
# sites that resolve into overlay code) are applied. --ws-config
# only adopts the widescreen lists, not the game's exe/paths.
'--ws-config', os.path.abspath(args.game_toml)]
cmd += recompiler_project_root_args(args)
print(f' recompile: {args.recompiler} ...{" [CPS]" if args.cps else ""}')
toml_dir = os.path.dirname(os.path.abspath(args.game_toml))
sub_env = dict(os.environ)
if args.cps:
sub_env['PSX_CPS'] = '1' # §25: emit continuation-passing overlay C
r = subprocess.run(cmd, capture_output=True, text=True,
cwd=toml_dir, env=sub_env)
if r.returncode != 0:
print(f' RECOMPILER ERROR:\n{r.stderr or r.stdout}')
stats.add_fail(_label, 'recompiler', r.stderr or r.stdout)
return
# Find the generated _full.c
stem = os.path.basename(psx_path)
full_c = os.path.join(out_dir_tmp, stem + '_full.c')
if not os.path.exists(full_c):
# psxrecomp-game uses filename() not stem()
candidates = [f for f in os.listdir(out_dir_tmp) if f.endswith('_full.c')]
if not candidates:
print(f' ERROR: no _full.c in {out_dir_tmp}')
stats.add_fail(_label, 'no_output', 'no _full.c emitted')
return
full_c = os.path.join(out_dir_tmp, candidates[0])
with open(full_c) as f:
src = f.read()
ranges_src = None
for fn in os.listdir(out_dir_tmp):
if fn.endswith('_full.ranges'):
ranges_src = os.path.join(out_dir_tmp, fn)
break
# Post-process (DLL path; static mode returned early above)
src = patch_generated_c(src, load_addr, size)
c_audit = audit_generated_c(src, load_addr, size, crc32, toml)
print_generated_c_audit(load_addr, size, crc32, c_audit)
# Always save the debug copy for inspection — including on audit
# failure, so opcode gaps / boundary artifacts can be classified.
os.makedirs(os.path.dirname(dll_path), exist_ok=True)
debug_c = os.path.join(os.path.dirname(dll_path),
f'{crc32:08X}_patched.c')
with open(debug_c, 'w') as f:
f.write(src)
if c_audit['unknown_bad'] or c_audit['unsupported_todo_addrs']:
fallback = optional_enrichment_fallback_capture(cap)
if fallback is not None:
print(' OPTIONAL ENRICHMENT AUDIT REJECTED; '
'retrying conservative recipe\n')
_do_capture(fallback, region_coverage_cache,
interior_frag_jobs, stats)
return
print(' GENERATED-C AUDIT FAILED\n')
stats.add_fail(_label, 'audit',
f'{len(c_audit["unknown_bad"])} unknown_bad, '
f'{len(c_audit["unsupported_todo_addrs"])} unsupported')
return
patched_c = os.path.join(tmp, 'overlay_patched.c')
with open(patched_c, 'w') as f:
f.write(src)
# overlay-cache v2 dedup: compute this capture's per-function
# identity set BEFORE the (expensive) gcc compile. The loader
# content-matches each function by (entry, code_crc) across ALL
# DLLs at this region_start, so if every function we'd produce is
# already provided by an existing DLL, a new DLL adds nothing —
# skip the build. This is what stops volatile-data regions (a
# changing whole-region CRC over byte-identical code) from minting
# an endless pile of redundant DLLs.
this_ids = (parse_overlay_func_ids(ranges_src, data, load_addr, size)
if ranges_src else [])
delay_errors = audit_func_id_delay_slots(
this_ids, data, load_addr)
if delay_errors:
entry, pc, reason = delay_errors[0]
where = f' at 0x{pc:08X}' if pc is not None else ''
print(f' DELAY-SLOT IDENTITY AUDIT FAILED: '
f'func 0x{entry:08X}{where}: {reason}\n')
stats.add_fail(_label, 'delay_slot_identity', reason)
return
this_set = {(ev, crc) for ev, crc, _ in this_ids}
with cov_lock:
covered = region_coverage_cache.get(coverage_key)
if covered is None:
covered = load_region_coverage(
seg_cache_dir, phys_addr,
overlay_abi_tag(args.runtime_include, args.flavor))
region_coverage_cache[coverage_key] = covered
fully_covered = (bool(this_set) and this_set <= covered
and not args.force
and cap.get('producer') !=
BIOS_RESIDENT_PRODUCER)
if fully_covered:
print(f' SKIP: all {len(this_set)} function(s) already '
f'covered by existing DLL(s) at this region — no new '
f'native code to build\n')
stats.add_skip()
return
if not this_ids:
print(' WARNING: recompiler emitted no usable _full.ranges -- '
'preserving any prior DLL/ranges pair; checking exact '
'fragment coverage after supplementation')
if cap.get('producer') == BIOS_RESIDENT_PRODUCER:
stats.add_fail(_label, 'no_ranges',
'resident producer requires its canonical bundle')
else:
pending_empty_primary.append((
_label, phys_addr, load_addr, size, data,
demanded_root_entries))
return
missing_exports = {
entry for entry, _crc, _ranges in this_ids
if entry not in c_audit['defs']
}
if missing_exports:
detail = ', '.join(
f'0x{entry:08X}' for entry in sorted(missing_exports))
print(' GENERATED-C/MANIFEST EXPORT AUDIT FAILED: '
f'{detail}\n')
stats.add_fail(
_label, 'manifest_exports',
f'{len(missing_exports)} manifest entries lack C definitions')
return
representability_error = validate_overlay_func_ids(
this_ids, 'region')
if representability_error:
print(' RANGE MANIFEST AUDIT FAILED: '
f'{representability_error}\n')
stats.add_fail(
_label, 'manifest_ranges', representability_error)
return
pair_id = overlay_pair_id(src, this_ids)
src = add_overlay_pair_export(src, pair_id)
# Retained source is the exact source compiled into the DLL.
with open(patched_c, 'w') as f:
f.write(src)
# Compile to DLL
include_dirs = [args.runtime_include]
recomp_root = os.path.dirname(os.path.dirname(args.recompiler))
for lib_inc in ['lib/fmt/include']:
p = os.path.join(recomp_root, lib_inc)
if os.path.isdir(p):
include_dirs.append(p)
publication = {}
success = compile_dll(patched_c, dll_path, include_dirs,
gcc=args.gcc, flavor=args.flavor,
compiler=args.compiler, tcc=args.tcc,
func_ids=this_ids,
pair_id=pair_id,
preserve_existing_pair=(
not args.force and
cap.get('producer') !=
BIOS_RESIDENT_PRODUCER),
expected_existing_abi=overlay_abi_tag(
args.runtime_include, args.flavor),
publication_result=publication,
candidate_cap=overlay_candidate_cap(
args.runtime_include))
if success:
# compile_dll transactionally published the per-entry range
# manifest beside the DLL from the same func-id list used by
# dedup. The loader keys it by the same filename stem.
ranges_out = os.path.splitext(dll_path)[0] + '.ranges'
if this_ids:
nfn = len(this_ids)
published = publication.get('published', True)
reconcile_bios_resident_marker(
dll_path, cap, published)
print(f' ranges: {nfn} functions -> {ranges_out}')
# New identities are now available for this region_start;
# keep the warm coverage set current so later captures in
# this same run dedup against them. (Parallel note: the
# check→build→update window is deliberately unlocked, so
# two concurrent captures can both build overlapping DLLs.
# That is redundancy, not corruption — the loader content-
# matches every function by (entry, code_crc) across all
# DLLs at a region.)
with cov_lock:
covered |= load_region_coverage(
seg_cache_dir, phys_addr,
overlay_abi_tag(
args.runtime_include, args.flavor))
if published:
print(f' OK -> {dll_path}\n')
stats.add_ok()
else:
print(f' PRESERVED concurrent valid pair -> '
f'{dll_path}\n')
stats.add_skip()
else:
print(' WARNING: recompiler emitted no _full.ranges — '
'loader will leave this region to the interpreter')
# A DLL with no .ranges is dead weight: the loader has no
# per-function identities to dispatch, so the region stays
# interpreted. Tally it as a failure, not a silent "OK".
stats.add_fail(_label, 'no_ranges',
'DLL built but no _full.ranges (undispatchable)')
else:
if publication.get('capacity_error'):
print(' CACHE CAPACITY REACHED; shard left to the '
'interpreter\n')
stats.add_skip()
return
fallback = optional_enrichment_fallback_capture(cap)
if fallback is not None:
print(' OPTIONAL ENRICHMENT COMPILE REJECTED; '
'retrying conservative recipe\n')
_do_capture(fallback, region_coverage_cache,
interior_frag_jobs, stats)
return
print(f' FAILED\n')
stats.add_fail(_label, 'compile',
'gcc/tcc compile failed (see COMPILE ERROR above)')
# Interior-entry "island" fragments (overlay-cache v2): the FMV-driver class.
# Run as a SEPARATE pass AFTER all region compiles, so it is DECOUPLED from
# region success — an executed orphan interior gets its isolated island shard
# even if its region's trusted compile failed/audit-failed (that is the whole
# point of the separate failure domain). For each region, retain validated
# dispatch PCs even when shared CFG ownership dropped their interior seeds.
# Select those that ACTUALLY EXECUTED but that NO current DLL serves (orphan
# interiors — host never discovered, so the region can't alias them), and
# compile each as its OWN isolated <region>_<key>.dll that ENTERS at the
# interior PC (recovers no host). Isolated => a bad fragment fails alone and
# never poisons a region's trusted DLL.
def _do_frags(interior_frag_jobs, stats):
# Fragments compile per segment, like region shards (§5.7): a job's
# PCs, its shards and their directory are its capture view's segment.
# KSEG0 always runs (as before); the others only when they have jobs.
by_seg = {}
for job in interior_frag_jobs:
by_seg.setdefault(segment_of(job['load_addr']), []).append(job)
for seg in [KSEG0] + [s for s in (KUSEG, KSEG1) if s in by_seg]:
with image_segment(seg):
_do_frags_in_segment(by_seg.get(seg, []), stats,
segment_cache_dir(cache_dir, seg))
def _do_frags_in_segment(interior_frag_jobs, stats, cache_dir):
frag_env = dict(os.environ)
if args.cps:
frag_env['PSX_CPS'] = '1'
# Persistent doomed-interior memo: skip interiors already proven to fail
# audit from IDENTICAL bytes, so autocompile stops re-walking data-as-code
# every cycle. --force ignores the skip (re-attempts) for triage/debug.
fail_memo = load_interior_fail_memo(cache_dir)
memo_skipped = 0
variant_coverage_cache = {}
expected_abi = overlay_abi_tag(args.runtime_include, args.flavor)
runtime_candidate_cap = overlay_candidate_cap(args.runtime_include)
capacity_probe = os.path.join(
cache_dir, f'00000000_00000000{overlay_ext()}')
candidate_capacity_exhausted, existing_candidates = \
cache_candidate_capacity_full(
capacity_probe, expected_abi, runtime_candidate_cap)
if candidate_capacity_exhausted:
print(f' overlay candidate capacity already full: '
f'{existing_candidates}/{runtime_candidate_cap}; '
'new fragments stay interpreted')
invocation_recipe = (
expected_abi, int(args.flavor), bool(args.cps), args.compiler,
hashlib.sha256(json.dumps(
toml, sort_keys=True, separators=(',', ':'),
default=str).encode('utf-8')).hexdigest(),
)
merged_jobs = merge_fragment_jobs_by_recipe(
interior_frag_jobs, invocation_recipe)
if len(merged_jobs) != len(interior_frag_jobs):
print(f' fragment recipes: merged {len(interior_frag_jobs)} capture '
f'job(s) into {len(merged_jobs)} byte-variant recipe(s)')
# A namespace that is already saturated cannot admit ordinary fragment
# growth. Avoid the expensive cross-variant donor/owner proof at this
# true fixed point, but preserve the only failure semantics that do not
# require that proof: a required executed demand already memoized as a
# deterministic compile failure remains loud. --force and explicit
# --force-interior work bypass this shortcut because a same-basename
# replacement can be capacity-neutral.
if fragment_capacity_fastpath_eligible(
candidate_capacity_exhausted, args.force, merged_jobs):
for job in merged_jobs:
memo_entries = {}
for entry in sorted(job['candidates']):
key = _interior_fail_key(
job['phys_addr'], entry, job['data'],
job['load_addr'], job['size'],
job['producer_ranges'], job['cross_call_allow'],
expected_abi, args.cps, toml,
job['guard_bytes'])
if key in fail_memo:
memo_entries[entry] = fail_memo[key]
if not memo_entries:
continue
current_entries, _current_ranges = \
load_region_current_variant_coverage(
cache_dir, job['phys_addr'], job['data'],
job['load_addr'], job['size'], expected_abi)
for entry, reason in memo_entries.items():
if (entry & 0x1FFFFFFF) in current_entries:
continue
memo_action = interior_fail_memo_action(
entry, job, args.force)
if memo_action == 'skip':
memo_skipped += 1
stats.add_skip()
else:
stats.add_fail(
f'interior 0x{entry:08X} @region '
f'0x{job["phys_addr"]:08X}',
'fragment_memo', reason)
stats.add_capacity_fastpath(len(merged_jobs))
print(f' fragment capacity fixed point: skipped donor/owner '
f'analysis for {len(merged_jobs)} byte-variant recipe job(s); '
'the exact unserved hosted-demand count was intentionally '
'not recomputed')
if memo_skipped:
print(f' interior fail-memo: skipped {memo_skipped} '
'known-doomed optional interior(s)')
return
# Global phase 1: publish every strong play-free root before freezing
# sibling donor authority. Freezing first deferred these roots until a
# second CLI invocation: the next run reloaded them from disk and could
# then nominate hosted interiors that a clean first run missed. Strong
# roots are the complete organic authority for cross-variant donors, so
# closing them for every recipe first makes an empty-cache invocation
# reproducible without allowing hosted aliases to recurse.
strong_handled_by_job = []
for job in merged_jobs:
phys_addr = job['phys_addr']
load_addr = job['load_addr']
size = job['size']
data = job['data']
current_variant_ids = load_region_current_variant_func_ids(
cache_dir, phys_addr, data, load_addr, size, expected_abi,
include_supplemental=False)
current_variant_entries, current_variant_ranges = \
current_variant_func_id_coverage(
current_variant_ids, data, load_addr, size)
strong_handled = set()
batch_candidates, isolated_candidates = \
partition_strong_root_demands(
job['static_exact_demands'], job['executed'],
job['forced'], job['static_interval_demands'],
current_variant_entries)
strong_candidates = batch_candidates + isolated_candidates
batch_pending = []
isolated_pending = []
isolated_candidate_set = set(isolated_candidates)
for entry in strong_candidates:
key = _interior_fail_key(
phys_addr, entry, data, load_addr, size,
job['producer_ranges'], job['cross_call_allow'],
expected_abi, args.cps, toml,
job['guard_bytes'])
memo_action = interior_fail_memo_action(
entry, job, args.force)
if key in fail_memo and memo_action != 'retry':
strong_handled.add(entry)
if memo_action == 'skip':
memo_skipped += 1
stats.add_skip()
else:
stats.add_fail(
f'interior 0x{entry:08X} @region '
f'0x{phys_addr:08X}',
'fragment_memo', fail_memo[key])
continue
if entry in isolated_candidate_set:
isolated_pending.append(entry)
else:
batch_pending.append(entry)
if candidate_capacity_exhausted:
capacity_pending = [
entry for entry in batch_pending + isolated_pending
if not fragment_capacity_allows_entry(
True, args.force, job['forced'], entry)
]
if capacity_pending:
print(f' strong-root supplement @0x{phys_addr:08X}: '
f'{len(capacity_pending)} demand(s) left to '
'interpreter: candidate capacity full')
strong_handled.update(capacity_pending)
for _entry in capacity_pending:
stats.add_skip()
capacity_pending_set = set(capacity_pending)
batch_pending = [
entry for entry in batch_pending
if entry not in capacity_pending_set
]
isolated_pending = [
entry for entry in isolated_pending
if entry not in capacity_pending_set
]
if not batch_pending and not isolated_pending:
strong_handled_by_job.append(strong_handled)
continue
def compile_strong_batch(entries):
return compile_fragment_batch(
entries, data, load_addr, size, phys_addr, cache_dir,
args, frag_env, toml, job['producer_ranges'],
job['cross_call_allow'],
guard_bytes=job['guard_bytes'])
def strong_batch_success(entries, frag_ids, status):
strong_handled.update(entries)
if status == 'cached':
stats.add_skip()
else:
stats.add_ok()
for ev, _cc, ranges in frag_ids:
current_variant_entries.add(ev & 0x1FFFFFFF)
current_variant_ranges.extend(
((start & 0x1FFFFFFF),
(start & 0x1FFFFFFF) + length)
for start, length in ranges)
current_variant_ids.extend(frag_ids)
if job.get('resident_cap'):
fragment_dll = os.path.join(
cache_dir,
f'{phys_addr:08X}_{fragment_shard_key(frag_ids):08X}'
f'{overlay_ext()}')
reconcile_bios_resident_marker(
fragment_dll, job['resident_cap'], status == 'built')
def strong_singleton_failure(entry, status):
nonlocal candidate_capacity_exhausted
strong_handled.add(entry)
if status.startswith('candidate-capacity:'):
if status.startswith('candidate-capacity: full'):
candidate_capacity_exhausted = True
print(f' fragment 0x{entry:08X} skipped safely: '
f'{status}')
stats.add_skip()
return
key = _interior_fail_key(
phys_addr, entry, data, load_addr, size,
job['producer_ranges'], job['cross_call_allow'],
expected_abi, args.cps, toml,
job['guard_bytes'])
optional_reject = optional_static_fragment_rejection(
entry, job, status)
if optional_reject:
print(f' static fragment 0x{entry:08X} rejected safely: '
f'{status}')
stats.add_skip()
else:
stats.add_fail(
f'interior 0x{entry:08X} @region '
f'0x{phys_addr:08X}', 'fragment', status)
if interior_failure_is_deterministic(status):
fail_memo[key] = status
append_interior_fail_memo(cache_dir, key, status)
def strong_batch_failure(entries, status):
nonlocal candidate_capacity_exhausted
if status.startswith('candidate-capacity:'):
strong_handled.update(entries)
if status.startswith('candidate-capacity: full'):
candidate_capacity_exhausted = True
for _entry in entries:
stats.add_skip()
print(f' strong-root batch: {len(entries)} demand(s) '
f'left to interpreter: {status}')
return
strong_handled.update(entries)
stats.add_fail(
f'strong-root batch ({len(entries)} entries) '
f'@region 0x{phys_addr:08X}',
'fragment_batch', status)
strong_shards = compile_batched_fragment_roots(
batch_pending, compile_strong_batch, strong_batch_success,
strong_singleton_failure,
lambda entry: ((entry & 0x1FFFFFFF) not in
current_variant_entries),
fragment_batch_failure_is_partitionable,
strong_batch_failure)
# A root that is also live/forced/interval evidence retains its
# historical singleton failure domain, but still closes before the
# donor snapshot. A successful batch may incidentally cover it.
for entry in isolated_pending:
if not fragment_capacity_allows_entry(
candidate_capacity_exhausted, args.force,
job['forced'], entry):
if entry not in strong_handled:
strong_handled.add(entry)
stats.add_skip()
continue
if (entry & 0x1FFFFFFF) in current_variant_entries:
strong_handled.add(entry)
continue
frag_ids, status = compile_strong_batch([entry])
if frag_ids:
strong_batch_success([entry], frag_ids, status)
strong_shards += 1
else:
strong_singleton_failure(entry, status)
if strong_candidates:
print(f' strong-root supplement @0x{phys_addr:08X}: '
f'{len(strong_candidates)} demand(s) -> '
f'{strong_shards} batched shard(s)')
strong_handled_by_job.append(strong_handled)
# A strong-root publication can consume the last Candidate slots.
# Re-probe under the shared namespace lock before hosted admission.
# The outer invocation lock excludes cooperating writers for the rest
# of this pass; an uncoordinated external deletion can only cause safe
# coverage loss until the next invocation.
was_capacity_exhausted = candidate_capacity_exhausted
candidate_capacity_exhausted, existing_candidates = \
cache_candidate_capacity_full(
capacity_probe, expected_abi, runtime_candidate_cap)
if candidate_capacity_exhausted and not was_capacity_exhausted:
print(f' overlay candidate capacity reached after strong-root '
f'closure: {existing_candidates}/{runtime_candidate_cap}; '
'new fragments stay interpreted')
# Global phase 2: snapshot native coverage and authority separately.
# Coverage includes prior hosted shards so a valid alias is not rebuilt.
# Donor/owner authority excludes provenance-tagged hosted output, whose
# incidental rooted callees must not manufacture another wave on a later
# invocation. Strong roots above reconstruct durable authority first.
initial_variant_ids = [
load_region_current_variant_func_ids(
cache_dir, job['phys_addr'], job['data'], job['load_addr'],
job['size'], expected_abi)
for job in merged_jobs
]
initial_authority_ids = [
load_region_current_variant_func_ids(
cache_dir, job['phys_addr'], job['data'], job['load_addr'],
job['size'], expected_abi, include_supplemental=False)
for job in merged_jobs
]
# Build sorted sibling static-donor streams once. Each recipe needs at
# most its maximum current-entry population plus one nomination batch:
# any later address necessarily has HOSTED_NOMINATION_CAP earlier
# entries from that same sibling after a recipient filters its current
# coverage. The recipient heap-merge below applies the final cap only
# after excluding self evidence and already native entries.
hosted_donors_by_region = {}
initial_entry_bound_by_region = Counter()
for index, indexed_job in enumerate(merged_jobs):
initial_entry_bound_by_region[indexed_job['phys_addr']] = max(
initial_entry_bound_by_region[indexed_job['phys_addr']],
len({_canonical_guest_addr(entry)
for entry, _crc, _ranges in
initial_authority_ids[index]}))
hosted_donor_index_drops = Counter()
for sibling_index, sibling in enumerate(merged_jobs):
organic_entries = organic_root_entries(
initial_authority_ids[sibling_index])
donor_entries = sorted({
_canonical_guest_addr(entry)
for entry in sibling.get('hosted_donor_demands', set())
if _canonical_guest_addr(entry) in organic_entries
})
# A recipient can filter no more than its current exact entry set.
# Normal publication bounds that by the runtime cap; include the
# observed on-disk maximum as well so a legacy/manual over-cap cache
# loses coverage safely without invalidating this prefix proof.
donor_index_limit = (
max(runtime_candidate_cap,
initial_entry_bound_by_region[sibling['phys_addr']]) +
HOSTED_NOMINATION_CAP)
dropped = max(0, len(donor_entries) - donor_index_limit)
if dropped:
hosted_donor_index_drops[sibling['phys_addr']] += dropped
hosted_donors_by_region.setdefault(
sibling['phys_addr'], {})[sibling_index] = tuple(
donor_entries[:donor_index_limit])
for job_index, job in enumerate(merged_jobs):
phys_addr = job['phys_addr']
load_addr = job['load_addr']
size = job['size']
data = job['data']
interior_pcs = set(job['candidates'])
executed = job['executed']
static_exact_demands = job['static_exact_demands']
static_interval_demands = job['static_interval_demands']
forced = job['forced']
region_crc = binascii.crc32(data) & 0xFFFFFFFF
# ENTRY-based orphan test, not range-based: native code is
# enterable only at manifest F entries, so "inside a compiled
# range" does NOT make a dispatch target servable — a range-
# covered PC with no F entry anywhere still interps its whole
# chain on every dispatch. Demand an entry at exactly this PC.
coverage_key = (phys_addr, hashlib.sha256(data).digest(),
load_addr, size)
current_coverage = variant_coverage_cache.get(coverage_key)
current_variant_ids = list(initial_variant_ids[job_index])
current_authority_ids = list(
initial_authority_ids[job_index])
current_authority_entries, _authority_ranges = \
current_variant_func_id_coverage(
current_authority_ids, data, load_addr, size)
if current_coverage is None:
current_coverage = current_variant_func_id_coverage(
current_variant_ids, data, load_addr, size)
variant_coverage_cache[coverage_key] = current_coverage
current_variant_entries, current_variant_ranges = current_coverage
def record_fragment_success(frag_ids, status,
provenance=None):
if status == 'cached':
stats.add_skip()
else:
stats.add_ok()
for ev, _cc, ranges in frag_ids:
current_variant_entries.add(ev & 0x1FFFFFFF)
current_variant_ranges.extend(
((start & 0x1FFFFFFF),
(start & 0x1FFFFFFF) + length)
for start, length in ranges)
current_variant_ids.extend(frag_ids)
if job.get('resident_cap'):
fragment_dll = os.path.join(
cache_dir,
f'{phys_addr:08X}_'
f'{fragment_shard_key(frag_ids, provenance):08X}'
f'{overlay_ext()}')
reconcile_bios_resident_marker(
fragment_dll, job['resident_cap'], status == 'built')
def record_fragment_failure(entry, status):
nonlocal candidate_capacity_exhausted
if status.startswith('candidate-capacity:'):
if status.startswith('candidate-capacity: full'):
candidate_capacity_exhausted = True
print(f' fragment 0x{entry:08X} skipped safely: {status}')
stats.add_skip()
return
key = _interior_fail_key(
phys_addr, entry, data, load_addr, size,
job['producer_ranges'], job['cross_call_allow'],
expected_abi, args.cps, toml,
job['guard_bytes'])
optional_reject = optional_static_fragment_rejection(
entry, job, status)
if optional_reject:
print(f' static fragment 0x{entry:08X} rejected safely: '
f'{status}')
stats.add_skip()
else:
stats.add_fail(
f'interior 0x{entry:08X} @region 0x{phys_addr:08X}',
'fragment', status)
if interior_failure_is_deterministic(status):
fail_memo[key] = status
append_interior_fail_memo(cache_dir, key, status)
strong_handled = set(strong_handled_by_job[job_index])
# Global phase 3: a static root proven in one captured byte recipe
# can identify an
# indirect-switch entry in a sibling recipe. Run this after local
# strong closure, so owner availability is independent of the
# poorer base-region bundle. Donor and owner authority remain
# frozen to the post-strong snapshot and cannot recurse through a
# hosted shard. Sibling bytes only nominate the address; all
# executable authority comes from this recipient's current-byte
# owner and publication audit.
sibling_proofs, sibling_proofs_truncated = \
recipient_sibling_proofs(
hosted_donors_by_region.get(phys_addr, {}), job_index,
current_authority_entries)
hosted_audit = {}
hosted = select_hosted_interior_demands(
job, sibling_proofs, current_authority_ids, hosted_audit)
if sibling_proofs_truncated:
hosted_audit['region_nomination_cap_truncated'] = 1
donor_index_drops = hosted_donor_index_drops.get(phys_addr, 0)
if donor_index_drops:
hosted_audit['region_donor_index_cap_dropped'] = \
donor_index_drops
def compile_hosted_batch(specs):
return compile_fragment_batch(
specs, data, load_addr, size, phys_addr, cache_dir,
args, frag_env, toml, job['producer_ranges'],
job['cross_call_allow'], hosted_owners=specs,
guard_bytes=job['guard_bytes'])
hosted_rejected = set()
hosted_published = False
def hosted_success(batch, frag_ids, status):
nonlocal hosted_published
record_fragment_success(
frag_ids, status, HOSTED_MANIFEST_PROVENANCE)
if status == 'built':
hosted_published = True
emitted = {
_canonical_guest_addr(entry)
for entry, _crc, _ranges in frag_ids
}
missing = sorted(set(batch) - emitted)
if missing:
print(f' hosted batch: {len(batch) - len(missing)}/'
f'{len(batch)} target(s) passed recipient CFG proof; '
f'{len(missing)} retrying in smaller batches')
return set(batch) - set(missing)
def hosted_singleton_failure(entry, status):
nonlocal candidate_capacity_exhausted
print(f' hosted interior 0x{entry:08X} rejected safely: '
f'{status}')
if status.startswith('candidate-capacity: full'):
candidate_capacity_exhausted = True
if args.force:
stats.add_skip()
return
hosted_rejected.add(_canonical_guest_addr(entry))
if status.startswith((
'generated-c-audit:', 'hosted-entry-audit:',
'hosted-no-target:', 'candidate-capacity:',
'no-generated-output', 'no-func-ids')):
stats.add_skip()
else:
stats.add_fail(
f'hosted interior 0x{entry:08X} '
f'@region 0x{phys_addr:08X}',
'hosted_fragment', status)
def hosted_batch_failure(batch, status):
nonlocal candidate_capacity_exhausted
if status.startswith('candidate-capacity: full'):
candidate_capacity_exhausted = True
print(f' hosted batch: 0/{len(batch)} target(s) passed '
'recipient CFG proof; candidate capacity full')
if args.force:
for _entry in batch:
stats.add_skip()
return
hosted_rejected.update(
_canonical_guest_addr(entry) for entry in batch)
if status.startswith((
'hosted-no-target:',
'hosted-entry-audit: compile attempt cap',
'candidate-capacity:', 'no-generated-output',
'no-func-ids')):
for _entry in batch:
stats.add_skip()
print(f' hosted batch: 0/{len(batch)} target(s) passed '
'recipient CFG proof; rejected safely')
return
stats.add_fail(
f'hosted-interior batch ({len(batch)} entries) '
f'@region 0x{phys_addr:08X}',
'hosted_fragment_batch', status)
hosted_shards = compile_batched_hosted_aliases(
hosted, compile_hosted_batch, hosted_success,
hosted_singleton_failure,
lambda entry: ((entry & 0x1FFFFFFF) not in
current_variant_entries),
fragment_batch_failure_is_partitionable,
hosted_batch_failure,
stop_requested=lambda: (
candidate_capacity_exhausted and not args.force))
if hosted_published:
was_capacity_exhausted = candidate_capacity_exhausted
candidate_capacity_exhausted, existing_candidates = \
cache_candidate_capacity_full(
capacity_probe, expected_abi, runtime_candidate_cap)
if (candidate_capacity_exhausted and
not was_capacity_exhausted):
print(f' overlay candidate capacity reached after '
f'hosted closure: {existing_candidates}/'
f'{runtime_candidate_cap}; ordinary fragments stay '
'interpreted')
hosted_capacity_skipped = set()
if candidate_capacity_exhausted and not args.force:
already_accounted = hosted_rejected | {
_canonical_guest_addr(entry) for entry in strong_handled
}
capacity_missing = hosted_entries_still_missing(
hosted, current_variant_entries, already_accounted)
if capacity_missing:
hosted_capacity_skipped.update(capacity_missing)
hosted_audit['candidate_capacity_full'] = \
len(capacity_missing)
for _entry in capacity_missing:
stats.add_skip()
if sibling_proofs:
print(f' cross-variant hosted supplement @0x{phys_addr:08X}: '
f'{len(hosted)}/{len(sibling_proofs)} sibling-proven '
f'target(s) had a unique current-byte local owner -> '
f'{hosted_shards} shard(s); '
f'audit={dict(sorted(hosted_audit.items()))}')
orphans = select_fragment_orphans(
interior_pcs, executed, forced, static_exact_demands,
static_interval_demands, current_variant_entries,
current_variant_ranges)
orphans = [entry for entry in orphans
if (entry not in strong_handled and
_canonical_guest_addr(entry) not in
hosted_capacity_skipped)]
if not orphans:
continue
capacity_orphans = [
entry for entry in orphans
if not fragment_capacity_allows_entry(
candidate_capacity_exhausted, args.force,
job['forced'], entry)
]
if capacity_orphans:
print(f' orphan fragments: {len(capacity_orphans)} '
'demand(s) left to interpreter: candidate capacity full')
for _entry in capacity_orphans:
stats.add_skip()
capacity_orphan_set = set(capacity_orphans)
orphans = [
entry for entry in orphans
if entry not in capacity_orphan_set
]
if not orphans:
continue
built = 0
for a in orphans:
# A preceding orphan can consume the final slot or receive the
# first authoritative full verdict. Stop ordinary toolchain
# work immediately while still allowing a later explicit
# capacity-neutral repair candidate.
if not fragment_capacity_allows_entry(
candidate_capacity_exhausted, args.force,
job['forced'], a):
stats.add_skip()
continue
key = _interior_fail_key(
phys_addr, a, data, load_addr, size,
job['producer_ranges'], job['cross_call_allow'],
expected_abi, args.cps, toml,
job['guard_bytes'])
memo_action = interior_fail_memo_action(a, job, args.force)
if key in fail_memo and memo_action != 'retry':
# Deterministically doomed from these exact bytes — skip the
# ~16k-word data walk. NOT a fresh failure (already counted
# when first memoized); count as a skip so the summary is honest.
if memo_action == 'skip':
memo_skipped += 1
stats.add_skip()
else:
stats.add_fail(
f'interior 0x{a:08X} @region 0x{phys_addr:08X}',
'fragment_memo', fail_memo[key])
continue
frag_ids, status = compile_interior_fragment(
a, data, load_addr, size, phys_addr, cache_dir, args,
frag_env, toml, job['producer_ranges'],
job['cross_call_allow'],
guard_bytes=job['guard_bytes'])
if frag_ids:
built += 1
record_fragment_success(
frag_ids, status, ORPHAN_MANIFEST_PROVENANCE)
else:
# An executed orphan interior that would NOT build is a real
# coverage hole: that PC's whole dispatch chain runs
# interpreted forever. Tally it loudly with the reason.
# Persist only deterministic generated-code audit verdicts;
# compiler/spawn errors and cache-pair mismatches depend on
# mutable external state and must remain retryable.
record_fragment_failure(a, status)
print(f' interior fragments @0x{phys_addr:08X}: {built}/{len(orphans)} '
f'exact-demand orphan interior(s) -> isolated island shards')
if memo_skipped:
print(f' interior fail-memo: skipped {memo_skipped} known-doomed '
f'interior(s) (data-as-code from identical bytes; '
f'{INTERIOR_FAIL_MEMO})')
# ---- Drive the capture list -------------------------------------------
# Dynamic cache publication is deliberately canonical and sequential. A
# parallel recompile race is harmless below capacity but chooses an
# irreproducible winning subset at the hard Candidate boundary. Static
# combined-C mode has no runtime cache publication and keeps its historical
# capture order.
stats = ShardStats()
for label, detail in segment_view_errors:
print(f' SEGMENT RECORD REJECTED: {label}: {detail}')
stats.add_fail(label, 'segments', detail)
if args.static:
if args.jobs > 1 and len(captures) > 1:
# Each capture is an independent recompiler subprocess plus a
# Python-side audit/namespace pass; the latter is CPU-bound, so
# processes, not threads. pool.map yields in submission order, and
# the merge runs in that order, so the result is byte-identical to
# the sequential loop.
from concurrent.futures import ProcessPoolExecutor
print(f'Static mode: {args.jobs} worker processes, '
f'results merged in capture order\n')
with ProcessPoolExecutor(max_workers=args.jobs) as pool:
for res in pool.map(static_capture_job, captures,
itertools.repeat(args),
itertools.repeat(toml),
itertools.repeat(forced_interiors),
itertools.repeat(static_out),
chunksize=1):
_merge_static_result(res)
else:
for cap in captures:
_do_capture(cap, region_coverage_cache, interior_frag_jobs, stats)
else:
# Publication order is part of the reproducible cache result whenever
# the process-lifetime Candidate cap can bind. Sort complete capture
# recipes and serialize whole compiler invocations in this namespace;
# a mutex around only the final rename would still admit whichever
# worker happened to arrive first at cap-minus-N.
capacity_lock, _tier_dirs = _candidate_capacity_namespace(
os.path.join(cache_dir, f'00000000_00000000{overlay_ext()}'))
invocation_lock = capacity_lock + '.invocation'
with _exclusive_file_lock(invocation_lock, timeout=None):
expected_abi = overlay_abi_tag(
args.runtime_include, args.flavor)
runtime_candidate_cap = overlay_candidate_cap(
args.runtime_include)
capacity_probe = os.path.join(
cache_dir, f'00000000_00000000{overlay_ext()}')
existing_candidates, candidate_counts = \
cache_candidate_capacity_snapshot(
capacity_probe, expected_abi)
saturated_single_tier = candidate_capacity_saturated_in_tier(
existing_candidates, candidate_counts,
runtime_candidate_cap, cache_dir)
capacity_fixedpoint = (
saturated_single_tier and not args.force and
not forced_interiors)
captures_to_process = captures
if capacity_fixedpoint:
# At a single-tier full namespace, ordinary compilation cannot
# publish growth and there is no lower compiler tier to promote
# with a capacity-neutral basename replacement. This is an
# intentionally coarse fixed-point result: coverage_report is
# the exact gap authority, while this invocation avoids decoding
# and sorting a potentially gigabyte-scale capture collection.
# BIOS resident bundles are the exception: normal policy lets a
# richer capture replace its canonical bundle without --force,
# and that replacement can be capacity-neutral. Process only
# these tiny captures, then re-probe before suppressing the
# gigabyte-scale ordinary collection.
resident_captures = [
cap for cap in captures
if cap.get('producer') == BIOS_RESIDENT_PRODUCER
]
resident_captures.sort(key=lambda cap: (
int(cap['load_addr'], 16) & 0x1FFFFFFF,
binascii.crc32(base64.b64decode(
cap['bytes_b64'])) & 0xFFFFFFFF,
hashlib.sha256(json.dumps(
cap, sort_keys=True, separators=(',', ':'),
default=str).encode('utf-8')).digest(),
))
for cap in resident_captures:
_do_capture(
cap, region_coverage_cache, interior_frag_jobs, stats)
resident_ids = {id(cap) for cap in resident_captures}
captures_to_process = [
cap for cap in captures if id(cap) not in resident_ids
]
existing_candidates, candidate_counts = \
cache_candidate_capacity_snapshot(
capacity_probe, expected_abi)
capacity_fixedpoint = candidate_capacity_saturated_in_tier(
existing_candidates, candidate_counts,
runtime_candidate_cap, cache_dir)
if capacity_fixedpoint:
stats.add_capacity_fastpath(len(captures_to_process))
print(f'Overlay candidate fixed point: '
f'{existing_candidates}/{runtime_candidate_cap} '
f'candidates in the active compiler tier; processed '
f'{len(resident_captures)} BIOS resident recipe(s) '
f'and skipped {len(captures_to_process)} ordinary '
'capture recipe(s) before byte decode. Use --force or '
'--force-interior for capacity-neutral repair '
'attempts; coverage_report remains the exact gap '
'authority.\n')
if not capacity_fixedpoint:
captures_to_process.sort(key=lambda cap: (
int(cap['load_addr'], 16) & 0x1FFFFFFF,
binascii.crc32(base64.b64decode(
cap['bytes_b64'])) & 0xFFFFFFFF,
hashlib.sha256(json.dumps(
cap, sort_keys=True, separators=(',', ':'),
default=str).encode('utf-8')).digest(),
))
if args.jobs > 1:
print('Deterministic candidate-cap admission: region '
'recipes run in canonical order '
'(parallel publication disabled)\n')
for cap in captures_to_process:
_do_capture(
cap, region_coverage_cache, interior_frag_jobs, stats)
_do_frags(interior_frag_jobs, stats)
# B-2: write combined static C file
if args.static and static_parts:
# CPS can yield at every compiled block leader, not just leaders that a
# particular capture happened to observe. Give every block in every
# variant a content-gated resume wrapper. This makes static coverage
# independent of host timing / slice boundaries.
synthesized = 0
def synthesize_all_resume_wrappers(parts):
nonlocal synthesized
for part in parts:
done = part.setdefault('resume_entries', set())
# Same per-entry rules as before, but the resume arms are
# inserted per HOST in one pass (add_cps_resume_cases) and the
# wrappers appended in one concatenation. Entry order is the
# sorted order the per-entry loop used, so the emitted C is
# byte-identical; the cost drops from quadratic in the part
# size to linear.
candidates = []
by_host = {}
for entry, host in sorted(part['continuation_owners'].items()):
if entry in part['func_addrs'] or entry in done:
continue
if host not in part['ids_by_addr']:
continue
candidates.append((entry, host))
by_host.setdefault(host, []).append(entry)
requests = [(part['symbols'][host], host, entries)
for host, entries in by_host.items()]
part['src'], ok_by_symbol = add_cps_resume_cases(
part['src'], requests)
wrappers = []
for entry, host in candidates:
host_symbol = part['symbols'][host]
if entry not in ok_by_symbol.get(host_symbol, ()):
continue
symbol = f'{part["namespace"]}_func_{entry:08X}'
wrappers.append(
f'\n/* CPS block resume entry owned by 0x{host:08X}. */\n'
f'void {symbol}(CPUState* cpu)\n{{\n'
f' cpu->pc = 0x{entry:08X}u;\n'
f' {host_symbol}(cpu);\n'
f'}}\n')
for code_crc, ranges in part['ids_by_addr'][host]:
part['variants'].append({
'addr': entry,
'symbol': symbol,
'crc': code_crc,
'ranges': ranges,
})
done.add(entry)
synthesized += 1
part['src'] += ''.join(wrappers)
synthesize_all_resume_wrappers(static_parts)
# Captured entries not owned by any compiled host are genuine orphan
# interiors. Compile each as an isolated dispatch-root shard, then give
# every block in those fragments the same universal resume treatment.
#
# Demands are (entry, image crc), one per capture that observed the
# address, and a demand is served only by a part built from that
# capture's bytes (select_static_fragment_demands). A band with many
# occupants therefore gets one fragment per occupant that observed the
# address -- the dispatcher's CRC gate picks the resident one.
unresolved = select_static_fragment_demands(
static_requested_entries, static_parts)
# PSX_STATIC_NO_ISOLATED=1: A/B guard -- skip the isolated-fragment pass
# entirely while testing the universal-resume machinery.
if os.environ.get('PSX_STATIC_NO_ISOLATED'):
unresolved = []
# Persistent doomed-fragment memo, as in the DLL path: an interior
# that is data in THIS image fails the generated-C audit every time,
# after a full recompiler walk. Band-attributed captures (an observed
# PC attached to every occupant of its band) make that the common
# case, so remember the verdict per (image bytes, interior, toml,
# cps, guard) next to the static output. --force cannot be the retry
# switch here (static mode needs it just to overwrite the output);
# --force-interior PC retries that entry, deleting the memo file
# retries everything.
fail_memo = load_interior_fail_memo(args.out_dir)
expected_abi = overlay_abi_tag(args.runtime_include, args.flavor)
forced_phys = {entry & 0x1FFFFFFF for entry in forced_interiors}
fragment_jobs = [] # (demand key, memo key, image key)
image_sources = {} # image key -> (data, load, size, phys, guard)
fragment_memo_skipped = 0
for key in unresolved:
source = static_entry_sources.get(key)
if source is None:
continue
entry, image_crc = key
data, load_addr, size, phys_addr, guard_bytes = source
memo_key = _interior_fail_key(
phys_addr, entry, data, load_addr, size, [], [],
expected_abi, bool(args.cps), toml, guard_bytes=guard_bytes)
if (memo_key in fail_memo and
(entry & 0x1FFFFFFF) not in forced_phys):
fragment_memo_skipped += 1
continue
# One image per segment view (§5.7): every view of a capture has
# the same bytes, phys and size, but a demand compiles against its
# own view's load address, which fixes the segment it is spelled,
# named and baked in.
image_key = (image_crc, phys_addr, size, segment_of(load_addr))
image_sources.setdefault(image_key, source)
fragment_jobs.append((key, memo_key, image_key))
# Fragments are independent recompiler subprocesses; run them on the
# same pool as the regions. Results are merged in demand order and a
# result whose entry an earlier-merged part already serves for the
# same image is discarded, so the output is byte-identical to the
# sequential loop (which skipped such entries before compiling).
if fragment_jobs:
print(f'Static isolated interior fragments: {len(fragment_jobs)} '
f'(entry, image) demand(s) over {len(image_sources)} '
f'image(s) to compile'
+ (f', {fragment_memo_skipped} known-doomed skipped'
if fragment_memo_skipped else ''))
if args.jobs > 1 and len(fragment_jobs) > 1:
from concurrent.futures import ProcessPoolExecutor
with ProcessPoolExecutor(
max_workers=args.jobs,
initializer=_init_static_fragment_worker,
initargs=(image_sources,)) as pool:
fragment_results = list(pool.map(
static_fragment_job,
[job[0][0] for job in fragment_jobs],
[job[2] for job in fragment_jobs],
itertools.repeat(args),
chunksize=8))
else:
_init_static_fragment_worker(image_sources)
fragment_results = [static_fragment_job(key[0], image_key, args)
for key, _memo_key, image_key in fragment_jobs]
fragment_built = 0
fragment_rejected = 0
fragment_failed = []
new_fragment_parts = []
existing_keys = served_static_variant_keys(static_parts)
for (key, memo_key, _image_key), (part, reason) in zip(
fragment_jobs, fragment_results):
if key in existing_keys:
continue
if part is None:
if interior_failure_is_deterministic(reason):
fragment_rejected += 1
append_interior_fail_memo(args.out_dir, memo_key, reason)
else:
fragment_failed.append((key, reason))
continue
static_parts.append(part)
new_fragment_parts.append(part)
existing_keys.update(
(variant['addr'], part['image_crc'])
for variant in part['variants'])
fragment_built += 1
synthesize_all_resume_wrappers(new_fragment_parts)
unresolved = select_static_fragment_demands(
static_requested_entries, static_parts)
print(f'Static universal CPS resume wrappers: {synthesized}')
print(f'Static isolated interior shards: {fragment_built}')
if fragment_rejected or fragment_memo_skipped:
# Data-as-code in that capture's bytes. Expected whenever a
# dispatch entry is attributed to more occupants of a band than
# actually ran it: safe coverage loss, never a shard failure --
# the resident occupant's own fragment is what serves the entry.
print(f'Static fragments rejected as data in their image: '
f'{fragment_rejected} this run (memoized), '
f'{fragment_memo_skipped} skipped from the memo')
stats.add_skip(fragment_rejected + fragment_memo_skipped)
hard = [(key, reason) for key, reason in fragment_failed]
if hard:
sample = ', '.join(f'0x{key[0]:08X}/{key[1]:08X}'
for key, _reason in hard[:12])
print(f'STATIC COVERAGE WARNING: {len(hard)} captured dispatch '
f'entry(s) have no compiled body/owner (entry/image): '
f'{sample}')
for key, reason in hard:
stats.add_fail(f'static entry 0x{key[0]:08X} image {key[1]:08X}',
'static_unresolved',
f'captured dispatch entry with no compiled '
f'body/owner: {reason}')
all_variants = []
for part in static_parts:
all_variants.extend(part['variants'])
dispatch_src = generate_overlay_dispatch(all_variants)
written = write_static_outputs(static_out, static_parts, all_variants,
dispatch_src,
single_file=args.static_single_file)
print(f'Static output: {static_out} '
f'({len(all_variants)} exact function identities total, '
f'{len(written)} translation unit(s))')
# A conservative primary scan can emit no functions while exact-entry
# supplementation still serves every requested root. Decide only after
# that pass, using ABI-valid, current-byte guarded native coverage. Missing
# even one primary root remains a failure; toolchain/audit failures above
# are never cleared by this reconciliation.
#
# DLL mode only. The reconciliation reads the per-game DLL cache, and
# cache_dir is bound only on the non-static path; --static emits one
# self-contained overlays_static.c and _do_capture returns into
# static_capture_job before anything is queued. Calling it unconditionally
# raised UnboundLocalError at the very end of every --static run, after
# the output was already written, and turned a clean compile into exit 1.
if args.static:
if pending_empty_primary:
raise SystemExit(
'internal error: --static queued %d empty-primary '
'reconciliation(s), but static mode has no DLL cache to '
'reconcile against' % len(pending_empty_primary))
else:
reconcile_empty_primary_scans(
pending_empty_primary, cache_dir,
overlay_abi_tag(args.runtime_include, args.flavor), stats)
# LOUD summary + machine-readable result line, then a non-zero exit when any
# shard that should have built failed. The runtime's autocompile watcher and
# any CI/dev invocation now SEE shard failures instead of a green exit 0 that
# masked a header-drift breakage. Skips (data-only / already-covered) and a
# zero-capture run are NOT failures.
n_fail = stats.print_summary()
if _check_tmp:
import shutil
shutil.rmtree(_check_tmp, ignore_errors=True)
print(f'[check] removed throwaway dir {_check_tmp}')
print('Done.')
if n_fail:
raise SystemExit(2)
if __name__ == '__main__':
main()
Reference in New Issue View Git Blame Copy Permalink