diff --git a/docs/AOT_SHARDING.md b/docs/AOT_SHARDING.md index 1f898c5c..ead49592 100644 --- a/docs/AOT_SHARDING.md +++ b/docs/AOT_SHARDING.md @@ -58,11 +58,13 @@ Profiles declare these reusable methods: | --- | --- | | `disc_hashes` | Hash the cue's original data track before extracting. | | `images[].method = fixed_address_files` | Whole original files placed at a verified `load_addr`. Optional duplicate-leaf checks require identical bytes. | +| `images[].method = tagged_relocated_files` | Apply a trailing two-bit ABS32/HI16/LO16/J26 relocation stream to a verified `load_addr`, retaining only the original loader's image interval. Destinations are explicit evidence, never inferred from a historical RAM capture. | | `images[].method = psx_exe` | Read the load address and image from a PS-X EXE header; the generic extractor may split resident and overlay floors. | | `images[].method = packed_sector_members` | Decode u32 count/offset sector descriptors across ordered payload files. Verify every member classification and requested full-payload coverage; use loader-established addresses for executable members. | | `images[].method = sector_extent_members` | Decode `{sector offset, byte size}` pairs in an archive. Use archive-relative offsets, verify the exact count and terminator, reject gaps/overlaps, and account for every member after sector rounding. Each member needs a verified load address or an explicit exclusion. | | `images[].method = aligned_lzss_banks` | Read aligned stored-size pairs, decode the metadata and its tagged bank members using parameterized LZSS, verify every container and bank classification, and deduplicate exact decoded images at verified destinations. External-RAM references fail extraction. | | `checks[].method = words` | Verify loader instructions or descriptors at explicit file offsets / virtual addresses. | +| `checks[].method = tagged_relocations` | Validate a complete relocation stream, retained `image_size` and `relocation_count`, including modules whose heap placement remains unresolved and which are excluded from native production. | | `checks[].method = pointer_strings` | Verify a pointer-indexed filename table against exact expected strings. | | `checks[].method = bcd_extent_table` | Verify an indexed BCD-MSF/size table against an ISO file's actual extent. | | `checks[].method = adjacent_files` | Require the named ISO extents to be sector aligned and physically adjacent, in order. | @@ -90,6 +92,11 @@ coverage or native semantics; keep fallback and perform gameplay spot checks. Jersey Devil uses `aligned_lzss_banks` for five renderer variants across 70 bank occurrences and `psx_exe` for five secondary executables. Its profile accounts for all 93 BZZ containers, including 23 without code banks. +Exact-BIOS resident helper recipes also publish a `.resident` preload marker. +The pipeline preserves it through consolidation and staging, verifies its BIOS +hash against the recipe, and includes its hash in the audit receipt. Omitting +that marker can leave an otherwise valid helper unused at runtime. + Current consumers: - Tomba! USA: `TombaRecomp/aot/overlays.json`, 25 fixed-address images, including diff --git a/runtime/include/mod_memory.h b/runtime/include/mod_memory.h index 7406c39a..5f3e0fd0 100644 --- a/runtime/include/mod_memory.h +++ b/runtime/include/mod_memory.h @@ -9,19 +9,24 @@ extern "C" { /* * GPU linked-list tags carry only 24 address bits. Trusted enhancements that - * need a larger primitive arena use this otherwise-unmapped 1 MiB aperture so + * need a larger primitive arena use this otherwise-unmapped 4 MiB aperture so * their CPU pointers survive the tag truncation. Nothing is mapped until an * enhancement explicitly allocates from it. */ -#define PSX_MOD_GPU_DMA_APERTURE_BASE 0x00F00000u -#define PSX_MOD_GPU_DMA_APERTURE_SIZE (1u * 1024u * 1024u) -#define PSX_MOD_GPU_DMA_GUEST_BASE 0x80F00000u +#define PSX_MOD_GPU_DMA_APERTURE_BASE 0x00C00000u +#define PSX_MOD_GPU_DMA_APERTURE_SIZE (4u * 1024u * 1024u) +#define PSX_MOD_GPU_DMA_GUEST_BASE 0x80C00000u static inline int psx_mod_gpu_dma_aperture_offset_for( uint32_t address, uint32_t width, uint32_t used, uint32_t *offset) { - uint32_t canonical = address & 0x00FFFFFFu; + /* CPU addresses retain all physical bits after KSEG normalization. The + * BIOS at 0x1FC00000 must never alias the aperture at 0x00C00000. + * DMA callers pass an already-truncated 24-bit tag separately below. */ + uint32_t canonical = address & 0x1FFFFFFFu; uint32_t off; - if (canonical < PSX_MOD_GPU_DMA_APERTURE_BASE) return 0; + if (canonical < PSX_MOD_GPU_DMA_APERTURE_BASE || + canonical >= 0x01000000u || used > PSX_MOD_GPU_DMA_APERTURE_SIZE) + return 0; off = canonical - PSX_MOD_GPU_DMA_APERTURE_BASE; if (off > used || width > used - off) return 0; if (offset) *offset = off; diff --git a/runtime/tests/test_mod_gpu_dma_aperture.c b/runtime/tests/test_mod_gpu_dma_aperture.c index 0daa0613..2033c64e 100644 --- a/runtime/tests/test_mod_gpu_dma_aperture.c +++ b/runtime/tests/test_mod_gpu_dma_aperture.c @@ -15,29 +15,50 @@ static void check(int condition, const char *name) { int main(void) { uint32_t off = 0; + uint32_t base = PSX_MOD_GPU_DMA_APERTURE_BASE; + uint32_t size = PSX_MOD_GPU_DMA_APERTURE_SIZE; + uint32_t aliases[] = {0x1FC00000u, 0x9FC00000u, 0xBFC00000u, + 0x1F000000u, 0x9F000000u, 0x1F801810u, + 0xBF801810u, 0x1EFFFFFFu}; + unsigned i; - check(psx_mod_gpu_dma_resolve_address_for(0x00F01234u, 0u) == - (0x00F01234u & 0x001FFFFCu), + check(psx_mod_gpu_dma_resolve_address_for(base + 0x1234u, 0u) == + ((base + 0x1234u) & 0x001FFFFCu), "unallocated aperture preserves retail 2 MiB folding"); - check(psx_mod_gpu_dma_resolve_address_for(0x80F01234u, 0x20000u) == - 0x00F01234u, + check(psx_mod_gpu_dma_resolve_address_for(0x80000000u + base + 0x1234u, 0x20000u) == + base + 0x1234u, "allocated KSEG0 aperture pointer survives 24-bit DMA"); - check(psx_mod_gpu_dma_resolve_address_for(0x00F30000u, 0x20000u) == - (0x00F30000u & 0x001FFFFCu), + check(psx_mod_gpu_dma_resolve_address_for(base + 0x30000u, 0x20000u) == + ((base + 0x30000u) & 0x001FFFFCu), "unallocated aperture tail remains folded"); check(psx_mod_gpu_dma_resolve_address_for(0x001ABCDEu, 0x20000u) == 0x001ABCDCu, "ordinary main RAM remains word-aligned and unchanged"); check(psx_mod_gpu_dma_aperture_offset_for( - 0x00F1FFFCu, 4u, 0x20000u, &off) && + base + 0x1FFFCu, 4u, 0x20000u, &off) && off == 0x1FFFCu, "last allocated aperture word is accessible"); check(!psx_mod_gpu_dma_aperture_offset_for( - 0x00F20000u, 4u, 0x20000u, &off), + base + 0x20000u, 4u, 0x20000u, &off), "first unallocated aperture word is inaccessible"); check(!psx_mod_gpu_dma_aperture_offset_for( 0x00FFFFFCu, 8u, PSX_MOD_GPU_DMA_APERTURE_SIZE, &off), "cross-boundary access is rejected"); + for (i = 0; i < sizeof(aliases) / sizeof(aliases[0]); ++i) + check(!psx_mod_gpu_dma_aperture_offset_for(aliases[i], 4u, size, &off), + "CPU BIOS/expansion/MMIO addresses cannot alias DMA storage"); + check(psx_mod_gpu_dma_aperture_offset_for(0xA0000000u + base, 4u, size, &off) + && off == 0u, "KSEG1 CPU aperture alias resolves correctly"); + check(psx_mod_gpu_dma_aperture_offset_for(base + size - 4u, 4u, size, &off) + && off == size - 4u, "last word of full 4 MiB aperture is accessible"); + check(!psx_mod_gpu_dma_aperture_offset_for(base - 4u, 4u, size, &off), + "word before aperture is rejected"); + check(!psx_mod_gpu_dma_aperture_offset_for(base + size, 4u, size, &off), + "word after aperture is rejected"); + check(!psx_mod_gpu_dma_aperture_offset_for(base, 4u, size + 4u, &off), + "invalid allocated size is rejected"); + check(psx_mod_gpu_dma_resolve_address_for(base + size - 4u, size) == + base + size - 4u, "full aperture DMA tag keeps its high address bits"); return failures ? 1 : 0; } diff --git a/tools/aot_overlay_pipeline.py b/tools/aot_overlay_pipeline.py index 86a8426e..9182764b 100644 --- a/tools/aot_overlay_pipeline.py +++ b/tools/aot_overlay_pipeline.py @@ -22,6 +22,7 @@ from aot_overlay_spike import extract_generic as extractor from packed_sector_table import extract_members as extract_sector_members from sector_extent_archive import extract_members as extract_extent_members from aligned_lzss_banks import banks as extract_lzss_banks +from mips_tagged_relocations import parse as parse_tagged_relocations, relocate as relocate_tagged_image FRAMEWORK = Path(__file__).resolve().parents[1] @@ -74,7 +75,11 @@ def verify_evidence(disc, checks): origin = number(check.get('file_offset', 0)) base = number(check.get('base', 0)) method = check['method'] - if method == 'words': + if method == 'tagged_relocations': + image, relocations = parse_tagged_relocations(data) + require(len(image) == number(check['image_size']), 'Relocated image inventory changed') + require(len(relocations) == number(check['relocation_count']), 'Relocation inventory changed') + elif method == 'words': for address, expected in check['values'].items(): offset = origin + number(address) - base require(0 <= offset <= len(data) - 4, 'Evidence word outside source') @@ -223,6 +228,11 @@ def positioned_sources(disc, specifications): body = body[offset:] elif method == 'fixed_address_files': base = number(spec['load_addr']) + elif method == 'tagged_relocated_files': + base = number(spec['load_addr']) + require(not spec.get('verify_duplicate_names'), + 'Relocated sources need individual source identities') + body, _ = relocate_tagged_image(body, base) else: raise ValueError(f'Unknown image method: {method}') require(0x80000000 <= base < base + len(body) <= 0x80200000, f'{name}: image outside RAM') @@ -339,7 +349,7 @@ def prepare(profile, disc, records, output): if (lo, hi) == (source['base'], source['base'] + len(source['body'])): singles[source['name']] = record for source in sources: - if source['spec']['method'] in ('fixed_address_files', 'packed_sector_members', 'sector_extent_members', 'aligned_lzss_banks') and source['name'] not in singles: + if source['spec']['method'] in ('fixed_address_files', 'packed_sector_members', 'sector_extent_members', 'aligned_lzss_banks', 'tagged_relocated_files') and source['name'] not in singles: record = make_fixed_record(source, disc) singles[source['name']] = record records.append(record) @@ -487,9 +497,15 @@ def build(inventory, game_toml, recompiler, work, gcc, workers, project_root=Non if destination.exists(): require(library.with_suffix('.ranges').read_bytes() == destination.with_suffix('.ranges').read_bytes(), f'Conflicting native pair identity: {library.name}') - continue - shutil.copy2(library, destination) - shutil.copy2(library.with_suffix('.ranges'), destination.with_suffix('.ranges')) + else: + shutil.copy2(library, destination) + shutil.copy2(library.with_suffix('.ranges'), destination.with_suffix('.ranges')) + marker = library.with_suffix('.resident') + if marker.exists(): + target_marker = destination.with_suffix('.resident') + require(not target_marker.exists() or marker.read_bytes() == target_marker.read_bytes(), + f'Conflicting resident metadata: {library.name}') + shutil.copy2(marker, target_marker) return cache @@ -500,8 +516,11 @@ def stage(cache, destination, receipt): target.mkdir(parents=True, exist_ok=True) expected = set() for pair in receipt['pairs']: - for name, key in [(pair['dll'], 'dll_sha256'), - (Path(pair['dll']).with_suffix('.ranges').name, 'manifest_sha256')]: + artifacts = [(pair['dll'], 'dll_sha256'), + (Path(pair['dll']).with_suffix('.ranges').name, 'manifest_sha256')] + if 'resident_sha256' in pair: + artifacts.append((Path(pair['dll']).with_suffix('.resident').name, 'resident_sha256')) + for name, key in artifacts: expected.add(name) require(digest(source / name) == pair[key], f'Audited artifact changed: {name}') shutil.copy2(source / name, target / name) diff --git a/tools/audit_aot_cache.py b/tools/audit_aot_cache.py index a0b516a0..c3634b8b 100644 --- a/tools/audit_aot_cache.py +++ b/tools/audit_aot_cache.py @@ -11,6 +11,24 @@ from pathlib import Path import sys +def resident_metadata(dll, record): + """The compiler's preload opt-in must agree with the verified recipe.""" + marker = dll.with_suffix('.resident') + resident = record.get('producer') == 'bios_resident_manifest' + assert marker.exists() == resident, f'Resident preload marker/recipe mismatch: {dll.name}' + if not resident: + return {} + body = marker.read_bytes() + document = json.loads(body) + assert document.get('schema') == 'psxrecomp bios resident shard v1', f'Invalid resident marker: {dll.name}' + bios_hash = record.get('bios_sha256', '') + assert len(bios_hash) == 64 and all(c in '0123456789abcdef' for c in bios_hash), 'Invalid resident BIOS hash' + assert document.get('bios_sha256') == bios_hash, f'Resident BIOS provenance mismatch: {dll.name}' + assert document.get('producer_name') == str(record.get('producer_name', 'BIOS resident code')), \ + f'Resident producer mismatch: {dll.name}' + return dict(resident_sha256=hashlib.sha256(body).hexdigest()) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--framework-root', type=Path, @@ -46,6 +64,7 @@ def main(): dict(name=f'record-{index:03d}', record=record) for index, record in enumerate(document)] recipes = [] + records_by_name = {} for job in jobs: if args.inventory: record = json.loads(Path(job['input']).read_text(encoding='utf-8-sig'))[0] @@ -65,6 +84,7 @@ def main(): bounds = [(lo & 0x1fffffff, hi & 0x1fffffff) for lo, hi in known] assert all(lo < hi for lo, hi in bounds) recipes.append((job['name'], load, data, bounds)) + records_by_name[job['name']] = record pairs = [] native_ids = [] @@ -89,7 +109,8 @@ def main(): assert matched, f'No original-input guard proof for {dll.name}' pairs.append(dict(dll=dll.name, functions=len(ids), matched_recipe=matched, dll_sha256=hashlib.sha256(dll.read_bytes()).hexdigest(), - manifest_sha256=hashlib.sha256(dll.with_suffix('.ranges').read_bytes()).hexdigest())) + manifest_sha256=hashlib.sha256(dll.with_suffix('.ranges').read_bytes()).hexdigest(), + **resident_metadata(dll, records_by_name[matched]))) if args.expected_pairs is not None: assert len(pairs) == args.expected_pairs, (len(pairs), args.expected_pairs) image_coverage = [] diff --git a/tools/mips_tagged_relocations.py b/tools/mips_tagged_relocations.py new file mode 100644 index 00000000..cb90919f --- /dev/null +++ b/tools/mips_tagged_relocations.py @@ -0,0 +1,75 @@ +"""Reconstruct MIPS images with a trailing two-bit tagged relocation stream. + +The first little-endian word is the byte offset of the stream and therefore +the retained image size. Stream words encode an aligned image offset plus: +0 = ABS32, 1 = HI16 with a following full addend, 2 = LO16, 3 = J26. +0xffffffff ends the stream. This format carries no load address: callers must +establish each destination independently from the original loader. +""" +from dataclasses import dataclass +import struct + + +@dataclass(frozen=True) +class Relocation: + offset: int + kind: int + addend: int | None = None + + +def parse(data: bytes) -> tuple[bytes, tuple[Relocation, ...]]: + """Validate the entire file and return its retained image and relocations.""" + if len(data) < 8 or len(data) % 4: + raise ValueError("Relocation file must contain aligned header and terminator") + end = struct.unpack_from(" len(data) - 4: + raise ValueError("Invalid relocation stream offset") + cursor, relocations, written = end, [], set() + while cursor <= len(data) - 4: + tag = struct.unpack_from(" end: + raise ValueError("Relocation target outside retained image") + if offset in written: + raise ValueError("Repeated relocation target") + written.add(offset) + if kind == 1: + if cursor > len(data) - 4: + raise ValueError("Truncated HI16 addend") + addend = struct.unpack_from(" tuple[bytes, tuple[Relocation, ...]]: + """Apply the loader's 32-bit and halfword writes at a verified RAM base.""" + source, relocations = parse(data) + if load_addr % 4 or not 0x80000000 <= load_addr < load_addr + len(source) <= 0x80200000: + raise ValueError("Relocated image must fit aligned PlayStation main RAM") + image = bytearray(source) + for item in relocations: + offset = item.offset + if item.kind == 0: + word = struct.unpack_from("> 16 + struct.pack_into("> 6) + struct.pack_into("