diff --git a/docs/AOT_SHARDING.md b/docs/AOT_SHARDING.md index b5b3d00f..ead49592 100644 --- a/docs/AOT_SHARDING.md +++ b/docs/AOT_SHARDING.md @@ -58,11 +58,13 @@ Profiles declare these reusable methods: | --- | --- | | `disc_hashes` | Hash the cue's original data track before extracting. | | `images[].method = fixed_address_files` | Whole original files placed at a verified `load_addr`. Optional duplicate-leaf checks require identical bytes. | +| `images[].method = tagged_relocated_files` | Apply a trailing two-bit ABS32/HI16/LO16/J26 relocation stream to a verified `load_addr`, retaining only the original loader's image interval. Destinations are explicit evidence, never inferred from a historical RAM capture. | | `images[].method = psx_exe` | Read the load address and image from a PS-X EXE header; the generic extractor may split resident and overlay floors. | | `images[].method = packed_sector_members` | Decode u32 count/offset sector descriptors across ordered payload files. Verify every member classification and requested full-payload coverage; use loader-established addresses for executable members. | | `images[].method = sector_extent_members` | Decode `{sector offset, byte size}` pairs in an archive. Use archive-relative offsets, verify the exact count and terminator, reject gaps/overlaps, and account for every member after sector rounding. Each member needs a verified load address or an explicit exclusion. | | `images[].method = aligned_lzss_banks` | Read aligned stored-size pairs, decode the metadata and its tagged bank members using parameterized LZSS, verify every container and bank classification, and deduplicate exact decoded images at verified destinations. External-RAM references fail extraction. | | `checks[].method = words` | Verify loader instructions or descriptors at explicit file offsets / virtual addresses. | +| `checks[].method = tagged_relocations` | Validate a complete relocation stream, retained `image_size` and `relocation_count`, including modules whose heap placement remains unresolved and which are excluded from native production. | | `checks[].method = pointer_strings` | Verify a pointer-indexed filename table against exact expected strings. | | `checks[].method = bcd_extent_table` | Verify an indexed BCD-MSF/size table against an ISO file's actual extent. | | `checks[].method = adjacent_files` | Require the named ISO extents to be sector aligned and physically adjacent, in order. | diff --git a/tools/aot_overlay_pipeline.py b/tools/aot_overlay_pipeline.py index 09ef4d8c..9182764b 100644 --- a/tools/aot_overlay_pipeline.py +++ b/tools/aot_overlay_pipeline.py @@ -22,6 +22,7 @@ from aot_overlay_spike import extract_generic as extractor from packed_sector_table import extract_members as extract_sector_members from sector_extent_archive import extract_members as extract_extent_members from aligned_lzss_banks import banks as extract_lzss_banks +from mips_tagged_relocations import parse as parse_tagged_relocations, relocate as relocate_tagged_image FRAMEWORK = Path(__file__).resolve().parents[1] @@ -74,7 +75,11 @@ def verify_evidence(disc, checks): origin = number(check.get('file_offset', 0)) base = number(check.get('base', 0)) method = check['method'] - if method == 'words': + if method == 'tagged_relocations': + image, relocations = parse_tagged_relocations(data) + require(len(image) == number(check['image_size']), 'Relocated image inventory changed') + require(len(relocations) == number(check['relocation_count']), 'Relocation inventory changed') + elif method == 'words': for address, expected in check['values'].items(): offset = origin + number(address) - base require(0 <= offset <= len(data) - 4, 'Evidence word outside source') @@ -223,6 +228,11 @@ def positioned_sources(disc, specifications): body = body[offset:] elif method == 'fixed_address_files': base = number(spec['load_addr']) + elif method == 'tagged_relocated_files': + base = number(spec['load_addr']) + require(not spec.get('verify_duplicate_names'), + 'Relocated sources need individual source identities') + body, _ = relocate_tagged_image(body, base) else: raise ValueError(f'Unknown image method: {method}') require(0x80000000 <= base < base + len(body) <= 0x80200000, f'{name}: image outside RAM') @@ -339,7 +349,7 @@ def prepare(profile, disc, records, output): if (lo, hi) == (source['base'], source['base'] + len(source['body'])): singles[source['name']] = record for source in sources: - if source['spec']['method'] in ('fixed_address_files', 'packed_sector_members', 'sector_extent_members', 'aligned_lzss_banks') and source['name'] not in singles: + if source['spec']['method'] in ('fixed_address_files', 'packed_sector_members', 'sector_extent_members', 'aligned_lzss_banks', 'tagged_relocated_files') and source['name'] not in singles: record = make_fixed_record(source, disc) singles[source['name']] = record records.append(record) diff --git a/tools/tests/test_aot_overlay_pipeline.py b/tools/tests/test_aot_overlay_pipeline.py index 3e58770d..a3be35e2 100644 --- a/tools/tests/test_aot_overlay_pipeline.py +++ b/tools/tests/test_aot_overlay_pipeline.py @@ -24,6 +24,26 @@ class FakeDisc: class AotMethodsTest(unittest.TestCase): + def test_tagged_relocated_original_file_and_inventory_check(self): + original = struct.pack('<8I', 16, 8, 0x03e00008, 0, 4, 0xFFFFFFFF, 0, 0)[:24] + disc = FakeDisc({'MODULE.DLL': original}) + spec = dict(method='tagged_relocated_files', files=['MODULE.DLL'], + load_addr='0x80100000', allow_missing=True, entries=['0x80100008']) + source, = pipeline.positioned_sources(disc, [spec]) + self.assertEqual(source['body'], struct.pack('<4I', 16, 0x80100008, 0x03e00008, 0)) + self.assertEqual(disc.read('MODULE.DLL'), original) + check = dict(method='tagged_relocations', file='MODULE.DLL', image_size=16, relocation_count=1) + pipeline.verify_evidence(disc, [check]) + with self.assertRaisesRegex(ValueError, 'inventory changed'): + pipeline.verify_evidence(disc, [{**check, 'relocation_count': 2}]) + with tempfile.TemporaryDirectory() as directory: + disc.binary = Path(directory) / 'source.bin' + disc.binary.write_bytes(original) + inventory = pipeline.prepare(dict(game_id='TEST', images=[spec], checks=[check], + expected_records=1, strict_bounds=True), disc, [], Path(directory)) + self.assertEqual(inventory['required_images'], ['MODULE.DLL']) + self.assertEqual(inventory['jobs'][0]['required_entries'], [0x80100008]) + def test_resident_preload_metadata_survives_build_audit_and_stage(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory)