diff --git a/docs/AOT_SHARDING.md b/docs/AOT_SHARDING.md index 5b374368..bbd4eed9 100644 --- a/docs/AOT_SHARDING.md +++ b/docs/AOT_SHARDING.md @@ -62,6 +62,8 @@ Profiles declare these reusable methods: | `mod_packages[]` / `mod_package` | A named, verified selection of a trusted mod package. Images and checks carrying `mod_package` read the original disc with that selection applied; see below. | | `images[].transfer_entries` | `{from: "mod_package_writes", count}`: J/JAL targets inside the image, taken only from words the selected package wrote. The count pins the inventory. Optional `exclude_writes: [{start, end, reason}]` names intervals of package-written DATA (for example a pack linked into the EXE that the package replaced) whose words are not transfers. | | `images[].method = tagged_relocated_files` | Apply a trailing two-bit ABS32/HI16/LO16/J26 relocation stream to a verified `load_addr`, retaining only the original loader's image interval. Destinations are explicit evidence, never inferred from a historical RAM capture. | +| `images[].method = counted_relocated_members` | Verify each original container's SHA-256, classify every counted member, and relocate its image at each separately established heap placement. Pin `image_size` and `relocation_count`; each member needs nonempty unique `placements` or an explained exclusion. New placements remain interpreted. See [counted containers](COUNTED_MIPS_RELOCATIONS.md). | +| `images[].entry_pointer_offsets` | Aligned image-relative offsets of loader-established callable exports. Read the pointer after relocation, ignore null callbacks, and require every nonnull callback to target aligned code inside the image. Pointer-shaped script/data fields are not callable exports. | | `images[].method = psx_exe` | Read the load address and image from a PS-X EXE header; the generic extractor may split resident and overlay floors. | | `images[].method = packed_sector_members` | Decode u32 count/offset sector descriptors across ordered payload files. Verify every member classification and requested full-payload coverage; use loader-established addresses for executable members. | | `images[].method = sector_extent_members` | Decode `{sector offset, byte size}` pairs in an archive. Use archive-relative offsets, verify the exact count and terminator, reject gaps/overlaps, and account for every member after sector rounding. Each member needs a verified load address or an explicit exclusion. | diff --git a/docs/COUNTED_MIPS_RELOCATIONS.md b/docs/COUNTED_MIPS_RELOCATIONS.md index f0728634..d0d4a85e 100644 --- a/docs/COUNTED_MIPS_RELOCATIONS.md +++ b/docs/COUNTED_MIPS_RELOCATIONS.md @@ -39,7 +39,28 @@ The container has **no destination addresses**. `relocate` takes an aligned KSEG0 main-RAM image base; this is not the allocation base of a container whose headers remain in memory. Callers must establish placement and executable/data boundaries from the original loader before constructing native AOT producers. -The decoder alone does not provide dynamically placed native module dispatch. +The original-disc AOT pipeline accepts `counted_relocated_members` with +`containers: [{file, sha256, members: [...]}]`. Each member pins its `index`, +`image_size`, `relocation_count` and nonempty `placements: [{load_addr, ...}]`. +Every member must be classified, including an explicit reason for exclusions. +The placement is the image address, which can differ from the container's +allocation address by the header size. Native emission uses relocated original +bytes and the normal guarded dispatcher. A newly encountered heap placement +continues through the interpreter until its own native variant is established. + +`entry_pointer_offsets` declares image-relative callback fields proven by the +loader's consumers. Reading those fields after relocation lets the same export +declaration apply at multiple bases. Null callbacks are ignored; unaligned, +out-of-image and excluded required entries fail extraction. Script pointers and +other data fields must not be nominated as MIPS entry points merely because they +look like addresses. + +The generic disc discoverer now recognizes complete counted containers and +writes `.relocatable.json` alongside its positioned recipes. The sidecar +records original hashes, member extents, sizes and relocation counts with +`placement_required: true`. It does not infer a heap address or claim native +coverage. This exposes unresolved loader work during initial discovery instead +of waiting for gameplay to reveal interpreter callbacks. The implementation was compared byte for byte with the owned USA MediEvil II relocation routine (`0x80078328`) for 24 modules at two distinct RAM bases: @@ -49,4 +70,6 @@ all four relocation kinds, opcode preservation and malformed inputs: ```sh python -m unittest discover -s tools/tests -p test_counted_mips_relocations.py +python -m unittest discover -s tools/tests -p test_counted_relocation_discovery.py +python -m unittest discover -s tools/tests -p test_aot_overlay_pipeline.py ``` diff --git a/tools/aot_overlay_pipeline.py b/tools/aot_overlay_pipeline.py index 5f95622b..88abc4c1 100644 --- a/tools/aot_overlay_pipeline.py +++ b/tools/aot_overlay_pipeline.py @@ -25,6 +25,7 @@ from sector_extent_archive import extract_members as extract_extent_members from aligned_lzss_banks import banks as extract_lzss_banks from indexed_lzss_pack import members as extract_pack_members from mips_tagged_relocations import parse as parse_tagged_relocations, relocate as relocate_tagged_image +from counted_mips_relocations import parse as parse_counted_relocations, relocate as relocate_counted_image from mod_package_images import ModPackageView FRAMEWORK = Path(__file__).resolve().parents[1] @@ -369,6 +370,8 @@ def positioned_sources(disc, specifications, views=None, ram_bytes=RETAIL_RAM_BY def spec_sources(disc, spec): sources = [] method = spec['method'] + if method == 'counted_relocated_members': + return counted_relocated_sources(disc, spec) if method == 'fixed_address_extents': return extent_sources(disc, spec) if method == 'aligned_lzss_banks': @@ -469,6 +472,48 @@ def spec_sources(disc, spec): return sources +def counted_relocated_sources(disc, spec): + """Original members at declared heap placements, never captured RAM bytes. + + Counted containers describe image boundaries and relocation writes, but no + destination. Each placement must be established separately from the loader. + A new heap placement remains interpreted until a guarded variant is added. + """ + sources = [] + for container in spec['containers']: + file = container['file'].upper() + data = disc.read(file) + require(hashlib.sha256(data).hexdigest() == container['sha256'], + f'Counted relocation container changed: {file}') + members = parse_counted_relocations(data) + configured = {number(item['index']): item for item in container['members']} + excluded = {number(item['index']): item for item in container.get('excluded_members', [])} + require(len(configured) == len(container['members']) and + len(excluded) == len(container.get('excluded_members', [])), + 'Duplicate counted member inventory') + require(not configured.keys() & excluded.keys() and + configured.keys() | excluded.keys() == set(range(len(members))), + 'Counted member inventory has missing or conflicting classifications') + require(all(item.get('reason', '').strip() for item in excluded.values()), + 'Excluded counted member needs a reason') + for index, member in enumerate(members): + item = configured.get(index) + if item is None: + continue + require(len(member.image) == number(item['image_size']) and + len(member.relocations) == number(item['relocation_count']), + f'Counted member inventory changed: {file} {index}') + require(item.get('placements'), 'Counted member needs an established placement') + bases = [number(p['load_addr']) for p in item['placements']] + require(len(bases) == len(set(bases)), 'Duplicate counted member placement') + for placement, base in zip(item['placements'], bases): + body = relocate_counted_image(member, base) + sources.append(dict(name=f'{file}:IMAGE_{index:04X}@{base:08X}', + base=base, body=body, spec={**spec, **item, **placement, 'allow_missing': True}, + source_file=file, source_offset=member.image_offset, aliases=[])) + return sources + + def match_sources(record, sources): require(not record.get('executed_pcs'), 'Runtime observations cannot be AOT inputs') data = base64.b64decode(record['bytes_b64'], validate=True) @@ -495,6 +540,15 @@ def declared_entries(source, disc): offset = number(item.get('file_offset', 0)) + number(item['address']) - number(item.get('base', 0)) entries.add(struct.unpack_from(' one shard set, not N (same as hand tool). dcrc=binascii.crc32(data)&0xFFFFFFFF @@ -1271,6 +1298,9 @@ def main(): f"sha256={record['bios_sha256'][:12]}...") with open(a.out, 'w') as f: json.dump(records, f) + with open(a.out + '.relocatable.json', 'w') as f: + json.dump(dict(schema='psxrecomp unresolved original-disc relocations v1', + containers=relocatable), f, indent=2) print(f"producers: {np} PS-X EXE (full-discovery), {nh} header-table, " f"{nr} raw-code, {na} indexed-archive members, " f"{ne} HED-companion members, " diff --git a/tools/tests/test_aot_overlay_pipeline.py b/tools/tests/test_aot_overlay_pipeline.py index 91e8a6f4..fa634ae2 100644 --- a/tools/tests/test_aot_overlay_pipeline.py +++ b/tools/tests/test_aot_overlay_pipeline.py @@ -348,6 +348,57 @@ class ModPackageImageTest(unittest.TestCase): class AotMethodsTest(unittest.TestCase): + def test_counted_original_members_at_multiple_heap_placements(self): + # A relocated callback pointer enters a frameless return stub. + image = struct.pack('<4I', 8, 0x12345678, 0x03e00008, 0) + original = struct.pack('<4I', 1, 0, len(image), 0) + image + struct.pack('<2I', 0, 0xffffffff) + disc = FakeDisc({'RELOCS/MODULE.LVB': original}) + member = dict(index=0, image_size=16, relocation_count=1, + entry_pointer_offsets=[0], placements=[dict(load_addr='0x80101010'), + dict(load_addr='0x80102020')]) + container = dict(file='RELOCS/MODULE.LVB', sha256=sha(original), members=[member]) + spec = dict(method='counted_relocated_members', containers=[container]) + sources = pipeline.positioned_sources(disc, [spec]) + self.assertEqual([s['base'] for s in sources], [0x80101010, 0x80102020]) + self.assertEqual([pipeline.declared_entries(s, disc) for s in sources], + [{0x80101018}, {0x80102028}]) + self.assertEqual(disc.read('RELOCS/MODULE.LVB'), original) + with tempfile.TemporaryDirectory() as directory: + disc.binary = Path(directory) / 'source.bin'; disc.binary.write_bytes(original) + inventory = pipeline.prepare(dict(game_id='TEST', images=[spec], strict_bounds=True, + expected_records=2), disc, [], Path(directory)) + self.assertEqual(inventory['recipe_count'], 2) + self.assertEqual([j['required_entries'] for j in inventory['jobs']], + [[0x80101018], [0x80102028]]) + self.assertEqual(inventory['jobs'][0]['known_ranges'], [(0x80101010, 0x80101020)]) + + def test_counted_sources_reject_inventory_and_callback_drift(self): + image = struct.pack('<4I', 8, 0, 0x03e00008, 0) + original = struct.pack('<4I', 1, 0, 16, 0) + image + struct.pack('<2I', 0, 0xffffffff) + disc = FakeDisc({'MODULE.LVB': original}) + item = dict(index=0, image_size=16, relocation_count=1, + placements=[dict(load_addr='0x80100000')]) + container = dict(file='MODULE.LVB', sha256=sha(original), members=[item]) + def sources(c): + return pipeline.positioned_sources(disc, [dict(method='counted_relocated_members', + containers=[c])]) + for change, error in ((dict(sha256='0'*64), 'container changed'), + (dict(members=[]), 'missing'), + (dict(members=[{**item, 'relocation_count': 2}]), 'inventory changed'), + (dict(members=[{**item, 'placements': []}]), 'placement'), + (dict(members=[{**item, 'placements': item['placements']*2}]), 'Duplicate')): + with self.subTest(change=change), self.assertRaisesRegex(ValueError, error): + sources({**container, **change}) + source, = sources(container) + for offset in (-4, 1, 16): + with self.subTest(offset=offset), self.assertRaisesRegex(ValueError, 'outside image'): + pipeline.declared_entries({**source, 'spec': {**source['spec'], + 'entry_pointer_offsets': [offset]}}, disc) + source['body'] = struct.pack('<4I', 0x80020000, 0, 0x03e00008, 0) + with self.assertRaisesRegex(ValueError, 'Entry outside image'): + pipeline.declared_entries({**source, 'spec': {**source['spec'], + 'entry_pointer_offsets': [0]}}, disc) + def test_tagged_relocated_original_file_and_inventory_check(self): original = struct.pack('<8I', 16, 8, 0x03e00008, 0, 4, 0xFFFFFFFF, 0, 0)[:24] disc = FakeDisc({'MODULE.DLL': original}) diff --git a/tools/tests/test_counted_relocation_discovery.py b/tools/tests/test_counted_relocation_discovery.py new file mode 100644 index 00000000..1e9e5573 --- /dev/null +++ b/tools/tests/test_counted_relocation_discovery.py @@ -0,0 +1,31 @@ +"""Unpositioned module discovery must report facts without inventing a base.""" +from pathlib import Path +import struct +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from aot_overlay_spike.extract_generic import counted_relocation_inventory + + +class CountedDiscoveryTest(unittest.TestCase): + def test_complete_container_reports_members_and_requires_placement(self): + image = struct.pack('<4I', 8, 0x0c000002, 0x03e00008, 0) + original = struct.pack('<4I', 1, 0, 16, 0) + image + struct.pack('<3I', 0, 7, 0xffffffff) + inventory = counted_relocation_inventory(original) + self.assertTrue(inventory['placement_required']) + self.assertNotIn('load_addr', inventory) + self.assertEqual(inventory['members'][0]['image_offset'], 16) + self.assertEqual(inventory['members'][0]['relocation_count'], 2) + self.assertEqual(inventory['members'][0]['image_size'], 16) + + def test_partial_stream_and_pointer_header_are_not_relocation_containers(self): + for data in (struct.pack('<4I', 1, 0x80100000, 0x03e00008, 0), + struct.pack('<4I', 1, 0, 4, 0) + struct.pack('<2I', 0, 0), + b'PS-X EXE' + bytes(2040)): + with self.subTest(data=data[:16]): + self.assertIsNone(counted_relocation_inventory(data)) + + +if __name__ == '__main__': + unittest.main()