From b43a7c9ea8d30838ea4160ec9614101ee0bbbc9c Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 26 Sep 2026 06:34:57 -0700 Subject: [PATCH] aot: indexed_lzss_members verifies identical pack copies, e.g. one linked into the boot EXE Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/AOT_SHARDING.md | 11 +++++++++-- tools/aot_overlay_pipeline.py | 15 +++++++++++++-- tools/tests/test_aot_overlay_pipeline.py | 10 ++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/AOT_SHARDING.md b/docs/AOT_SHARDING.md index f7d9ba96..e687618a 100644 --- a/docs/AOT_SHARDING.md +++ b/docs/AOT_SHARDING.md @@ -213,9 +213,16 @@ flag precedes a position and a count. Each output byte is also stored at the current window slot, so a reference may read bytes it has just written. Executable members carry `index`, `load_addr`, `decoded_sha256` and optionally verified `entries`; `excluded_members` need a `reason`. Every index must be -classified exactly once. +classified exactly once. When the loader can also take the same pack from +elsewhere, such as a copy linked into the boot EXE, list each place in +`identical_copies` (`{file, file_offset, size, reason}`); extraction requires +those bytes to equal the pack exactly and records them as aliases. -WipEout 3's `MenuControl` (0x801179B8) opens `c:\wipeout3\nfe.pb`, requires its +WipEout 3's `MenuControl` (0x801179B8) opens `c:\wipeout3\nfe.pb`. The first +open after boot reads no file: a preload flag starts at 1 and the pack is the +copy linked at the start of `SCES_028.45` (0x80010000, the main heap base); +once that pack is consumed the flag clears and later opens read the disc file. +Both copies are identical, including under the framerate package. It requires its allocation of the top member to land exactly at 0x800BF800, has the pack reader run `lzss.c` `ExpandData` (13-bit position, 4-bit count, three-byte minimum, start position 1, uninitialized 8 KiB window) and copies the decoded 0x21424 diff --git a/tools/aot_overlay_pipeline.py b/tools/aot_overlay_pipeline.py index 9601942f..038eac2e 100644 --- a/tools/aot_overlay_pipeline.py +++ b/tools/aot_overlay_pipeline.py @@ -237,12 +237,23 @@ def extent_sources(disc, spec): def pack_sources(disc, spec): """Every member of an offset-indexed LZSS pack decoded and classified.""" file = spec['file'].upper() + data = disc.read(file) + # Other places the loader can take the same pack from (for example a copy + # linked into the boot EXE). Each must hold these exact bytes. + aliases = [] + for copy in spec.get('identical_copies', []): + offset = number(copy.get('file_offset', 0)) + size = number(copy.get('size', len(data))) + require(copy.get('reason', '').strip(), 'Identical pack copy needs a reason') + require(disc.read(copy['file'])[offset:offset + size] == data, + f"Pack copy differs: {copy['file']} {offset:#x}") + aliases.append(f"{copy['file'].upper()}@{offset:X}+{size:X}") lzss = spec['lzss'] require(set(lzss) <= {'position_bits', 'length_bits', 'min_match', 'initial_position', 'window_fill'} and {'position_bits', 'length_bits', 'min_match', 'initial_position'} <= set(lzss), 'LZSS parameters must be declared explicitly') - members = extract_pack_members(disc.read(file), count_offset=number(spec['count_offset']), + members = extract_pack_members(data, count_offset=number(spec['count_offset']), table_offset=number(spec['table_offset']), alignment=number(spec['alignment']), lzss={key: None if value is None else number(value) for key, value in lzss.items()}, count=number(spec['count'])) @@ -266,7 +277,7 @@ def pack_sources(disc, spec): require(in_ram(base, len(body)) and base % 4 == 0, 'Pack member image outside RAM') sources.append(dict(name=f"{file}:ENTRY_{member['index']:04X}", base=base, body=body, spec={**spec, **item, 'allow_missing': True}, source_offset=member['source_offset'], - source_file=file, aliases=[])) + source_file=file, aliases=[f"{alias}:ENTRY_{member['index']:04X}" for alias in aliases])) return sources diff --git a/tools/tests/test_aot_overlay_pipeline.py b/tools/tests/test_aot_overlay_pipeline.py index 5c06de91..4d308320 100644 --- a/tools/tests/test_aot_overlay_pipeline.py +++ b/tools/tests/test_aot_overlay_pipeline.py @@ -699,6 +699,16 @@ class IndexedLzssPackTest(unittest.TestCase): self.assertEqual(inventory['jobs'][0]['required_entries'], [0x80100000]) self.assertEqual(inventory['source_images'][0]['method'], 'indexed_lzss_members') + def test_identical_pack_copy_elsewhere_is_verified_and_recorded(self): + pack = self.pack() + disc = FakeDisc({'PACK.PB': pack, 'BOOT.EXE': b'EXE' + pack + b'tail'}) + copy = dict(file='BOOT.EXE', file_offset=3, size=len(pack), reason='Linked-in copy') + source, = pipeline.positioned_sources(disc, [self.spec(identical_copies=[copy])]) + self.assertEqual(source['aliases'], [f'BOOT.EXE@3+{len(pack):X}:ENTRY_0001']) + for change, error in [(dict(file_offset=4), 'Pack copy differs'), (dict(reason=''), 'reason')]: + with self.subTest(error=error), self.assertRaisesRegex(ValueError, error): + pipeline.positioned_sources(disc, [self.spec(identical_copies=[{**copy, **change}])]) + def test_pipeline_inventory_hash_and_parameters_fail_closed(self): disc = FakeDisc({'PACK.PB': self.pack()}) member = self.spec()['members'][0]