From 12d13deff4a755d2e82ccabca337f3726f0d6c03 Mon Sep 17 00:00:00 2001 From: Matthew Stanley <1379tech@gmail.com> Date: Sat, 26 Sep 2026 06:26:48 -0700 Subject: [PATCH] aot: indexed_lzss_members method for offset-indexed bit-stream LZSS packs Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/AOT_SHARDING.md | 44 ++++++- tools/aot_overlay_pipeline.py | 41 +++++- tools/indexed_lzss_pack.py | 115 +++++++++++++++++ tools/tests/test_aot_overlay_pipeline.py | 151 +++++++++++++++++++++++ 4 files changed, 348 insertions(+), 3 deletions(-) create mode 100644 tools/indexed_lzss_pack.py diff --git a/docs/AOT_SHARDING.md b/docs/AOT_SHARDING.md index 1b0061ba..f7d9ba96 100644 --- a/docs/AOT_SHARDING.md +++ b/docs/AOT_SHARDING.md @@ -66,6 +66,7 @@ Profiles declare these reusable methods: | `images[].method = packed_sector_members` | Decode u32 count/offset sector descriptors across ordered payload files. Verify every member classification and requested full-payload coverage; use loader-established addresses for executable members. | | `images[].method = sector_extent_members` | Decode `{sector offset, byte size}` pairs in an archive. Use archive-relative offsets, verify the exact count and terminator, reject gaps/overlaps, and account for every member after sector rounding. Each member needs a verified load address or an explicit exclusion. | | `images[].method = aligned_lzss_banks` | Read aligned stored-size pairs, decode the metadata and its tagged bank members using parameterized LZSS, verify every container and bank classification, and deduplicate exact decoded images at verified destinations. External-RAM references fail extraction. | +| `images[].method = indexed_lzss_members` | Read a u32 count and `{offset, size, stored}` descriptors, decode every member (verbatim when `size == stored`, otherwise bit-stream LZSS with declared parameters), require exact sizes, zero padding and a fully accounted file, and classify every member as an executable image at a verified `load_addr` or an explained exclusion. | | `checks[].method = words` | Verify loader instructions or descriptors at explicit file offsets / virtual addresses. | | `checks[].method = tagged_relocations` | Validate a complete relocation stream, retained `image_size` and `relocation_count`, including modules whose heap placement remains unresolved and which are excluded from native production. | | `checks[].method = pointer_strings` | Verify a pointer-indexed filename table against exact expected strings. | @@ -121,8 +122,11 @@ Current consumers: - Ape Escape USA: `ApeEscapeRecomp/aot/overlays.json`, 47 established packed sector-table overlays plus two minigame PS-X EXEs. Two additional archive members have unresolved load paths and are explicitly excluded. -- WipEout 3 SE PAL: `aot/overlays.json`, no disc overlays; one mod-package - engine image linked with `static` (see below). +- WipEout 3 SE PAL: `aot/overlays.json`, the front-end menu overlay (member 5 + of `WIPEOUT3/NFE.PB`, `indexed_lzss_members`) from the stock disc and from the + framerate package's patched disc, plus that package's engine image; all + linked with `static` (see below). The other 88 packs on the disc (`TRACK*.PBP`) + hold data only. ### Executable images from a mod package @@ -186,6 +190,42 @@ Exclusions require a `reason`. `cover_payloads` names files that this entire classified range must cover exactly, without gaps or overlaps. The shared reader is `tools/packed_sector_table.py`; heuristic discovery uses that same parser. +For `indexed_lzss_members` (reader: `tools/indexed_lzss_pack.py`), declare +`file`, `count_offset` (u32 member count), `table_offset` (12-byte +`{u32 offset, u32 size, u32 stored}` descriptors), `alignment` of each member +start, the exact `count`, and every LZSS parameter explicitly; there are no +defaults. Members follow the table in descriptor order. Every other byte +before the end of the table, all padding and any tail after the last member +must be zero. A member whose stored size differs from its size is a bit-stream +LZSS stream that must decode to exactly `size` bytes while consuming exactly +`stored` bytes: + +| LZSS parameter | Meaning | +| --- | --- | +| `position_bits` | Width of a window position; the ring window holds `1 << position_bits` bytes. Position 0 ends the stream. | +| `length_bits` | Width of the copy count field. | +| `min_match` | Bytes copied for a count of zero; a reference copies `count + min_match` bytes. | +| `initial_position` | Window slot receiving the first output byte. | +| `window_fill` | Optional initial byte of every window slot. Omit it when the original decoder leaves its window uninitialized: reading a slot this stream never wrote then fails extraction as a dependency on pre-existing RAM. | + +Bits are read most significant first. A 1 flag precedes an 8-bit literal; a 0 +flag precedes a position and a count. Each output byte is also stored at the +current window slot, so a reference may read bytes it has just written. +Executable members carry `index`, `load_addr`, `decoded_sha256` and optionally +verified `entries`; `excluded_members` need a `reason`. Every index must be +classified exactly once. + +WipEout 3's `MenuControl` (0x801179B8) opens `c:\wipeout3\nfe.pb`, requires its +allocation of the top member to land exactly at 0x800BF800, has the pack reader +run `lzss.c` `ExpandData` (13-bit position, 4-bit count, three-byte minimum, +start position 1, uninitialized 8 KiB window) and copies the decoded 0x21424 +bytes there without relocation, then calls five of its functions directly. The +profile pins those instructions with `words` checks. The decoded image is +byte-identical to the RAM at 0x800BF800 in a stock run's menu. The same reader +and decoder account for all 5,178 members of the disc's 89 packs. This is the +only member the executable transfers control to, and the only one with MIPS +function shapes; the rest go to data loaders (textures, models, track tables). + See `tools/tests/test_aot_overlay_pipeline.py` for synthetic method and release failure tests. A new title should add a profile when these contracts fit; add and test a shared method only when its actual loader requires one. diff --git a/tools/aot_overlay_pipeline.py b/tools/aot_overlay_pipeline.py index 2d05891a..9601942f 100644 --- a/tools/aot_overlay_pipeline.py +++ b/tools/aot_overlay_pipeline.py @@ -22,6 +22,7 @@ from aot_overlay_spike import extract_generic as extractor from packed_sector_table import extract_members as extract_sector_members from sector_extent_archive import extract_members as extract_extent_members from aligned_lzss_banks import banks as extract_lzss_banks +from indexed_lzss_pack import members as extract_pack_members from mips_tagged_relocations import parse as parse_tagged_relocations, relocate as relocate_tagged_image from mod_package_images import ModPackageView @@ -233,6 +234,42 @@ def extent_sources(disc, spec): return sources +def pack_sources(disc, spec): + """Every member of an offset-indexed LZSS pack decoded and classified.""" + file = spec['file'].upper() + lzss = spec['lzss'] + require(set(lzss) <= {'position_bits', 'length_bits', 'min_match', 'initial_position', + 'window_fill'} and + {'position_bits', 'length_bits', 'min_match', 'initial_position'} <= set(lzss), + 'LZSS parameters must be declared explicitly') + members = extract_pack_members(disc.read(file), count_offset=number(spec['count_offset']), + table_offset=number(spec['table_offset']), alignment=number(spec['alignment']), + lzss={key: None if value is None else number(value) for key, value in lzss.items()}, + count=number(spec['count'])) + configured = {number(item['index']): item for item in spec['members']} + excluded = {number(item['index']): item for item in spec.get('excluded_members', [])} + require(len(configured) == len(spec['members']) and + len(excluded) == len(spec.get('excluded_members', [])), 'Duplicate member inventory') + require(not configured.keys() & excluded.keys() and + configured.keys() | excluded.keys() == set(range(len(members))), + 'Pack inventory has missing or conflicting classifications') + require(all(item.get('reason', '').strip() for item in excluded.values()), + 'Excluded pack member needs a reason') + sources = [] + for member in members: + item = configured.get(member['index']) + if item is None: + continue + body, base = member['body'], number(item['load_addr']) + require(hashlib.sha256(body).hexdigest() == item['decoded_sha256'], + f"Decoded pack member changed: {file} {member['index']}") + require(in_ram(base, len(body)) and base % 4 == 0, 'Pack member image outside RAM') + sources.append(dict(name=f"{file}:ENTRY_{member['index']:04X}", base=base, body=body, + spec={**spec, **item, 'allow_missing': True}, source_offset=member['source_offset'], + source_file=file, aliases=[])) + return sources + + def positioned_sources(disc, specifications, views=None, ram_bytes=RETAIL_RAM_BYTES): sources = [] for spec in specifications: @@ -303,6 +340,8 @@ def spec_sources(disc, spec): if method == 'packed_sector_members': sources.extend(sector_sources(disc, spec)) return sources + if method == 'indexed_lzss_members': + return pack_sources(disc, spec) if method == 'sector_extent_members': members = extract_extent_members(disc.read(spec['file']), sector_size=number(spec.get('sector_size', 2048)), @@ -474,7 +513,7 @@ def prepare(profile, disc, records, output, views=None): if (lo, hi) == (source['base'], source['base'] + len(source['body'])): singles[source['name']] = record for source in sources: - if source['spec']['method'] in ('fixed_address_files', 'fixed_address_extents', 'packed_sector_members', 'sector_extent_members', 'aligned_lzss_banks', 'tagged_relocated_files') and source['name'] not in singles: + if source['spec']['method'] in ('fixed_address_files', 'fixed_address_extents', 'packed_sector_members', 'sector_extent_members', 'aligned_lzss_banks', 'indexed_lzss_members', 'tagged_relocated_files') and source['name'] not in singles: record = make_fixed_record(source, disc) singles[source['name']] = record records.append(record) diff --git a/tools/indexed_lzss_pack.py b/tools/indexed_lzss_pack.py new file mode 100644 index 00000000..bc3f1442 --- /dev/null +++ b/tools/indexed_lzss_pack.py @@ -0,0 +1,115 @@ +"""Offset-indexed packs whose members are stored verbatim or as bit-stream LZSS. + +Container layout (all integers little-endian): + +* ``count_offset``: u32 member count. Every other byte before the end of the + descriptor table is unknown to this reader and must be zero. +* ``table_offset``: ``count`` descriptors ``{u32 offset, u32 size, u32 stored}``. + ``stored == size`` means the member is stored verbatim; otherwise its + ``stored`` bytes are an LZSS stream that decodes to exactly ``size`` bytes. +* Members follow the table in descriptor order, each starting at the next + ``alignment`` boundary. Every byte after the table is a member or zero + padding (including any tail after the last member, such as the sector fill + of a pack that a patch shortened), so every byte is accounted for. + +The LZSS stream is read one bit at a time, most significant bit first, a byte +at a time. A 1 flag is followed by an 8-bit literal; a 0 flag by a +``position_bits`` window position and a ``length_bits`` count, copying +``count + min_match`` bytes from consecutive window positions. Position 0 ends +the stream. Every output byte is also written to the ring window, starting at +``initial_position``. When ``window_fill`` is None the decoder's window starts +uninitialized, so reading a position that was never written depends on +pre-existing RAM and fails extraction. + +No emulated RAM, captured code or game-specific names are inputs. +""" +import struct + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def decode(data, offset, *, position_bits, length_bits, min_match, initial_position, + window_fill=None, max_output=4 << 20): + """Return (decoded bytes, stream bytes consumed from ``offset``).""" + require(1 <= position_bits <= 16 and 1 <= length_bits <= 8 and min_match >= 1, + 'Unsupported LZSS field widths') + size = 1 << position_bits + require(0 < initial_position < size, 'LZSS initial position outside window') + require(window_fill is None or 0 <= window_fill <= 255, 'Invalid LZSS window fill') + require(0 <= offset <= len(data) and max_output > 0, 'Invalid LZSS stream offset or limit') + window = [window_fill] * size + cursor, position, current, mask = initial_position, offset, 0, 0 + output = bytearray() + + def bits(count): + nonlocal position, current, mask + value = 0 + for _ in range(count): + if not mask: + require(position < len(data), 'Truncated LZSS stream') + current, mask = data[position], 0x80 + position += 1 + value = value << 1 | (1 if current & mask else 0) + mask >>= 1 + return value + + def emit(value): + nonlocal cursor + require(len(output) < max_output, 'LZSS output exceeds limit') + output.append(value) + window[cursor] = value + cursor = (cursor + 1) & (size - 1) + + while True: + if bits(1): + emit(bits(8)) + continue + source = bits(position_bits) + if source == 0: + return bytes(output), position - offset + for step in range(bits(length_bits) + min_match): + value = window[(source + step) & (size - 1)] + # The original decoder's window is not cleared: a slot never + # written by this stream holds whatever RAM held before. + require(value is not None, 'LZSS reference depends on external RAM') + emit(value) + + +def members(data, *, count_offset, table_offset, alignment, lzss, count=None): + """Decode and account for every member; return them in descriptor order.""" + require(alignment >= 1 and alignment & (alignment - 1) == 0, + 'Pack alignment must be a power of two') + require(0 <= count_offset and count_offset + 4 <= len(data), 'Pack count outside file') + actual = struct.unpack_from(' 0 and (count is None or actual == count), 'Pack member count changed') + table_end = table_offset + actual * 12 + require(0 <= table_offset and table_end <= len(data), 'Pack descriptor table outside file') + require(not (count_offset < table_end and table_offset < count_offset + 4), + 'Pack count overlaps descriptor table') + header = bytearray(data[:table_end]) + header[count_offset:count_offset + 4] = bytes(4) + header[table_offset:table_end] = bytes(table_end - table_offset) + require(not any(header), 'Unexpected pack header bytes') + align = lambda value: (value + alignment - 1) & ~(alignment - 1) + cursor, result = table_end, [] + for index in range(actual): + offset, size, stored = struct.unpack_from(' 0 and stored > 0 and offset + stored <= len(data), + 'Pack member exceeds file or is empty') + if stored == size: + body, compressed = data[offset:offset + size], False + else: + body, consumed = decode(data, offset, **lzss, max_output=size) + require(len(body) == size and consumed == stored, + f'Pack member {index} does not decode to its declared sizes') + compressed = True + result.append(dict(index=index, source_offset=offset, size=size, stored=stored, + compressed=compressed, body=body)) + cursor = offset + stored + require(not any(data[cursor:]), 'Pack has nonzero bytes outside its members') + return result diff --git a/tools/tests/test_aot_overlay_pipeline.py b/tools/tests/test_aot_overlay_pipeline.py index fa70a09f..5c06de91 100644 --- a/tools/tests/test_aot_overlay_pipeline.py +++ b/tools/tests/test_aot_overlay_pipeline.py @@ -12,6 +12,7 @@ from unittest import mock sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import aot_overlay_pipeline as pipeline import audit_aot_cache as auditor +import indexed_lzss_pack as lzss_pack class FakeDisc: @@ -563,5 +564,155 @@ class AotMethodsTest(unittest.TestCase): pipeline.stage(root / 'cache', root / 'stage', receipt) +def bit_lzss(tokens, position_bits, length_bits): + """Invented bit-stream LZSS: ints are literals, (position, count) references.""" + bits = [] + field = lambda value, width: [(value >> (width - 1 - i)) & 1 for i in range(width)] + for token in tokens: + if isinstance(token, int): + bits += [1] + field(token, 8) + else: + bits += [0] + field(token[0], position_bits) + field(token[1], length_bits) + bits += [0] + field(0, position_bits) + bits += [0] * (-len(bits) % 8) + return bytes(int(''.join(map(str, bits[i:i + 8])), 2) for i in range(0, len(bits), 8)) + + +def indexed_pack(members, count_offset=0x10, table_offset=0x18, alignment=4): + """members: (decoded size, stored bytes); equal lengths mean stored verbatim.""" + data = bytearray(table_offset + 12 * len(members)) + struct.pack_into('