diff --git a/docs/AOT_SHARDING.md b/docs/AOT_SHARDING.md index 5cc05958..d4154cec 100644 --- a/docs/AOT_SHARDING.md +++ b/docs/AOT_SHARDING.md @@ -61,6 +61,7 @@ Profiles declare these reusable methods: | `images[].method = psx_exe` | Read the load address and image from a PS-X EXE header; the generic extractor may split resident and overlay floors. | | `images[].method = packed_sector_members` | Decode u32 count/offset sector descriptors across ordered payload files. Verify every member classification and requested full-payload coverage; use loader-established addresses for executable members. | | `images[].method = sector_extent_members` | Decode `{sector offset, byte size}` pairs in an archive. Use archive-relative offsets, verify the exact count and terminator, reject gaps/overlaps, and account for every member after sector rounding. Each member needs a verified load address or an explicit exclusion. | +| `images[].method = aligned_lzss_banks` | Read aligned stored-size pairs, decode the metadata and its tagged bank members using parameterized LZSS, verify every container and bank classification, and deduplicate exact decoded images at verified destinations. External-RAM references fail extraction. | | `checks[].method = words` | Verify loader instructions or descriptors at explicit file offsets / virtual addresses. | | `checks[].method = pointer_strings` | Verify a pointer-indexed filename table against exact expected strings. | | `checks[].method = bcd_extent_table` | Verify an indexed BCD-MSF/size table against an ISO file's actual extent. | @@ -86,6 +87,8 @@ release packaging. Pair counts are results, not inherited success thresholds. An inventory plus valid guards still does not prove complete static execution coverage or native semantics; keep fallback and perform gameplay spot checks. +Jersey Devil uses `aligned_lzss_banks` for five renderer variants across 70 bank occurrences and `psx_exe` for five secondary executables. Its profile accounts for all 93 BZZ containers, including 23 without code banks. + Current consumers: - Tomba! USA: `TombaRecomp/aot/overlays.json`, 25 fixed-address images, including diff --git a/tools/aligned_lzss_banks.py b/tools/aligned_lzss_banks.py new file mode 100644 index 00000000..7b3be290 --- /dev/null +++ b/tools/aligned_lzss_banks.py @@ -0,0 +1,104 @@ +"""Sector-aligned size-pair containers with tagged, compressed code banks. + +The compression stream has a parameter byte, a big-endian 24-bit token count +minus one, LSB-first literal flags, and 16-bit big-endian backward references. +The parameter selects distance/length bits and a piecewise length table. +No emulated RAM, captured code, or game-specific names are inputs. +""" +import struct + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def decode(data, *, max_output=2_000_000): + require(len(data) >= 5 and max_output > 0, 'Truncated LZSS stream or invalid limit') + parameter = data[0] + require(parameter < 32, 'Unsupported LZSS parameter bits') + length_bits = 7 - (parameter & 7) + mask = 127 >> (parameter & 7) + threshold = 19 if mask >= 31 else mask // 2 + shift = (parameter >> 3) & 3 + lengths = [(i if i <= threshold else threshold + ((i - threshold) << shift)) + 3 + for i in range(mask + 1)] + tokens = int.from_bytes(data[1:4], 'big') + 1 + require(tokens <= max_output, 'LZSS token count exceeds output limit') + position, flags = 4, 0 + output = bytearray() + + def byte(): + nonlocal position + require(position < len(data), 'Truncated LZSS token') + value = data[position] + position += 1 + return value + + for _ in range(tokens): + if flags <= 1: + flags = byte() | 256 + literal = flags & 1 + flags >>= 1 + token = byte() + if literal: + require(len(output) < max_output, 'LZSS output exceeds limit') + output.append(token) + else: + token = token * 256 + byte() + distance, length = token >> length_bits, lengths[token & mask] + # A zero/out-of-image reference depends on pre-existing RAM. It + # cannot produce deterministic original-disc AOT code. + require(0 < distance <= len(output), 'LZSS reference depends on external RAM') + require(len(output) + length <= max_output, 'LZSS output exceeds limit') + for _ in range(length): + output.append(output[-distance]) + return bytes(output), position + + +def members(data, *, alignment=2048, version=1): + require(alignment >= 16 and len(data) >= alignment, 'Truncated size-pair archive') + actual_version, count = struct.unpack_from(' 0 and padded == (stored + alignment - 1) // alignment * alignment, + 'Invalid aligned member size') + require(offset + padded <= len(data), 'Truncated aligned member') + result.append(dict(index=index, offset=offset, size=stored, + body=data[offset:offset + stored])) + offset += padded + require(offset == len(data), 'Size-pair inventory does not cover archive') + return result + + +def banks(data, *, alignment=2048, version=1, bank_tag=0x4B, + terminal_tag=0x31, data_tags=(0x30,), max_output=2_000_000): + """Decode only metadata and its explicit leading bank members. + + Remaining members are data payloads, deliberately not decoded as code. + The caller supplies loader-established bank-index to RAM-address mappings. + """ + inventory = members(data, alignment=alignment, version=version) + metadata, _ = decode(inventory[0]['body'], max_output=max_output) + position, result = 0, [] + while True: + require(position + 4 <= len(metadata), 'Truncated bank selector metadata') + tag = struct.unpack_from('> 26 in (6, 7) and (word >> 16) & 31: + return False if _is_control_flow_word(word): delay = _word(body, at + 4) return delay is not None and not _is_control_flow_word(delay) @@ -1000,6 +1005,8 @@ def main(): aliases=[alias for alias in aliases_all if va <= alias[0] < span_hi and va <= alias[1] < alias[2] <= span_hi and + (alias[0] == entry_pc or + optional_entry_delay_valid(body, t_addr, alias[0])) and (alias[0] != t_addr or alias[0] == entry_pc)] records.append(rec( va,seg,sd,static_alias_ranges=aliases)) diff --git a/tools/tests/test_aligned_lzss_banks.py b/tools/tests/test_aligned_lzss_banks.py new file mode 100644 index 00000000..7aae6494 --- /dev/null +++ b/tools/tests/test_aligned_lzss_banks.py @@ -0,0 +1,100 @@ +import struct +import hashlib +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from aligned_lzss_banks import decode, members, banks +from aot_overlay_spike.extract_generic import filter_full_discovery_seeds +from aot_overlay_pipeline import positioned_sources + + +def literal_stream(body): + out = bytearray(b'\0' + (len(body) - 1).to_bytes(3, 'big')) + for offset in range(0, len(body), 8): + block = body[offset:offset + 8] + out.append((1 << len(block)) - 1) + out.extend(block) + return bytes(out) + + +def archive(*payloads): + head = bytearray(64) + struct.pack_into(' 24 more As. + self.assertEqual(decode(bytes.fromhex('0800000101410094'))[0], b'A' * 25) + # Parameter 7: zero length bits, distance occupies all 16 bits. + self.assertEqual(decode(bytes.fromhex('0700000101410001'))[0], b'AAAA') + + def test_external_ram_and_truncation_rejected(self): + for stream in [b'', bytes.fromhex('000000010141'), + bytes.fromhex('0000000101410000'), + bytes.fromhex('0000000101410100')]: + with self.subTest(stream=stream), self.assertRaises(ValueError): + decode(stream) + with self.assertRaises(ValueError): + decode(literal_stream(b'ab'), max_output=1) + + def test_selected_bank_and_uninterpreted_data(self): + meta = literal_stream(struct.pack('