From d1b3022c0df371b7cb32478b2ff93f916bc9676d Mon Sep 17 00:00:00 2001 From: Matt Stanley <1379tech@gmail.com> Date: Mon, 19 Oct 2026 12:29:25 -0700 Subject: [PATCH] Integrate PR645 prepared image discovery and bounded BPE decoding --- docs/AOT_SHARDING.md | 45 +++- recompiler/CMakeLists.txt | 1 + tools/aot_overlay_pipeline.py | 115 ++++++++-- tools/aot_overlay_spike/extract_generic.py | 15 +- tools/bpe_container.py | 115 ++++++++++ tools/prepared_files.py | 138 ++++++++++++ tools/tests/test_aot_overlay_pipeline.py | 244 +++++++++++++++++++++ tools/tests/test_bpe_container.py | 141 ++++++++++++ 8 files changed, 792 insertions(+), 22 deletions(-) create mode 100644 tools/bpe_container.py create mode 100644 tools/prepared_files.py create mode 100644 tools/tests/test_bpe_container.py diff --git a/docs/AOT_SHARDING.md b/docs/AOT_SHARDING.md index bbd4eed9..6d4c67cf 100644 --- a/docs/AOT_SHARDING.md +++ b/docs/AOT_SHARDING.md @@ -57,6 +57,11 @@ Profiles declare these reusable methods: | Field / method | Contract | | --- | --- | | `disc_hashes` | Hash the cue's original data track before extracting. | +| `disc_set` | Instead of `disc_hashes`: `[{name, disc_hashes}]`, every disc of a release whose executable images are identical. Any one matches; the images are pinned by their own content hashes, so which disc was read does not change the output. | +| `file_aliases` | `{logical: [per-disc names]}` for one file that discs name differently (`SCUS_944.91`, `SCUS_945.84`, ...). Exactly one candidate must exist; the logical name reads as that file everywhere in the profile. | +| `prepared_files[]` | Virtual files derived from disc bytes by a declared transform (see below). Each is verified, then usable as a `file` by every method and check. | +| `cross_image_call_roots` | Opt in (`true`): a `jal` target that other images call becomes a static discovery root of the image holding it, when the evidence at the target is the same as for local calls and exactly one image occupies that address. Finds routines only an overlay (or boot code) calls. | +| `generic_exclude_files[]` | `{files, reason}`: disc files the generic detector must not consider because the profile owns them (aliased program files are excluded automatically). | | `images[].method = fixed_address_files` | Whole original files placed at a verified `load_addr`. Optional duplicate-leaf checks require identical bytes. | | `images[].method = fixed_address_extents` | Declared `{file, file_offset, base, address, size, sha256}` byte extents placed at a verified `load_addr`. Offset rules match `words` checks; size and SHA-256 are required. | | `mod_packages[]` / `mod_package` | A named, verified selection of a trusted mod package. Images and checks carrying `mod_package` read the original disc with that selection applied; see below. | @@ -71,7 +76,7 @@ Profiles declare these reusable methods: | `images[].method = indexed_lzss_members` | Read a u32 count and `{offset, size, stored}` descriptors, decode every member (verbatim when `size == stored`, otherwise bit-stream LZSS with declared parameters), require exact sizes, zero padding and a fully accounted file, and classify every member as an executable image at a verified `load_addr` or an explained exclusion. | | `checks[].method = words` | Verify loader instructions or descriptors at explicit file offsets / virtual addresses. | | `checks[].method = tagged_relocations` | Validate a complete relocation stream, retained `image_size` and `relocation_count`, including modules whose heap placement remains unresolved and which are excluded from native production. | -| `checks[].method = pointer_strings` | Verify a pointer-indexed filename table against exact expected strings. | +| `checks[].method = pointer_strings` | Verify a pointer-indexed filename table against exact expected strings. Optional `stride` (default 4, a multiple of 4) reads tables of `{id, name*}` records. | | `checks[].method = bcd_extent_table` | Verify an indexed BCD-MSF/size table against an ISO file's actual extent. | | `checks[].method = adjacent_files` | Require the named ISO extents to be sector aligned and physically adjacent, in order. | | `compositions` | Combine declared simultaneous producers in address order. `max_gap` bounds their separation; gap and alignment bytes remain unowned. | @@ -127,12 +132,50 @@ Current consumers: - Ape Escape USA: `ApeEscapeRecomp/aot/overlays.json`, 47 established packed sector-table overlays plus two minigame PS-X EXEs. Two additional archive members have unresolved load paths and are explicitly excluded. +- The Legend of Dragoon USA: `aot/overlays.json`, `disc_set` over all four discs + (identical executable images). The BPE launcher's embedded container is the engine + (`prepared_files`, two code extents at 0x80010000) and the ten BPE overlays fill + slot A (0x800C6688) or slot B (0x800FB778); all linked with `static`. - WipEout 3 SE PAL: `aot/overlays.json`, the front-end menu overlay (member 5 of `WIPEOUT3/NFE.PB`, `indexed_lzss_members`) from the stock disc and from the NTSC / PAL Mode package's patched disc, plus that package's engine image; all linked with `static` (see below). The other 88 packs on the disc (`TRACK*.PBP`) hold data only. +### Prepared images: packed executables and overlays + +Some engines run code that no disc file holds verbatim: a launcher EXE that +unpacks the real executable, overlays stored compressed. The loader's transform +is deterministic, so its output is an original input like any other, provided the +transform and every byte it reads and writes are pinned. `prepared_files` declares +such outputs; the pipeline rebuilds them from the user's own disc every time and +nothing derived is stored or committed. The profile carries hashes, offsets, sizes +and addresses only. + +```json +"prepared_files": [{ + "name": "PREPARED/ENGINE", "method": "bpe_container", + "source": {"file": "BOOT.EXE", "file_offset": "0xA00"}, "magic": "0x1A455042", + "consumed": "0x36228", "source_sha256": "...", "size": "0xD7800", "sha256": "..." +}] +``` + +The decoder must consume exactly `consumed` bytes, whose SHA-256 is +`source_sha256`, and produce exactly `size` bytes with SHA-256 `sha256`; any +difference stops the run. A prepared name may not be a real disc file or alias, and +a later entry may read an earlier one. Prepared names then work in +`fixed_address_files`, `fixed_address_extents`, `words` and `pointer_strings`, so the +loader code that fixes the destination is verified against the very bytes that run. + +Methods: `bpe_container` (`tools/bpe_container.py`): block-wise byte-pair +expansion, `u32 size, u32 magic, {u32 block size, pair table, symbols}*, 0`. The +magic is a required parameter. + +A packed image is rarely all code (its BSS, tables and replaced overlay area are +data). Use `fixed_address_extents` on the prepared file to name only the code runs, +each with its `sha256`, so data is never a producer: a data run decoded as code +fails the generated-C audit and costs the whole image. + ### Executable images from a mod package A trusted package can add code that no disc file contains: patched boot-EXE diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index 6421bdae..85b19e6f 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -1171,6 +1171,7 @@ if(BUILD_TESTING) # template, scaffold, check_generated.sh, Project Studio audit/ops). foreach(_t IN ITEMS aligned_lzss_banks + bpe_container audit_playtest_diagnostics_package bundled_release_layout collect_game_misses diff --git a/tools/aot_overlay_pipeline.py b/tools/aot_overlay_pipeline.py index 9438b99b..2b490724 100644 --- a/tools/aot_overlay_pipeline.py +++ b/tools/aot_overlay_pipeline.py @@ -27,6 +27,7 @@ from indexed_lzss_pack import members as extract_pack_members from mips_tagged_relocations import parse as parse_tagged_relocations, relocate as relocate_tagged_image from counted_mips_relocations import parse as parse_counted_relocations, relocate as relocate_counted_image from mod_package_images import ModPackageView +import prepared_files FRAMEWORK = Path(__file__).resolve().parents[1] # KSEG0 main-RAM decode window. Retail 2 MiB DRAM mirrors across all of it and @@ -68,9 +69,29 @@ def load_profile(profile_path, config): profile = json.loads(Path(profile_path).read_text(encoding='utf-8-sig')) require(profile.get('schema') == 'psxrecomp AOT methods v1', 'Unsupported AOT profile') require(config['game']['id'] == profile['game_id'], 'AOT profile/game mismatch') + require(('disc_hashes' in profile) != ('disc_set' in profile), + 'An AOT profile declares exactly one of disc_hashes (one disc) and disc_set (several)') return profile +def disc_identities(profile): + """[(name, {algorithm: digest})] for every disc this profile accepts. + + `disc_hashes` is one disc. `disc_set` is a list of {name, disc_hashes}: a + release spread over several discs whose executable images are identical. + The profile's image sources are pinned by content hashes, so they - not the + disc - decide what is built; the disc identity only gates applicability.""" + if 'disc_hashes' in profile: + return [('disc', profile['disc_hashes'])] + entries = profile['disc_set'] + require(isinstance(entries, list) and entries, 'disc_set needs at least one disc') + for entry in entries: + require(entry.get('name') and entry.get('disc_hashes'), + 'Every disc_set entry needs a name and disc_hashes') + require(len({entry['name'] for entry in entries}) == len(entries), 'Duplicate disc_set name') + return [(entry['name'], entry['disc_hashes']) for entry in entries] + + def static_output(profile, project_root): """The overlays_static.c this title's build links, or None. @@ -214,8 +235,10 @@ def verify_evidence(disc, checks, views=None): actual = struct.unpack_from('= 4 and stride % 4 == 0, 'Invalid pointer table stride') for index, expected in enumerate(check['strings']): - offset = origin + number(check['table']) - base + index * 4 + offset = origin + number(check['table']) - base + index * stride pointer = struct.unpack_from('2_000_000: continue if exe_base and os.path.basename(p).upper()==exe_base: continue # skip the statically-compiled base executable diff --git a/tools/bpe_container.py b/tools/bpe_container.py new file mode 100644 index 00000000..1ed1ad81 --- /dev/null +++ b/tools/bpe_container.py @@ -0,0 +1,115 @@ +"""Block-wise byte-pair-expansion ("BPE") containers. + +Some PlayStation engines pack their executable image and overlays with a +byte-pair coder: a table of 256 optional (left, right) pairs per block, then a +stream of symbols each expanded through that table. The container is + + u32 decoded size (all blocks together) + u32 magic (a title-declared constant, "BPE\\x1a" in the + titles seen so far) + block*: + u32 block size bytes this block decodes to (0 ends the container) + table 256 entries, run-length coded (below) + symbols as many input bytes as it takes to produce + `block size` output bytes, each expanded + through the table + +The decoder reads its input as little-endian 32-bit words, handing the words +out as bytes. Anything left unread in the last word of a table or block is +discarded, so every block header starts on a word boundary; `consumed` counts +whole words including the terminating zero count. + +Table coding, entry index `c` starting at 0 until it reaches 256: + * a byte `n` >= 0x80 first skips `n - 0x7F` entries (they stay identity + pairs) and is then treated as `n = 0`; + * the next `n + 1` entries are given in turn, each by a byte `b` that sets + `left[c] = b` and, if `b != c`, a second byte that sets `right[c]`. An + entry with `left[c] == c` is a literal. +Expanding a symbol `b`: while `left[b] != b`, the byte `right[b]` is +queued to follow, and `b` becomes `left[b]`; the final `b` is emitted, then the +queued bytes are expanded the same way, nearest first. A queued byte is +processed before any unread input. Each emitted byte counts against the block +size, whether it came from a symbol or from the queue. + +This module is a pure function of its input. It rejects truncated input, a +declared size that the blocks do not produce, a different magic, and tables +that never finish expanding, instead of returning partial output. +""" +import struct + + +class BpeError(ValueError): + """A container is truncated, inconsistent, or not the declared format.""" + + +def decode(data, *, magic, offset=0, max_output=1 << 28): + """Decode the container at `offset`. Returns (output bytes, consumed bytes). + + `magic` is the second header word the title's loader checks; it is a + required argument because it is evidence about the title, not a default. + """ + if offset < 0 or offset % 4 or len(data) - offset < 12: + raise BpeError('BPE container is truncated or misaligned') + declared, found = struct.unpack_from(' max_output: + raise BpeError('BPE declared size exceeds the output limit') + end = len(data) + position = offset + 8 + output = bytearray() + + def word(): + nonlocal position + if position + 4 > end: + raise BpeError('BPE container is truncated') + value = struct.unpack_from(' declared: + raise BpeError('BPE blocks decode past the declared size') + left, right = list(range(256)), [0] * 256 + window = [] # next byte to hand out is window[-1] + + def take(): + if not window: + window.extend(reversed(data[position:position + 4])) + advance = word # consume the word through the bounds check + advance() + return window.pop() + + index = 0 + while index < 256: + run = take() + if run >= 0x80: + index += run - 0x7F + run = 0 + if index >= 256: + break + for _ in range(run + 1): + if index >= 256: + raise BpeError('BPE table run passes its last entry') + symbol = take() + left[index] = symbol + if symbol != index: + right[index] = take() + index += 1 + budget = (count + 1) * 256 # a table that never reaches a literal + for _ in range(count): + symbol = take() + while left[symbol] != symbol: + window.append(right[symbol]) + symbol = left[symbol] + budget -= 1 + if budget < 0: + raise BpeError('BPE table does not terminate') + output.append(symbol) + window.clear() + if len(output) != declared: + raise BpeError(f'BPE decoded {len(output):#x} bytes, header declares {declared:#x}') + return bytes(output), position - offset diff --git a/tools/prepared_files.py b/tools/prepared_files.py new file mode 100644 index 00000000..33ef9410 --- /dev/null +++ b/tools/prepared_files.py @@ -0,0 +1,138 @@ +"""Virtual files an AOT profile derives from the original disc. + +An engine can run code that no disc file contains verbatim: a boot EXE that +unpacks an inner image, overlays stored compressed. The loader's transform is +deterministic, so its output is as much an original input as a raw file, as +long as the transform and every byte it consumes and produces are pinned. + +A profile declares such outputs in `prepared_files`; each is then an ordinary +named file for every image method and evidence check (`fixed_address_files`, +`words`, `pointer_strings`, ...). Nothing derived is stored: it is rebuilt from +the user's own disc whenever the profile is used, and the profile carries only +hashes, offsets and sizes. + + "prepared_files": [{ + "name": "ENGINE.IMG", # unique; never a real disc file + "method": "bpe_container", + "source": {"file": "BOOT.EXE", "file_offset": "0xA00"}, + "magic": "0x1A455042", # declared by the method + "consumed": "0x36228", # input bytes the decoder reads + "source_sha256": "...", # of exactly those bytes + "size": "0xD7800", # decoded size + "sha256": "..." # of the decoded bytes + }] + +`file_aliases` maps a logical name to the per-disc names of one file, for sets +whose discs name an identical file differently (SCUS_944.91, SCUS_945.84, ...). +Exactly one candidate must exist on the disc being read. + +Methods are decoders over bytes only; they never see a path, the disc, or the +profile's other sections. +""" +import hashlib + +import bpe_container + + +class PreparedError(ValueError): + """A declared derived file did not reproduce from the disc.""" + + +def number(value): + return int(value, 0) if isinstance(value, str) else int(value) + + +def _bpe_container(data, spec): + return bpe_container.decode(data, magic=number(spec['magic']), offset=0) + + +# method -> (decoder(data_from_source_offset, spec) -> (output, consumed), +# keys the spec must declare beyond the common ones) +METHODS = {'bpe_container': (_bpe_container, ('magic',))} +COMMON_KEYS = ('name', 'method', 'source', 'consumed', 'source_sha256', 'size', 'sha256') + + +class PreparedDisc: + """An original disc plus the profile's aliases and derived files. + + Reads of a declared alias or prepared name are answered here; everything + else (`files`, `binary`, `reader`, ...) is the underlying disc's.""" + + def __init__(self, disc, aliases=None, prepared=None, receipts=None): + self.disc = disc + self.aliases = dict(aliases or {}) + self.prepared = dict(prepared or {}) + self.receipts = list(receipts or []) + + def __getattr__(self, name): + # Only reached for attributes this wrapper does not define. + return getattr(self.disc, name) + + def read(self, name): + key = name.upper() + if key in self.prepared: + return self.prepared[key] + return self.disc.read(self.aliases.get(key, key)) + + +def resolve_aliases(disc, profile): + """logical name -> the one candidate that exists on this disc.""" + resolved = {} + for logical, candidates in profile.get('file_aliases', {}).items(): + key = logical.upper() + if key in disc.files: + raise PreparedError(f'File alias {logical} is the name of a real disc file') + if not candidates or not all(isinstance(c, str) and c for c in candidates): + raise PreparedError(f'File alias {logical} needs candidate file names') + present = [c.upper() for c in candidates if c.upper() in disc.files] + if len(present) != 1: + raise PreparedError( + f'File alias {logical} needs exactly one of {sorted(c.upper() for c in candidates)} ' + f'on the disc, found {present}') + resolved[key] = present[0] + return resolved + + +def prepare(disc, profile): + """The disc as the profile sees it: aliases resolved, derived files built. + + Every derived file is verified (input extent, decoded size and hash) before + any image method can read it; a mismatch is a PreparedError.""" + aliases = resolve_aliases(disc, profile) + view = PreparedDisc(disc, aliases) + for spec in profile.get('prepared_files', []): + method = METHODS.get(spec.get('method')) + if method is None: + raise PreparedError(f"Unknown prepared file method: {spec.get('method')}") + decoder, extra = method + missing = [key for key in COMMON_KEYS + extra if key not in spec] + if missing: + raise PreparedError(f"Prepared file {spec.get('name')} lacks {missing}") + name = spec['name'].upper() + if name in view.prepared or name in aliases or name in disc.files: + raise PreparedError(f'Prepared file name is not unique: {name}') + source = spec['source'] + data = view.read(source['file']) + start = number(source.get('file_offset', 0)) + if not 0 <= start < len(data): + raise PreparedError(f'Prepared file {name}: source offset outside {source["file"]}') + try: + output, consumed = decoder(data[start:], spec) + except ValueError as exc: + raise PreparedError(f'Prepared file {name} does not decode: {exc}') from exc + if consumed != number(spec['consumed']): + raise PreparedError(f'Prepared file {name}: decoder consumed {consumed:#x} bytes, ' + f"profile pins {number(spec['consumed']):#x}") + if hashlib.sha256(data[start:start + consumed]).hexdigest() != spec['source_sha256']: + raise PreparedError(f'Prepared file {name}: source bytes changed') + if len(output) != number(spec['size']): + raise PreparedError(f'Prepared file {name}: decoded {len(output):#x} bytes, ' + f"profile pins {number(spec['size']):#x}") + if hashlib.sha256(output).hexdigest() != spec['sha256']: + raise PreparedError(f'Prepared file {name}: decoded bytes changed') + view.prepared[name] = output + view.receipts.append(dict(name=name, method=spec['method'], source=source['file'].upper(), + source_offset=start, consumed=consumed, + source_sha256=spec['source_sha256'], size=len(output), + sha256=spec['sha256'])) + return view diff --git a/tools/tests/test_aot_overlay_pipeline.py b/tools/tests/test_aot_overlay_pipeline.py index f3481e9f..46bf98d5 100644 --- a/tools/tests/test_aot_overlay_pipeline.py +++ b/tools/tests/test_aot_overlay_pipeline.py @@ -15,6 +15,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import aot_overlay_pipeline as pipeline import audit_aot_cache as auditor import indexed_lzss_pack as lzss_pack +import prepared_files +from test_bpe_container import MAGIC as BPE_MAGIC, container as bpe_container_bytes, encode_block class PgxpReleaseNamespaceTests(unittest.TestCase): @@ -904,6 +906,248 @@ class IndexedLzssPackTest(unittest.TestCase): pipeline.positioned_sources(disc, [self.spec(**change)]) +class PreparedImageTests(unittest.TestCase): + """Images a loader unpacks: derived files, per-disc aliases, disc sets.""" + CODE = bytes.fromhex('27bdffe8afbf001000000000') * 4 # invented words + OVERLAY = bytes.fromhex('3c028001244200108c430000') * 3 + + def boot(self): + """A launcher: 8 bytes of header, then a container holding CODE.""" + packed = bpe_container_bytes([encode_block(list(self.CODE), len(self.CODE))], len(self.CODE)) + return b'LAUNCH!!' + packed + b'tail', 8, len(packed), packed + + def profile(self, **overrides): + boot, offset, consumed, packed = self.boot() + profile = dict( + game_id='TEST', + file_aliases={'BOOT.EXE': ['SLUS_001.01', 'SLUS_002.02']}, + prepared_files=[dict(name='PREPARED/ENGINE', method='bpe_container', + source=dict(file='BOOT.EXE', file_offset=offset), magic=hex(BPE_MAGIC), + consumed=consumed, source_sha256=sha(packed), size=len(self.CODE), + sha256=sha(self.CODE))]) + return {**profile, **overrides} + + def disc(self, boot_name='SLUS_001.01', extra=None): + return FakeDisc({boot_name: self.boot()[0], **(extra or {})}) + + def test_prepared_file_reads_like_a_disc_file_for_images_and_evidence(self): + view = prepared_files.prepare(self.disc(), self.profile()) + self.assertEqual(view.read('prepared/engine'), self.CODE) + self.assertEqual(view.read('BOOT.EXE'), self.boot()[0]) + self.assertEqual(view.receipts[0]['sha256'], sha(self.CODE)) + source, = pipeline.positioned_sources(view, [dict( + method='fixed_address_files', files=['PREPARED/ENGINE'], load_addr='0x80010000')]) + self.assertEqual((source['name'], source['base'], source['body']), + ('PREPARED/ENGINE', 0x80010000, self.CODE)) + first, second = struct.unpack_from('> 2) + self.caller = self.image('CALLER', 0x80100000, [jal(self.target), nop, ret, nop]) + + def test_a_target_only_another_image_calls_becomes_a_root(self): + engine, caller = self.engine, self.caller + self.assertEqual(pipeline.foreign_call_roots(engine, [engine, caller]), [self.target]) + self.assertEqual(pipeline.foreign_call_roots(caller, [engine, caller]), []) + local = pipeline.make_fixed_record(engine, None, [engine, caller], False) + cross = pipeline.make_fixed_record(engine, None, [engine, caller], True) + self.assertNotIn(hex(self.target), local.get('static_discovery_entry_pcs', [])) + self.assertIn(hex(self.target), cross['static_discovery_entry_pcs']) + + def test_an_address_several_images_occupy_names_no_one_of_them(self): + engine = self.engine + twin = self.image('TWIN', 0x80010000, [0x03E00008, 0, 0, 0, 0x27BDFFE8, 0, 0x03E00008, 0]) + self.assertEqual(pipeline.foreign_call_roots(engine, [engine, twin, self.caller]), []) + + def test_a_target_without_boundary_evidence_is_not_a_root(self): + data = self.image('DATA', 0x80010000, [0x11111111] * 8) + self.assertEqual(pipeline.foreign_call_roots(data, [data, self.caller]), []) + + class StaticGenerationTest(unittest.TestCase): """The `static` action as `psxrecomp_cli.py generate` drives it.""" diff --git a/tools/tests/test_bpe_container.py b/tools/tests/test_bpe_container.py new file mode 100644 index 00000000..3cc77d77 --- /dev/null +++ b/tools/tests/test_bpe_container.py @@ -0,0 +1,141 @@ +"""BPE container contract on invented bytes; no game data.""" +import struct +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import bpe_container +from bpe_container import BpeError, decode + +MAGIC = 0x1A455042 + + +def encode_table(left, right): + """Run-length table coding with identity stretches skipped.""" + out, c = bytearray(), 0 + while c < 256: + stretch = 0 + while c + stretch < 256 and stretch < 128 and left[c + stretch] == c + stretch: + stretch += 1 + if stretch >= 3: + out.append(0x7E + stretch) # skips stretch - 1 entries, then one entry + out.append(c + stretch - 1) # the last identity entry of the stretch + c += stretch + continue + run = 0 + while c + run < 256 and run < 128 and not ( + left[c + run] == c + run and + all(left[c + run + k] == c + run + k for k in range(1, 3) if c + run + k < 256) and + c + run + 3 <= 256): + run += 1 + run = max(run, 1) + out.append(run - 1) + for k in range(c, c + run): + out.append(left[k]) + if left[k] != k: + out.append(right[k]) + c += run + return bytes(out) + + +def encode_block(symbols, decoded_size, left=None, right=None): + """A block of input `symbols` that decodes to `decoded_size` bytes.""" + left = list(range(256)) if left is None else left + right = [0] * 256 if right is None else right + body = encode_table(left, right) + bytes(symbols) + body += bytes(-len(body) % 4) + return struct.pack(' "AB" + left[0xF1], right[0xF1] = 0xF0, ord('C') # 0xF1 -> "ABC" (chained) + left[0x10], right[0x10] = 0x20, 0x30 # 0x10 -> 0x20 0x30 + return left, right + + def one(self, symbols, size, left=None, right=None): + return container([encode_block(symbols, size, left, right)], size) + + def test_literals_pairs_chains_and_multiple_blocks(self): + left, right = self.pairs() + first = encode_block([0xF1, ord('x'), 0xF0, 0x10], 8, left, right) + second = encode_block(list(b'plain'), 5) + data = container([first, second], 13) + body, consumed = decode(data, magic=MAGIC) + self.assertEqual(body, b'ABCxAB' + bytes([0x20, 0x30]) + b'plain') + self.assertEqual(consumed, len(data)) + + def test_consumed_counts_words_and_offset_selects_the_container(self): + data = self.one(list(b'abc'), 3) + for lead in (b'', bytes(8)): + with self.subTest(lead=len(lead)): + body, consumed = decode(lead + data + b'trailing', magic=MAGIC, offset=len(lead)) + self.assertEqual((body, consumed), (b'abc', len(data))) + + def test_skip_coded_identity_stretches_keep_tables_short_and_decode_correctly(self): + left, right = self.pairs() + table = encode_table(left, right) + self.assertLess(len(table), 40, 'identity stretches must be skip-coded') + body, _ = decode(self.one([0xF1, 0x10, 0x41, 0xF0], 8, left, right), magic=MAGIC) + self.assertEqual(body, b'ABC' + bytes([0x20, 0x30]) + b'A' + b'AB') + + def test_a_skip_that_reaches_the_last_entry_ends_the_table(self): + # 128 explicit entries (entry 0 is a pair), then a 128-entry skip: index 256. + table = bytes([127, ord('Z'), ord('Y')]) + bytes(range(1, 128)) + bytes([0xFF]) + block = struct.pack(' 255, entry 255 + block = struct.pack(' 256. + table = (bytes([127]) + bytes(range(128)) + bytes([100]) + bytes(range(128, 229)) + + bytes([40]) + bytes(range(229, 256)) + bytes(16)) + block = struct.pack('