Add declarative original-disc structural inventory inspection

docs/AOT_SHARDING.md

@@ -259,6 +259,34 @@ with table-address setup around the bounds branch. Register dependencies,
 producer ownership and table-target checks remain required. That improvement
 contains no Tomba-specific addresses and can benefit other matching binaries.
 
+## When discovery finds no game overlay
+
+An archive extension or a historical dirty-RAM capture does not establish an
+executable overlay. A resident-code game is a valid outcome of bounded disc
+inspection. Do not count an independently identified BIOS RAM helper as a game
+overlay, or manufacture a release inventory from runtime observations.
+
+`tools/inspect_disc_inventory.py` verifies a declarative `psxrecomp disc inventory
+v1` profile against the original disc. It reuses the AOT evidence-word checks,
+matches fixed-stride `{LBA, byte-size}` tables to ISO files, and exposes
+`typed_members(data, alignment)` for archives with a `{count, total-size}` header
+and `{type, byte-size}` descriptors followed by aligned members. Repeated types
+are allowed; declared size, count, bounds and full payload coverage are required.
+Header bytes beyond the counted descriptors are opaque.
+
+The metadata-only report accounts for every ISO file and reports member hashes,
+types and aligned MIPS instruction shapes. These shapes are observations, not
+proof of code presence or absence; loader destinations, callbacks and any
+decompression path still need bounded inspection of the original executable.
+The game profile supplies all addresses, filenames, counts and rationales. The
+shared tool contains no game-specific branches or bundled game bytes.
+
+```sh
+python3 psxrecomp-v4/tools/inspect_disc_inventory.py \
+  --profile aot/disc_inventory.json --cue "path/to/original-disc.cue" \
+  --output build/disc-inventory.json
+```
+
 ## Related material
 
 - [Compiling and packaging overlays](COMPILING_OVERLAYS.md): cache and static-link workflows.

tests/test_disc_inventory.py

@@ -0,0 +1,92 @@
+"""Synthetic structural fixtures; no retail game bytes."""
+from pathlib import Path
+import struct
+import sys
+import unittest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
+from inspect_disc_inventory import inspect, typed_members, mips_shapes
+
+
+class TypedArchiveTest(unittest.TestCase):
+    def fixture(self):
+        data = bytearray(3 * 32)
+        struct.pack_into('<6I', data, 0, 2, len(data), 7, 5, 7, 31)
+        data[32:37] = b'first'
+        data[64:95] = b'z' * 31
+        return data
+
+    def test_duplicate_types_and_partial_sectors(self):
+        members = typed_members(self.fixture(), 32)
+        self.assertEqual([(m['kind'], m['offset'], len(m['body'])) for m in members],
+                         [(7, 32, 5), (7, 64, 31)])
+
+    def test_reject_size_count_and_extent_errors(self):
+        for offset, value in [(0, 4), (4, 95), (12, 90), (20, 33)]:
+            with self.subTest(offset=offset):
+                data = self.fixture()
+                struct.pack_into('<I', data, offset, value)
+                with self.assertRaises(ValueError):
+                    typed_members(data, 32)
+
+    def test_header_padding_is_opaque(self):
+        data = self.fixture()
+        data[24:32] = b'metadata'
+        self.assertEqual(len(typed_members(data, 32)), 2)
+
+    def test_reject_unclaimed_payload(self):
+        data = self.fixture() + bytearray(32)
+        struct.pack_into('<I', data, 4, len(data))
+        with self.assertRaisesRegex(ValueError, 'coverage'):
+            typed_members(data, 32)
+
+    def test_shapes_are_aligned_and_not_a_classifier(self):
+        data = struct.pack('<III', 0x03e00008, 0x27bdffe0, 0x27bd0020) + b'x'
+        self.assertEqual(mips_shapes(data), {'aligned_jr_ra': 1, 'negative_sp_adjust': 1})
+
+
+class FileTableTest(unittest.TestCase):
+    def fixture(self):
+        class Disc:
+            files = {'EXE': (10, 12), 'ASSET': (20, 4)}
+
+            def read(self, name):
+                return {'EXE': struct.pack('<III', 20, 4, 123),
+                        'ASSET': struct.pack('<I', 123)}[name]
+        profile = dict(schema='psxrecomp disc inventory v1', game_id='synthetic',
+            disc_hashes={}, expected_groups={},
+            file_tables=[dict(file='EXE', address=0, stride=12, lba_offset=0,
+                size_offset=4, first_word_offset=8, files=['ASSET'])],
+            files=[dict(file=name, method='raw_file', role='Synthetic fixture')
+                   for name in Disc.files])
+        return profile, Disc()
+
+    def test_full_inventory_and_extent_match(self):
+        profile, disc = self.fixture()
+        self.assertEqual(len(inspect(profile, disc)['files']), 2)
+
+    def test_mismatched_iso_extent_is_rejected(self):
+        profile, disc = self.fixture()
+        disc.files = {**disc.files, 'ASSET': (21, 4)}
+        with self.assertRaisesRegex(ValueError, 'ISO/table mismatch'):
+            inspect(profile, disc)
+
+    def test_unclassified_file_is_rejected(self):
+        profile, disc = self.fixture()
+        profile['files'].pop()
+        with self.assertRaisesRegex(ValueError, 'complete ISO'):
+            inspect(profile, disc)
+
+    def test_malformed_record_layout_is_rejected(self):
+        for field, value in [('stride', 0), ('stride', 3), ('lba_offset', -4),
+                             ('size_offset', 1), ('size_offset', 12),
+                             ('first_word_offset', 12)]:
+            with self.subTest(field=field, value=value):
+                profile, disc = self.fixture()
+                profile['file_tables'][0][field] = value
+                with self.assertRaisesRegex(ValueError, 'stride|field offset'):
+                    inspect(profile, disc)
+
+
+if __name__ == '__main__':
+    unittest.main()

tools/inspect_disc_inventory.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Verify declarative original-disc inventories without producing code recipes.
+
+Reports contain metadata only. MIPS instruction-shape counts are observations,
+not an executable-code classifier or proof that compressed code cannot exist.
+"""
+import argparse
+from collections import Counter
+import hashlib
+import json
+from pathlib import Path
+import struct
+
+from aot_overlay_pipeline import Disc, digest, number, require, verify_evidence, write_json
+
+
+def typed_members(data, alignment=2048):
+    """Count/total-size header, then {type, byte-size}, aligned payloads."""
+    require(alignment >= 8 and alignment & (alignment - 1) == 0,
+            'Alignment must be a power of two')
+    require(len(data) >= alignment and len(data) % alignment == 0,
+            'Archive is not sector aligned')
+    count, total = struct.unpack_from('<II', data)
+    require(total == len(data), 'Archive total size mismatch')
+    require(0 < count <= (alignment - 8) // 8, 'Invalid descriptor count')
+    # The count bounds the descriptor table; remaining header bytes are opaque.
+    cursor, members = alignment, []
+    for index in range(count):
+        kind, size = struct.unpack_from('<II', data, 8 + index * 8)
+        require(size > 0 and cursor + size <= len(data), 'Member outside archive')
+        members.append(dict(index=index, kind=kind, offset=cursor,
+                            body=data[cursor:cursor + size]))
+        cursor = (cursor + size + alignment - 1) & ~(alignment - 1)
+    require(cursor == len(data), 'Archive payload coverage mismatch')
+    return members
+
+
+def mips_shapes(data):
+    words = (struct.unpack_from('<I', data, i)[0] for i in range(0, len(data) - 3, 4))
+    returns = frames = 0
+    for word in words:
+        returns += word == 0x03e00008
+        frames += (word & 0xffff8000) == 0x27bd8000
+    return dict(aligned_jr_ra=returns, negative_sp_adjust=frames)
+
+
+def inspect(profile, disc):
+    require(profile['schema'] == 'psxrecomp disc inventory v1', 'Unknown inventory schema')
+    for algorithm, expected in profile['disc_hashes'].items():
+        require(digest(disc.binary, algorithm) == expected, f'Unsupported disc {algorithm}')
+    verify_evidence(disc, profile.get('checks', []))
+    for table in profile.get('file_tables', []):
+        data = disc.read(table['file'])
+        origin = number(table.get('file_offset', 0))
+        offset = origin + number(table['address']) - number(table.get('base', 0))
+        stride = number(table['stride'])
+        require(stride >= 4, 'File table stride must be at least four bytes')
+        fields = ['lba_offset', 'size_offset']
+        if 'first_word_offset' in table:
+            fields.append('first_word_offset')
+        for field in fields:
+            field_offset = number(table[field])
+            require(0 <= field_offset <= stride - 4 and field_offset % 4 == 0,
+                    f'Invalid file table field offset: {field}')
+        for index, name in enumerate(table['files']):
+            at = offset + index * stride
+            require(0 <= at <= len(data) - stride, 'File table outside source')
+            lba = struct.unpack_from('<I', data, at + number(table['lba_offset']))[0]
+            size = struct.unpack_from('<I', data, at + number(table['size_offset']))[0]
+            require((lba, size) == disc.files[name.upper()], f'ISO/table mismatch: {name}')
+            if 'first_word_offset' in table:
+                word = struct.unpack_from('<I', data, at + number(table['first_word_offset']))[0]
+                require(word == struct.unpack_from('<I', disc.read(name))[0],
+                        f'File table/header mismatch: {name}')
+    groups = Counter()
+    files, seen = [], set()
+    for spec in profile['files']:
+        name = spec['file'].upper()
+        require(name not in seen, f'Duplicate file classification: {name}')
+        seen.add(name)
+        require(spec.get('role', '').strip(), f'Missing classification rationale: {name}')
+        data = disc.read(name)
+        info = dict(file=name, role=spec['role'], lba=disc.files[name][0], size=len(data),
+                    sha256=hashlib.sha256(data).hexdigest())
+        if spec['method'] == 'typed_aligned_archive':
+            members = typed_members(data, number(spec.get('alignment', 2048)))
+            require(len(members) == spec['member_count'], f'Member count drift: {name}')
+            info['members'] = []
+            for member in members:
+                body = member.pop('body')
+                groups[member['kind'] >> number(profile.get('group_shift', 16))] += 1
+                info['members'].append(dict(**member, size=len(body),
+                    sha256=hashlib.sha256(body).hexdigest(), mips_shapes=mips_shapes(body)))
+        elif spec['method'] == 'raw_file':
+            if spec.get('inspect_mips_shapes'):
+                info['mips_shapes'] = mips_shapes(data)
+        else:
+            raise ValueError(f"Unknown inventory method: {spec['method']}")
+        files.append(info)
+    require(seen == set(disc.files), 'Inventory does not classify the complete ISO file list')
+    require(dict(groups) == {number(k): v for k, v in profile['expected_groups'].items()},
+            'Member group count drift')
+    return dict(schema='psxrecomp disc inspection v1', game_id=profile['game_id'],
+                disc_hashes=profile['disc_hashes'], files=files, groups=dict(groups),
+                limitation='Structural verification and instruction shapes do not prove code absence.')
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--profile', required=True, type=Path)
+    parser.add_argument('--cue', required=True, type=Path)
+    parser.add_argument('--output', required=True, type=Path)
+    args = parser.parse_args()
+    profile = json.loads(args.profile.read_text(encoding='utf-8'))
+    report = inspect(profile, Disc(args.cue))
+    write_json(args.output, report)
+    print(f"Verified {len(report['files'])} files, {sum(report['groups'].values())} typed members")
+
+
+if __name__ == '__main__':
+    main()