Merge pull request #211 from mstan/split/pr169-overlay-high-bank-tools

tools: support high-bank overlay captures

tools/compile_overlays.py

@@ -439,7 +439,9 @@ NON_AUTHORITY_MANIFEST_PROVENANCES = {
 HOSTED_MANIFEST_MARKER = (
     f'# psxrecomp overlay provenance {HOSTED_MANIFEST_PROVENANCE}')
 HOSTED_UNIQUE_GUARDED_BYTE_CAP = 1024 * 1024
-PSX_RAM_SIZE = 2 * 1024 * 1024
+# Full 8 MiB host capacity (PSX_RAM_CAPACITY): 8 MB-mod captures carry
+# high-bank enhancement code and their ranges must validate.
+PSX_RAM_SIZE = 8 * 1024 * 1024
 
 
 class ShardCandidateCapacityError(RuntimeError):
@@ -1306,7 +1308,28 @@ def _walk_overlay_function(data: bytes, load_addr: int, size: int,
     }
 
 
-def _collect_toml_overlay_entries(toml_doc: dict, load_addr: int, crc32: int) -> set[int]:
+def _masked_region_crc(data: bytes, load_addr: int, ignore) -> int:
+    """CRC32 of a captured region with declared-volatile byte ranges zeroed."""
+    if not ignore:
+        return binascii.crc32(data) & 0xFFFFFFFF
+    buf = bytearray(data)
+    hi = load_addr + len(data)
+    for raw in ignore:
+        if isinstance(raw, dict):
+            start, end = _parse_addr(raw['start']), _parse_addr(raw['end'])
+        else:
+            raise RuntimeError(
+                'bytes_crc_ignore entries must be tables with start/end')
+        if not (load_addr <= start < end <= hi):
+            raise RuntimeError(
+                f'bytes_crc_ignore range 0x{start:08X}..0x{end:08X} outside '
+                f'capture 0x{load_addr:08X}..0x{hi:08X}')
+        buf[start - load_addr:end - load_addr] = b'\0' * (end - start)
+    return binascii.crc32(bytes(buf)) & 0xFFFFFFFF
+
+
+def _collect_toml_overlay_entries(toml_doc: dict, load_addr: int, crc32: int,
+                                  data: bytes = b'') -> set[int]:
     entries = set()
     for ov in toml_doc.get('overlays', []) or []:
         if not isinstance(ov, dict):
@@ -1315,8 +1338,16 @@ def _collect_toml_overlay_entries(toml_doc: dict, load_addr: int, crc32: int) ->
         if ov_load is not None and _parse_addr(ov_load) != load_addr:
             continue
         ov_crc = ov.get('bytes_crc') or ov.get('crc32') or ov.get('crc')
-        if ov_crc is not None and _parse_addr(ov_crc) != crc32:
-            continue
+        if ov_crc is not None:
+            ignore = ov.get('bytes_crc_ignore')
+            live = _masked_region_crc(data, load_addr, ignore) if ignore else crc32
+            if _parse_addr(ov_crc) != live:
+                print(f'  declared entries DETACHED for 0x{load_addr:08X}: '
+                      f'bytes_crc=0x{_parse_addr(ov_crc):08X} but region hashes '
+                      f'0x{live:08X}'
+                      f'{" (masked)" if ignore else ""} -- update game.toml',
+                      file=sys.stderr)
+                continue
         for key in ('entry', 'entries', 'function_entry_pcs', 'function_entries'):
             if key not in ov:
                 continue
@@ -1412,7 +1443,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
     captured_function_entries = _parse_addr_list(cap.get('function_entry_pcs', []))
     static_discovery_entries = _parse_addr_list(
         cap.get('static_discovery_entry_pcs', []))
-    toml_entries = _collect_toml_overlay_entries(toml_doc, load_addr, crc32)
+    toml_entries = _collect_toml_overlay_entries(toml_doc, load_addr, crc32, data)
     legacy_callable_seeds = {a for a in legacy_seeds
                              if region(a) and _callable_legacy_seed(data, load_addr, a)}
 
@@ -6595,6 +6626,10 @@ def main():
         # interiors. Compile each as an isolated dispatch-root shard, then give
         # every block in those fragments the same universal resume treatment.
         unresolved = sorted(static_requested_entries - existing_entries)
+        # PSX_STATIC_NO_ISOLATED=1: A/B guard -- skip the isolated-fragment pass
+        # entirely while testing the universal-resume machinery.
+        if os.environ.get('PSX_STATIC_NO_ISOLATED'):
+            unresolved = []
         fragment_built = 0
         new_fragment_parts = []
         for entry in unresolved:

tools/coverage_vault.py

@@ -162,9 +162,12 @@ def _compact_region(region, drop_invalid_evidence=False):
     if len(image) != size:
         raise ValueError("capture size mismatch at 0x%08X: size=%d bytes=%d" %
                          (load, size, len(image)))
-    phys_lo = load & 0x1FFFFF
+    # 8 MB dev-RAM aware: titles running the 8MB enhancement stream code into
+    # the extended banks (0x200000..0x7FFFFF); the old 2 MB mask corrupted
+    # their capture addresses and rejected legitimate regions.
+    phys_lo = load & 0x7FFFFF
     phys_hi = phys_lo + size
-    if phys_hi > 2 * 1024 * 1024:
+    if phys_hi > 8 * 1024 * 1024:
         raise ValueError("capture exceeds PSX RAM at 0x%08X" % load)
 
     parsed = {}
@@ -178,7 +181,7 @@ def _compact_region(region, drop_invalid_evidence=False):
         entries = []
         for value in values:
             address = _address(value, field)
-            phys = address & 0x1FFFFF
+            phys = address & 0x7FFFFF
             if phys < phys_lo or phys >= phys_hi or (phys & 3):
                 if not drop_invalid_evidence:
                     raise ValueError(