aot: recover linked archives and static aliases

docs/AOT_OVERLAY_PLAN.md

@@ -24,6 +24,31 @@ continue to fail safely to the interpreter.
 
 ---
 
+## 2026-07-18: byte-proven archive producers
+
+- The generic extractor now recognizes strict 0x800-aligned `{id,size}` member
+  archives (MMX6 `ROCK_X6.BIN`) and companion HED sector tables spanning DAT/BNS
+  payload namespaces (Ape Escape `KKIIDDZZ.*`).
+- Link bases are not guessed from title constants. Multiple independent members
+  must anchor one sharp jal/prologue consensus; a sibling then needs at least two
+  votes for that exact trusted base.
+- Mixed archive members seed only locally callable direct-JAL targets. The first
+  MMX6 pointer-table expansion demonstrated why: seven variants hit the generated-C
+  unsupported-instruction audit. Direct-only roots compile all 30 members cleanly,
+  plus the exact-hash BIOS resident shard.
+- Ape's two PS-X EXE mirrors are deduplicated in favor of the self-describing EXE
+  producer. Its remaining 44 HED/BNS members plus two minigame EXEs and the BIOS
+  recipe are also audit-clean in a disposable cache.
+- Normal PS-X discovery now exports its exact overlapping `F/R` alias recipes.
+  Clean overlay builds therefore preserve decoder-proven indirect entries without
+  promoting every entry to a hard root or borrowing a previous cache manifest.
+  T2 compiled 53/53 disposable candidates with zero audit failures and restored
+  100% combined BIOS code-range coverage for both vault and live history.
+- Large archive reads are now linear rather than quadratic sector concatenation.
+  These validations wrote only `%TEMP%` caches; title caches remain untouched.
+
+---
+
 ## ⭐ SESSION-2 BREAKTHROUGH (2026-07-17, VERIFIED FROM DISC + CORPUS)
 
 Adversarial disc analysis overturned the pessimistic framing below. For Tomba's

recompiler/tests/test_aot_base_recovery.py

@@ -38,9 +38,48 @@ def main():
     struct.pack_into("<I", data, 4, 0x0C000000 | ((0x80105678 >> 2) & 0x03FFFFFF))
     assert MOD.jal_targets(data) == {0x80105678}
 
+    # Psy-Q may schedule a global load before allocating the stack frame.  The
+    # exact function starts immediately after the previous return, not at the
+    # otherwise recognizable addiu-sp instruction eight bytes later.
+    base = 0x80100000
+    data = bytearray(0x40)
+    struct.pack_into('<I', data, 0x00, 0x03E00008)  # jr ra
+    struct.pack_into('<I', data, 0x04, 0x00000000)  # delay slot
+    struct.pack_into('<I', data, 0x08, 0x3C03800A)  # lui v1,0x800a
+    struct.pack_into('<I', data, 0x0C, 0x9463BCC8)  # lhu v1,-0x4338(v1)
+    struct.pack_into('<I', data, 0x10, 0x27BDFFE8)  # addiu sp,sp,-0x18
+    assert MOD.prologues(data, base) == [base + 0x08]
+    # A load through a different register is not sufficient evidence to move
+    # the entry; fail closed at the ordinary stack-frame prologue.
+    struct.pack_into('<I', data, 0x0C, 0x9443BCC8)  # lhu v1,-0x4338(v0)
+    assert MOD.prologues(data, base) == [base + 0x10]
+
+    # Normal-mode discovery must not turn a PS-X body's provenance-free leading
+    # pointer table into a function. Keep the declared entry and the additive
+    # decoder-derived interior set; those entries have already passed normal
+    # mode's return/prologue/call/control-flow discovery and classification.
+    data = bytearray(0x100)
+    for n in range(4):
+        struct.pack_into('<I', data, n * 4, base + 0x40 + n * 4)
+    struct.pack_into('<I', data, 0x40, 0x27BDFFF0)
+    struct.pack_into('<I', data, 0x44, 0x0C000000 |
+                     (((base + 0x80) >> 2) & 0x03FFFFFF))
+    struct.pack_into('<I', data, 0x80, 0x24020001)
+    assert MOD.filter_full_discovery_seeds(
+        data, base, [base, base + 0x40, base + 0x80], base + 0x40) == [
+            base + 0x40, base + 0x80]
+    assert MOD.filter_full_discovery_seeds(
+        data, base, [base, base + 0x40, base + 0x80], base) == [
+            base, base + 0x40, base + 0x80]
+
+    entries, aliases = MOD.parse_full_discovery_ranges([
+        'F 80100040\n', 'R 80100040 20\n',
+        'F 80100048\n', 'R 80100040 20\n'])
+    assert entries == [base + 0x40, base + 0x48]
+    assert aliases == [(base + 0x48, base + 0x40, base + 0x60)]
+
     # Dense function-pointer tables may recover framed or frameless functions,
     # but only when every target has independent callable-boundary evidence.
-    base = 0x80100000
     data = bytearray(0x100)
     # Three adjacent leaves, each immediately after a previous return.
     for off in (0x18, 0x28, 0x38):
@@ -56,6 +95,60 @@ def main():
     struct.pack_into('<I', data, 0x84, 0)
     assert MOD.pointer_table_targets(data, base) == set()
 
+    # A strict aligned {id,size} archive can share an independently voted link
+    # base across members.  Direct JAL targets are the roots; unrelated padding
+    # and prologue-shaped words do not become code merely by being in a member.
+    archive_base = 0x800E9060
+    members = []
+    target_offsets = (0x80, 0x94, 0xB0, 0xD8,
+                      0x110, 0x154, 0x1A8, 0x210)
+    for ident in range(1, 5):
+        body = bytearray(0x240)
+        for n, target_off in enumerate(target_offsets):
+            target = archive_base + target_off
+            struct.pack_into('<I', body, n * 4, 0x0C000000 |
+                             ((target >> 2) & 0x03FFFFFF))
+            struct.pack_into('<I', body, target_off, 0x27BDFFF0)
+        members.append((ident, bytes(body)))
+    archive = bytearray(0x800)
+    pos = 0x800
+    payload = bytearray()
+    for n, (ident, body) in enumerate(members):
+        struct.pack_into('<II', archive, n * 8, ident, len(body))
+        payload.extend(body)
+        payload.extend(b'\0' * ((-len(payload)) & 0x7FF))
+    archive.extend(payload)
+    split = MOD.split_indexed_archive(bytes(archive))
+    assert split is not None and [x[0] for x in split] == [1, 2, 3, 4]
+    recovered = MOD.recover_indexed_archive(
+        bytes(archive), 0x80010000, 0x80200000)
+    assert len(recovered) == 4
+    assert {x['base'] for x in recovered} == {archive_base}
+    assert all(len(x['direct_seeds']) == 8 for x in recovered)
+
+    # Companion HED tables may address a logical sector namespace spanning DAT
+    # then BNS. Runs are discovered independently across zero-filled table gaps.
+    hed = bytearray(0x800)
+    for n in range(4):
+        struct.pack_into('<I', hed, n * 4, (1 << 20) | n)
+        struct.pack_into('<I', hed, 0x100 + n * 4, (1 << 20) | (4 + n))
+    payload_a = b''.join(bytes([n + 1]) * 0x800 for n in range(4))
+    payload_b = b''.join(bytes([n + 5]) * 0x800 for n in range(4))
+
+    class FakeDisc:
+        blobs = {1: bytes(hed), 2: payload_a, 3: payload_b}
+
+        def read_file_bytes(self, lba, size):
+            return self.blobs[lba][:size]
+
+    groups = MOD.hed_companion_members(FakeDisc(), [
+        ('ARC.HED', 1, len(hed)), ('ARC.DAT', 2, len(payload_a)),
+        ('ARC.BNS', 3, len(payload_b)),
+    ])
+    assert len(groups) == 1 and len(groups[0]['members']) == 8
+    assert [x[1] for x in groups[0]['members']] == [n * 0x800
+                                                    for n in range(8)]
+
     # Exact-hash BIOS resident recipes synthesize only their declared code and
     # data words. Entries must point into code fragments; a different BIOS is a
     # safe empty result rather than a speculative shard.

recompiler/tests/test_aot_overlay_discovery.py

@@ -90,6 +90,30 @@ def check_static_discovery_provenance():
     assert audit['included_reasons'][entry] == 'STATIC_DISCOVERY_ROOT'
 
 
+def check_static_alias_recipe():
+    data = bytearray(0x40)
+    put(data, 0x00, 0x03E00008)
+    put(data, 0x04, 0x00000000)
+    put(data, 0x20, 0x24020001)
+    put(data, 0x24, 0x03E00008)
+    put(data, 0x28, 0x00000000)
+    cap = {
+        'schema': 'psxrecomp overlay capture v2',
+        'function_entry_pcs': [f'0x{LOAD:08X}', f'0x{LOAD + 0x20:08X}'],
+        'static_alias_ranges': [{
+            'entry': f'0x{LOAD + 0x20:08X}',
+            'start': f'0x{LOAD:08X}',
+            'end': f'0x{LOAD + 0x2C:08X}',
+        }],
+    }
+    seeds, audit = MOD.classify_overlay_seeds(
+        cap, bytes(data), LOAD, len(data), 0, {})
+    assert (f'retained_alias 0x{LOAD + 0x20:08X} '
+            f'0x{LOAD:08X} 0x{LOAD + 0x2C:08X}') in seeds
+    assert audit['static_alias_ranges'] == {
+        (LOAD + 0x20, LOAD, LOAD + 0x2C)}
+
+
 def check_forward_branch_root():
     data = bytearray(0x80)
     target = LOAD + 0x40
@@ -220,6 +244,7 @@ def main():
 
     check_composite_call_boundaries()
     check_static_discovery_provenance()
+    check_static_alias_recipe()
     check_forward_branch_root()
     check_recompiler_composite_contract(args.recompiler)
     check_retained_alias_contract(args.recompiler)

tools/aot_overlay_spike/README.md

@@ -128,14 +128,16 @@ prototype exposed padding as a seventh function and was not retained.
   recompiler in NORMAL mode (jr-$ra boundary scan) finds frameless functions
   overlay-mode's prologue-scan misses. Tomba2 MAIN.EXE: prologue 710 fns / 62%
   vault -> full-discovery 786 fns / 69% vault (3296 shards, 0 audit fails);
-  the tool now emits 3500 discovered seeds vs 1210 prologues.
-- **Generality is ENGINE-SPECIFIC, not universal.** The two producer types
+  the current tool emits 3503 discovered entries vs 1210 prologues. It also
+  carries normal mode's exact overlapping-range recipes into clean overlay
+  builds, so alias coverage does not depend on an older cache manifest.
+- **Generality is ENGINE-SPECIFIC, not universal.** The initial two producer types
   (self-describing PS-X-EXE + {count,ptr[]} header-table) cover the Whoopee Camp
   engine (Tomba 1 header-table; Tomba 2 MAIN.EXE + BIN) but detect ZERO overlays
-  on Vigilante 8 (Luxoflux) and Mega Man X6 (Capcom) — those load overlays in a
-  different container/format (likely compressed/archived). Each engine family
-  needs its own producer. Matches the Legaia-decomp finding: overlay enumeration
-  is bespoke per engine.
+  on Vigilante 8 (Luxoflux) and initially detected zero on Mega Man X6 (Capcom).
+  The indexed/HED archive producers below now cover MMX6 and Ape Escape; other
+  engine families still need their own byte-proven container parser. Matches the
+  Legaia-decomp finding: overlay enumeration is bespoke per engine.
 - **SOLVED (2026-07-17) — header-table base recovery via jal-target self-consistency.**
   The old delta-sweep matched the export-table pointers against prologues, but those
   pointers point at mid-function DISPATCH entries, not prologues (measured: 0/29 hit
@@ -195,6 +197,48 @@ verified append-only live-history needed set to **100% native code-range
 coverage**. This avoids generating duplicate overlay shards for code that is
 already native.
 
+## Indexed and companion archive recovery (2026-07-18)
+
+Large-container extraction first exposed an unrelated quadratic I/O bug:
+`DiscReader.read_file_bytes()` appended immutable bytes once per 2 KiB sector.
+It now joins sector chunks once; reading MMX6's 50.9 MB DAT fell from minutes to
+about 0.25 seconds on the validation machine.
+
+`ROCK_X6.BIN` is a strict 0x800-aligned `{id,size}[]` archive, not compressed
+code. Its members independently vote three link bases (`0x800E9060`,
+`0x800F9800`, `0x801E9800`). A base is trusted only after at least two members
+produce a sharp >=8-vote, >=2x jal-to-prologue peak; siblings need at least two
+votes for that exact consensus. Mixed archive members use **direct-JAL roots
+only**. A broader pointer-table experiment compiled 24/31 but correctly failed
+seven generated-C audits; removing those speculative roots produced
+`PSX_SHARD_RESULT ok=31 failed=0 skipped=0` (30 members + BIOS resident). Against
+the union of 16 legacy MMX6 capture files this first conservative pass covers
+71.8% of entries by combined native code range. The remaining gap is discovery
+inside byte-proven members, not container enumeration.
+
+Ape Escape's `KKIIDDZZ.HED` encodes contiguous
+`size_sectors:12 | logical_sector:20` runs spanning sibling DAT then BNS files.
+The same cross-member vote gate proves `0x80136000` (10 anchors), `0x8013D000`
+(27), and two mirrored minigame EXEs. Ordinary PS-X EXE handling owns the latter;
+the HED producer emits 44 BNS members. Together with two self-describing minigame
+EXEs and the resident BIOS recipe, the disposable validation cache has all 47
+candidates compiled with zero unsupported/bad targets.
+
+Normal-mode PS-X discovery quarantines its one provenance-free fallback: the
+image body start is excluded when the PS-X header declares a different entry.
+This removes Ape MINI2's body-leading address/string table at `0x80100000` while
+retaining its declared `0x80100C84` entry. All decoder-classified interior
+entries remain ordinary candidates, and the extractor serializes their `F/R`
+overlapping alias recipes instead of promoting them to hard call roots. A clean
+Tomba 2 disposable rebuild compiled 53/53 candidates with zero unsupported/bad
+targets and, together with the base BIOS, reproduced **100% code-range coverage**
+against both the full-playthrough vault and 765-entry append-only live history.
+No prior MAIN manifest was present for the two regenerated shards. Position-fixed
+framed scans likewise recover Psy-Q's exact
+`lui R; load ...,off(R); addiu sp,sp,-N` entry after a previous return, replacing
+the old root eight bytes late. This closes Tomba 1's historical `0x80110E30` gap
+in the clean X00 variant without minting overlapping functions.
+
 ## Next to raise coverage (future session)
 
 1. ~~Direct-call frameless-leaf discovery~~ — DONE. Reachable `jal` targets are

tools/aot_overlay_spike/extract_generic.py

@@ -2,7 +2,7 @@
 """Generic play-free overlay extractor (AOT spike, v2 — full-discovery producers).
 
 Given a game.toml (disc + load_address + text_size), enumerate the disc and emit
-a synthetic overlay_captures.json for compile_overlays.py. Two producer types:
+a synthetic overlay_captures.json for compile_overlays.py. Producer types:
 
  1. PS-X EXE files (magic "PS-X EXE") — the load address is self-described in the
     header. Run the recompiler in NORMAL mode (full discovery: jr-$ra boundary
@@ -15,6 +15,12 @@ a synthetic overlay_captures.json for compile_overlays.py. Two producer types:
     prologue-scan seeds at a delta-swept base. LESS robust (base recovery
     unreliable when header ptrs aren't clean function starts); see plan doc.
 
+ 3. Strict 0x800-aligned {id,size} member archives — multiple members must
+    independently vote the same link base; mixed members use direct-call roots.
+
+ 4. Companion HED/DAT/BNS archives — contiguous packed-sector runs address a
+    logical namespace across payload siblings, with the same consensus gate.
+
 Usage:
   extract_generic.py --game-toml <path> --recompiler <psxrecomp-game.exe>
                      --out <captures.json> [--tmp <dir>]
@@ -44,9 +50,33 @@ def parse_cue_datatrack(cue_path):
     return eo.parse_cue(cue_path)   # fallback
 
 def prologues(data, base):
-    return [base+off for off in range(0,len(data)-4,4)
-            if (struct.unpack_from('<I',data,off)[0]>>16)==0x27BD
-            and (struct.unpack_from('<I',data,off)[0]&0x8000)]
+    """Return framed function entries, including a proven pre-frame prelude.
+
+    Psy-Q commonly schedules ``lui R`` / ``load ...,off(R)`` before the usual
+    ``addiu sp,sp,-N``.  A raw prologue scan consequently starts those functions
+    eight bytes late and leaves the real dispatch entry uncovered.  Recover the
+    earlier entry only for that exact register-linked pair immediately following
+    a previous ``jr ra`` + delay-slot boundary.  Replace (rather than supplement)
+    the late prologue root so exact-entry compilation cannot split one function
+    into overlapping siblings.
+
+    This deliberately does not scan arbitrary instructions before a prologue:
+    the return boundary, LUI/load opcode pair, and matching address register are
+    all required data-as-code gates.
+    """
+    entries=[]
+    for off in prologue_offsets(data):
+        entry=off
+        if off >= 16 and _word(data, off-16) == 0x03E00008:
+            lui=_word(data, off-8)
+            load=_word(data, off-4)
+            lui_rt=(lui >> 16) & 0x1F
+            if (lui >> 26) == 0x0F and lui_rt != 0 and \
+                    0x20 <= (load >> 26) <= 0x26 and \
+                    ((load >> 21) & 0x1F) == lui_rt:
+                entry=off-8
+        entries.append(base+entry)
+    return entries
 
 def prologue_offsets(data):
     """File offsets whose word is an `addiu $sp,$sp,-N` stack-frame prologue."""
@@ -213,6 +243,192 @@ def recover_raw_base(data, base_lo, base_hi=0x80200000):
         return None
     return base,score,second
 
+def raw_base_votes(data, base_lo, base_hi=0x80200000):
+    """Return jal/prologue base votes for a position-fixed code blob."""
+    import bisect
+    from collections import Counter
+    P=prologue_offsets(data)
+    T=jal_targets(data)
+    hist=Counter()
+    if not P or not T:
+        return hist
+    max_base=base_hi-len(data)
+    for t in T:
+        lo_i=bisect.bisect_left(P,t-max_base)
+        hi_i=bisect.bisect_right(P,t-base_lo)
+        for o in P[lo_i:hi_i]:
+            base=t-o
+            if (base&3)==0:
+                hist[base]+=1
+    return hist
+
+def split_indexed_archive(data, alignment=0x800):
+    """Recognize a strict ``{id,size}[]`` + aligned-payload archive.
+
+    Mega Man X6's ROCK_X6.BIN uses one monotonically increasing table in the
+    first sector.  Payload members follow in table order, each rounded up to a
+    0x800-byte boundary.  Require the complete layout to account for the file
+    (apart from at most one format trailer sector) so ordinary data cannot
+    be mistaken for this container merely because its first words look small.
+    """
+    if len(data) < alignment*2 or len(data) % alignment:
+        return None
+    entries=[]
+    for off in range(0, alignment, 8):
+        ident,size=struct.unpack_from('<II',data,off)
+        if ident == 0 and size == 0:
+            break
+        if ident == 0 or size < 4 or size > 2_000_000:
+            return None
+        if entries and ident <= entries[-1][0]:
+            return None
+        entries.append((ident,size))
+    if len(entries) < 4:
+        return None
+    pos=alignment
+    members=[]
+    for ident,size in entries:
+        if pos+size > len(data):
+            return None
+        members.append((ident,pos,data[pos:pos+size]))
+        pos=(pos+size+alignment-1)&~(alignment-1)
+    if pos > len(data) or len(data)-pos > alignment:
+        return None
+    return members
+
+def recover_consensus_members(members, base_lo, base_hi=0x80200000):
+    """Recover linked code from members that independently vote shared bases.
+
+    An anchor member needs >=8 agreeing jal/prologue pairs and a 2x vote peak.
+    A base becomes trusted only when at least two independent members anchor it.
+    Other members may reuse that exact consensus with >=2 agreeing pairs.  Only
+    locally callable direct-JAL targets become roots; pointer tables and apparent
+    prologues are deliberately insufficient inside a mixed code/data archive.
+    """
+    from collections import Counter
+    if not members:
+        return []
+    votes=[]
+    anchor_support=Counter()
+    for ident,off,body in members:
+        hist=raw_base_votes(body,base_lo,base_hi)
+        votes.append((ident,off,body,hist))
+        top=hist.most_common(2)
+        if top:
+            base,score=top[0]
+            second=top[1][1] if len(top)>1 else 0
+            if score >= 8 and score >= second*2:
+                anchor_support[base]+=1
+    trusted={base for base,count in anchor_support.items() if count >= 2}
+    if not trusted:
+        return []
+
+    recovered=[]
+    for ident,off,body,hist in votes:
+        ranked=sorted(((hist.get(base,0),base) for base in trusted),reverse=True)
+        score,base=ranked[0]
+        runner=ranked[1][0] if len(ranked)>1 else 0
+        if score < 2 or score < runner*2:
+            continue
+        lo,hi=base,base+len(body)
+        direct={t for t in jal_targets(body)
+                if lo <= t < hi and (t&3)==0 and
+                (_callable_boundary(body,t-base) or
+                 ((_word(body,t-base) or 0)>>16)==0x27BD)}
+        seeds=sorted(direct)
+        if len(direct) < 2:
+            continue
+        recovered.append({
+            'id':ident, 'file_offset':off, 'data':body, 'base':base,
+            'score':score, 'runner':runner, 'direct_seeds':sorted(direct),
+            'seeds':seeds,
+        })
+    return recovered
+
+def recover_indexed_archive(data, base_lo, base_hi=0x80200000):
+    """Recover independently linked code members from a strict archive."""
+    members=split_indexed_archive(data)
+    return recover_consensus_members(members or (),base_lo,base_hi)
+
+def hed_companion_members(disc, files):
+    """Return strict sector-table members from sibling HED/DAT/BNS files.
+
+    Ape Escape stores 32-bit descriptors as ``size_sectors:12 | lba:20``.
+    Descriptor runs are contiguous, while the payload namespace concatenates
+    same-stem DAT then BNS companions.  Require runs of at least four entries,
+    exact sector continuity, sector-aligned companions, and no cross-file member.
+    """
+    by_upper={p.upper():(p,l,s) for p,l,s in files}
+    groups=[]
+    for hed_path,hed_lba,hed_size in files:
+        if not hed_path.upper().endswith('.HED') or hed_size > 0x10000:
+            continue
+        stem=hed_path[:-4]
+        companions=[]
+        for ext in ('.DAT','.BNS'):
+            item=by_upper.get((stem+ext).upper())
+            if item:
+                companions.append(item)
+        if not companions or any(size%0x800 for _,_,size in companions):
+            continue
+        hed=disc.read_file_bytes(hed_lba,hed_size)
+        words=[struct.unpack_from('<I',hed,off)[0]
+               for off in range(0,len(hed)-3,4)]
+        runs=[]; i=0
+        while i < len(words):
+            word=words[i]; sector=word&0xFFFFF; count=word>>20
+            if not count:
+                i+=1; continue
+            run=[]; expected=sector; j=i
+            while j < len(words):
+                value=words[j]; at=value&0xFFFFF; span=value>>20
+                if not span or at != expected:
+                    break
+                run.append((j,at,span)); expected+=span; j+=1
+            if len(run) >= 4:
+                runs.extend(run); i=j
+            else:
+                i+=1
+        if not runs:
+            continue
+
+        logical=[]; cursor=0
+        for path,lba,size in companions:
+            body=disc.read_file_bytes(lba,size)
+            logical.append((cursor,cursor+size//0x800,path,body))
+            cursor+=size//0x800
+        members=[]; seen=set()
+        for table_index,sector,count in runs:
+            key=(sector,count)
+            if key in seen:
+                continue
+            seen.add(key)
+            owner=next((x for x in logical
+                        if x[0] <= sector and sector+count <= x[1]),None)
+            if owner is None:
+                continue
+            lo,hi,path,body=owner
+            off=(sector-lo)*0x800
+            member=body[off:off+count*0x800]
+            members.append((table_index,sector*0x800,member))
+        if members:
+            groups.append({
+                'hed_path':hed_path,
+                'members':members,
+                'consumed':{hed_path.upper(),
+                            *(path.upper() for path,_,_ in companions)},
+            })
+    return groups
+
+def page_aligned_region(base, data):
+    """Build a runtime-keyed region and discard only proven all-zero lead pages."""
+    page_base=base&~0xFFF
+    region=b'\x00'*(base-page_base)+data
+    while len(region)>0x1000 and not any(region[:0x1000]):
+        page_base+=0x1000
+        region=region[0x1000:]
+    return page_base,region
+
 def is_psx_exe(data):
     return len(data) >= 0x20 and data[:8] == b'PS-X EXE'
 
@@ -225,28 +441,62 @@ def is_header_table(data):
     nul = sum(1 for p in ptrs if p == 0)
     return cnt if (inr+nul == cnt and inr >= 1) else None
 
+def parse_full_discovery_ranges(lines):
+    """Return entries and overlapping alias recipes from normal-mode ranges."""
+    seeds=set(); aliases=[]; pending=None
+    for line in lines:
+        m=re.match(r'F ([0-9A-Fa-f]+)',line)
+        if m:
+            pending=int(m.group(1),16)
+            seeds.add(pending if pending>=0x80000000 else (0x80000000|pending))
+            continue
+        m=re.match(r'R ([0-9A-Fa-f]+) ([0-9A-Fa-f]+)',line)
+        if m and pending is not None:
+            lo=int(m.group(1),16); size=int(m.group(2),16)
+            entry=pending if pending>=0x80000000 else (0x80000000|pending)
+            lo=lo if lo>=0x80000000 else (0x80000000|lo)
+            if size and entry != lo and lo <= entry < lo+size:
+                aliases.append((entry,lo,lo+size))
+            pending=None
+    return sorted(seeds),sorted(set(aliases))
+
 def full_discovery_seeds(data, recompiler, tmp):
-    """Run recompiler NORMAL mode on a PS-X EXE; return discovered function entries."""
+    """Run NORMAL mode; return discovered entries plus exact alias recipes."""
     exe_path = os.path.join(tmp, 'producer.exe')
     open(exe_path,'wb').write(data)
-    out = os.path.join(tmp, 'disc_out'); os.makedirs(out, exist_ok=True)
+    out = os.path.join(tmp, f'disc_out_{binascii.crc32(data)&0xFFFFFFFF:08X}')
+    os.makedirs(out, exist_ok=True)
     try:
         # errors='replace': the recompiler prints non-ASCII (✓) that would raise
         # a decode error under text=True and lose the (already-written) ranges.
         subprocess.run([recompiler, exe_path, '--out-dir', out],
                        capture_output=True, text=True, errors='replace', timeout=300)
     except Exception as e:
-        print(f"    full-discovery failed ({e}); falling back to prologue scan"); return None
-    seeds=set()
+        print(f"    full-discovery failed ({e}); falling back to prologue scan")
+        return None,[]
+    lines=[]
     for rf in [f for f in os.listdir(out) if f.endswith('.ranges')]:
-        for line in open(os.path.join(out,rf), errors='ignore'):
-            m=re.match(r'F ([0-9A-Fa-f]+)', line)   # normal-mode: "F <entry>" (no crc suffix)
-            if m:
-                a=int(m.group(1),16); seeds.add(a if a>=0x80000000 else (0x80000000|a))
-    return sorted(seeds) or None
+        lines.extend(open(os.path.join(out,rf), errors='ignore'))
+    seeds,aliases=parse_full_discovery_ranges(lines)
+    return (seeds,aliases) if seeds else (None,[])
+
+def filter_full_discovery_seeds(body, base, candidates, declared_entry):
+    """Quarantine normal mode's unproven image-body fallback entry.
+
+    Normal mode derives interior entries from return/prologue/call/control-flow
+    evidence and classifies their reachable extents.  Its one provenance-free
+    fallback is the image load address: a backward return scan that reaches the
+    beginning promotes that first body word.  Ape MINI2 begins with a pointer
+    table there, while its PS-X header declares the real entry later.  Quarantine
+    only that unproven body-start fallback; preserving the additive interior set
+    is essential because removing selected roots can split otherwise broad
+    functions and create native code-range holes.
+    """
+    return sorted({addr for addr in candidates
+                   if addr != base or addr == declared_entry})
 
 def rec(load_addr, data, seeds, dispatch_extra=None, producer_ranges=None,
-        static_discovery=None):
+        static_discovery=None, static_alias_ranges=None):
     # function_entry_pcs = prologue-scanned function starts (walk roots).
     # dispatch_entry_pcs = those PLUS any extra dispatch targets (e.g. the
     # overlay's own export table). compile_overlays promotes clean starts to
@@ -266,6 +516,11 @@ def rec(load_addr, data, seeds, dispatch_extra=None, producer_ranges=None,
     if static_discovery:
         out["static_discovery_entry_pcs"]=[
             f"0x{addr:08X}" for addr in sorted(set(static_discovery))]
+    if static_alias_ranges:
+        out["static_alias_ranges"]=[
+            {"entry":f"0x{entry:08X}", "start":f"0x{lo:08X}",
+             "end":f"0x{hi:08X}"}
+            for entry,lo,hi in sorted(set(static_alias_ranges))]
     return out
 
 def bios_resident_records(bios_path=None, manifest_path=None):
@@ -393,13 +648,18 @@ def main():
     bp,raw=parse_cue_datatrack(disc)   # multi-bin safe: pick track-1 data bin
     dr=eo.DiscReader(bp,raw=raw)
     files=list(eo.enumerate_files(dr))
+    hed_groups=hed_companion_members(dr,files)
+    hed_consumed=set().union(*(group['consumed'] for group in hed_groups)) \
+        if hed_groups else set()
     tmp=a.tmp or tempfile.mkdtemp()
     os.makedirs(tmp, exist_ok=True)
     # The boot EXE (game.toml `exe`) is the STATIC-recompiled base, NOT an overlay.
     exe_base=os.path.basename(str(game.get('exe',''))).upper()
-    records=[]; np=nh=nr=nc=0; seen_crc=set(); header_files=[]; raw_files=[]
+    records=[]; np=nh=nr=na=ne=nc=0; seen_crc=set(); header_files=[]; raw_files=[]
     positioned=[]
     for p,l,s in sorted(files):
+        if p.upper() in hed_consumed:
+            continue
         if s<8 or s>2_000_000: continue
         if exe_base and os.path.basename(p).upper()==exe_base:
             continue   # skip the statically-compiled base executable
@@ -410,11 +670,15 @@ def main():
         if dcrc in seen_crc: continue
         seen_crc.add(dcrc)
         if is_psx_exe(data):
+            entry_pc=struct.unpack_from('<I',data,0x10)[0]
             t_addr=struct.unpack_from('<I',data,0x18)[0]
             body=data[2048:]
-            seeds_all=full_discovery_seeds(data, a.recompiler, tmp)
-            used_full_discovery=seeds_all is not None
-            if seeds_all is None: seeds_all=prologues(body, t_addr)
+            seeds_all,aliases_all=full_discovery_seeds(data, a.recompiler, tmp)
+            if seeds_all is None:
+                seeds_all=prologues(body, t_addr)
+            else:
+                seeds_all=filter_full_discovery_seeds(
+                    body,t_addr,seeds_all,entry_pc)
             # split body into floor-clamped window regions
             base=t_addr & 0x1FFFFFFF
             end=base+len(body)
@@ -425,9 +689,18 @@ def main():
                 seg=body[lo-base:hi-base]
                 va=0x80000000|lo
                 sd=[x for x in seeds_all if lo<=(x&0x1FFFFFFF)<hi]
+                # Normal-mode entries are already decoder/classifier output.
+                # Leave them as ordinary captured candidates so overlay mode
+                # can preserve overlapping alias groups. Marking them as hard
+                # STATIC_DISCOVERY_ROOT call roots truncates broad owners and
+                # makes a clean build lose native code-range coverage.
+                span_hi=va+len(seg)
+                aliases=[alias for alias in aliases_all
+                         if va <= alias[0] < span_hi and
+                         va <= alias[1] < alias[2] <= span_hi and
+                         (alias[0] != t_addr or alias[0] == entry_pc)]
                 records.append(rec(
-                    va,seg,sd,
-                    static_discovery=sd if used_full_discovery else None))
+                    va,seg,sd,static_alias_ranges=aliases))
             np+=1
             print(f"  [PS-X EXE] {p}: {len(body)}B @0x{0x80000000|base:08X}, full-disc seeds={len(seeds_all)}")
         else:
@@ -498,22 +771,68 @@ def main():
     raw_base_lo=0x80000000|floor_page
     for p,data in raw_files:
         rb=recover_raw_base(data, raw_base_lo)
-        if rb is None: continue
-        base,score,second=rb
-        prologue_seeds=prologues(data,base)
-        supplemental=supplemental_callable_seeds(data,base)
-        seeds=sorted(set(prologue_seeds)|supplemental)
-        page_base=base&~0xFFF
-        fill=base-page_base
-        region=b'\x00'*fill+data
-        records.append(rec(page_base,region,seeds))
-        positioned.append((p,data,base,seeds,()))
-        nr+=1
-        print(f"  [raw-code] {p}: {len(data)}B file@0x{base:08X} "
-              f"region@0x{page_base:08X}(+{fill}) "
-              f"(strict jal-fit score={score}, runner-up={second}), "
-              f"prologue seeds={len(prologue_seeds)} "
-              f"+{len(supplemental-set(prologue_seeds))} pointer-table seeds")
+        if rb is not None:
+            base,score,second=rb
+            prologue_seeds=prologues(data,base)
+            supplemental=supplemental_callable_seeds(data,base)
+            seeds=sorted(set(prologue_seeds)|supplemental)
+            page_base,region=page_aligned_region(base,data)
+            records.append(rec(page_base,region,seeds))
+            positioned.append((p,data,base,seeds,()))
+            nr+=1
+            print(f"  [raw-code] {p}: {len(data)}B file@0x{base:08X} "
+                  f"region@0x{page_base:08X} "
+                  f"(strict jal-fit score={score}, runner-up={second}), "
+                  f"prologue seeds={len(prologue_seeds)} "
+                  f"+{len(supplemental-set(prologue_seeds))} pointer-table seeds")
+            continue
+
+        for member in recover_indexed_archive(data,raw_base_lo):
+            base=member['base']; body=member['data']
+            page_base,region=page_aligned_region(base,body)
+            producer_lo=max(base,page_base)
+            record=rec(
+                page_base,region,member['seeds'],
+                producer_ranges=((producer_lo,base+len(body)),),
+                static_discovery=member['direct_seeds'])
+            record['producer_name']=(
+                f"{p}:member_{member['id']:04X}@{member['file_offset']:08X}")
+            records.append(record)
+            na+=1
+            print(f"  [indexed-archive] {p} member 0x{member['id']:X}: "
+                  f"{len(body)}B file+0x{member['file_offset']:X} "
+                  f"@0x{base:08X}, region@0x{page_base:08X}, "
+                  f"trusted-fit={member['score']}:{member['runner']}, "
+                  f"direct seeds={len(member['direct_seeds'])}, "
+                  f"total seeds={len(member['seeds'])}")
+
+    for group in hed_groups:
+        for member in recover_consensus_members(
+                group['members'],raw_base_lo):
+            base=member['base']; body=member['data']
+            # Some HED archives mirror ordinary PS-X EXE files verbatim. The
+            # self-describing producer above already emits their body at the
+            # header-declared address; do not also emit the staging buffer that
+            # includes the 0x800-byte executable header.
+            if is_psx_exe(body):
+                continue
+            page_base,region=page_aligned_region(base,body)
+            producer_lo=max(base,page_base)
+            record=rec(
+                page_base,region,member['seeds'],
+                producer_ranges=((producer_lo,base+len(body)),),
+                static_discovery=member['direct_seeds'])
+            record['producer_name']=(
+                f"{group['hed_path']}:entry_{member['id']:04X}"
+                f"@logical_{member['file_offset']:08X}")
+            records.append(record)
+            ne+=1
+            print(f"  [HED archive] {group['hed_path']} entry "
+                  f"0x{member['id']:X}: {len(body)}B "
+                  f"logical+0x{member['file_offset']:X} @0x{base:08X}, "
+                  f"region@0x{page_base:08X}, "
+                  f"trusted-fit={member['score']}:{member['runner']}, "
+                  f"direct seeds={len(member['direct_seeds'])}")
 
     # Runtime dirty-page capture coalesces directly adjacent producers into one
     # region keyed by the earlier page. Emit each provable two-file composition
@@ -555,7 +874,9 @@ def main():
     with open(a.out, 'w') as f:
         json.dump(records, f)
     print(f"producers: {np} PS-X EXE (full-discovery), {nh} header-table, "
-          f"{nr} raw-code, {nc} adjacent composites, {nb} BIOS resident; "
+          f"{nr} raw-code, {na} indexed-archive members, "
+          f"{ne} HED-companion members, "
+          f"{nc} adjacent composites, {nb} BIOS resident; "
           f"{len(records)} regions -> {a.out}")
 
 if __name__=='__main__': main()

tools/compile_overlays.py

@@ -854,6 +854,26 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
            for i in range(1, len(producer_ranges))):
         raise RuntimeError('producer_ranges overlap')
 
+    static_alias_ranges = set()
+    for raw_alias in cap.get('static_alias_ranges', []) or []:
+        if not isinstance(raw_alias, dict):
+            raise RuntimeError('static_alias_ranges entries must be objects')
+        try:
+            entry = _parse_addr(raw_alias['entry'])
+            range_lo = _parse_addr(raw_alias['start'])
+            range_hi = _parse_addr(raw_alias['end'])
+        except (KeyError, TypeError, ValueError) as exc:
+            raise RuntimeError(
+                f'invalid static_alias_ranges entry: {raw_alias}') from exc
+        if not (lo <= range_lo <= entry < range_hi <= hi and
+                (entry & 3) == 0 and (range_lo & 3) == 0 and
+                (range_hi & 3) == 0):
+            raise RuntimeError(
+                f'static alias 0x{entry:08X} -> '
+                f'0x{range_lo:08X}..0x{range_hi:08X} outside/misaligned for '
+                f'capture 0x{lo:08X}..0x{hi:08X}')
+        static_alias_ranges.add((entry, range_lo, range_hi))
+
     schema_keys = {'executed_pcs', 'observed_pcs', 'dispatch_entry_pcs',
                    'function_entry_pcs'}
     has_split_schema = any(k in cap for k in schema_keys)
@@ -1108,6 +1128,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
         'counts': counts,
         'excluded_counts': excluded_counts,
         'producer_ranges': producer_ranges,
+        'static_alias_ranges': static_alias_ranges,
         'rejected_cross_producer_calls': rejected_cross_producer_calls,
         'accepted_cross_producer_calls': accepted_cross_producer_calls,
     }
@@ -1131,7 +1152,10 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
         seeds.append(f'producer_range 0x{range_lo:08X} 0x{range_hi:08X}')
     for addr in sorted(accepted_cross_producer_calls):
         seeds.append(f'cross_call_allow 0x{addr:08X}')
-    for addr, range_lo, range_hi in cap.get('_prior_aliases', []):
+    retained_aliases = static_alias_ranges | {
+        (int(addr), int(range_lo), int(range_hi))
+        for addr, range_lo, range_hi in cap.get('_prior_aliases', [])}
+    for addr, range_lo, range_hi in sorted(retained_aliases):
         seeds.append(
             f'retained_alias 0x{addr:08X} 0x{range_lo:08X} 0x{range_hi:08X}')
     seeds.extend(seed_line(addr) for addr in sorted(included))