diff --git a/docs/AOT_SHARDING.md b/docs/AOT_SHARDING.md index a646b479..6d4c67cf 100644 --- a/docs/AOT_SHARDING.md +++ b/docs/AOT_SHARDING.md @@ -60,6 +60,7 @@ Profiles declare these reusable methods: | `disc_set` | Instead of `disc_hashes`: `[{name, disc_hashes}]`, every disc of a release whose executable images are identical. Any one matches; the images are pinned by their own content hashes, so which disc was read does not change the output. | | `file_aliases` | `{logical: [per-disc names]}` for one file that discs name differently (`SCUS_944.91`, `SCUS_945.84`, ...). Exactly one candidate must exist; the logical name reads as that file everywhere in the profile. | | `prepared_files[]` | Virtual files derived from disc bytes by a declared transform (see below). Each is verified, then usable as a `file` by every method and check. | +| `cross_image_call_roots` | Opt in (`true`): a `jal` target that other images call becomes a static discovery root of the image holding it, when the evidence at the target is the same as for local calls and exactly one image occupies that address. Finds routines only an overlay (or boot code) calls. | | `generic_exclude_files[]` | `{files, reason}`: disc files the generic detector must not consider because the profile owns them (aliased program files are excluded automatically). | | `images[].method = fixed_address_files` | Whole original files placed at a verified `load_addr`. Optional duplicate-leaf checks require identical bytes. | | `images[].method = fixed_address_extents` | Declared `{file, file_offset, base, address, size, sha256}` byte extents placed at a verified `load_addr`. Offset rules match `words` checks; size and SHA-256 are required. | @@ -131,6 +132,10 @@ Current consumers: - Ape Escape USA: `ApeEscapeRecomp/aot/overlays.json`, 47 established packed sector-table overlays plus two minigame PS-X EXEs. Two additional archive members have unresolved load paths and are explicitly excluded. +- The Legend of Dragoon USA: `aot/overlays.json`, `disc_set` over all four discs + (identical executable images). The BPE launcher's embedded container is the engine + (`prepared_files`, two code extents at 0x80010000) and the ten BPE overlays fill + slot A (0x800C6688) or slot B (0x800FB778); all linked with `static`. - WipEout 3 SE PAL: `aot/overlays.json`, the front-end menu overlay (member 5 of `WIPEOUT3/NFE.PB`, `indexed_lzss_members`) from the stock disc and from the NTSC / PAL Mode package's patched disc, plus that package's engine image; all @@ -171,11 +176,6 @@ data). Use `fixed_address_extents` on the prepared file to name only the code ru each with its `sha256`, so data is never a producer: a data run decoded as code fails the generated-C audit and costs the whole image. -Legend of Dragoon (`aot/overlays.json`): the launcher's embedded container becomes -the engine image at 0x80010000 (two code runs), and each of the ten `OVL/*.OV_` -files becomes an image in overlay slot A (0x800C6688) or slot B (0x800FB778). One -`disc_set` profile covers all four discs. - ### Executable images from a mod package A trusted package can add code that no disc file contains: patched boot-EXE diff --git a/tools/aot_overlay_pipeline.py b/tools/aot_overlay_pipeline.py index 7be0ced7..2b490724 100644 --- a/tools/aot_overlay_pipeline.py +++ b/tools/aot_overlay_pipeline.py @@ -579,10 +579,31 @@ def declared_entries(source, disc): return entries -def make_fixed_record(source, disc): +def foreign_call_roots(source, sources): + """Entries of `source` that other images call directly (`jal`). + + A call target is a function entry by construction, but an image's own scan + cannot see calls made from elsewhere: an overlay calling a frameless engine + routine, boot code calling its constructors. Only targets inside exactly one + image count: an address that several images occupy alternatively (overlays + sharing a load slot) names no particular one. The same boundary evidence as + for local calls applies to the target bytes.""" + lo, hi = source['base'], source['base'] + len(source['body']) + others = [other for other in sources if other is not source] + targets = set() + for other in others: + targets |= {t for t in extractor.jal_targets(other['body']) if lo <= t < hi} + targets = {t for t in targets + if not any(o['base'] <= t < o['base'] + len(o['body']) for o in others)} + return extractor.direct_jal_roots(source['body'], source['base'], targets) + + +def make_fixed_record(source, disc, sources=(), cross_image=False): body, base, spec = source['body'], source['base'], source['spec'] require(spec.get('allow_missing'), f"Generic extractor missed required image: {source['name']}") direct = set(extractor.direct_jal_roots(body, base)) + if cross_image: + direct |= set(foreign_call_roots(source, sources)) seeds = set(extractor.prologues(body, base)) | extractor.frameless_leaf_entries(body, base) if spec.get('supplemental_entries', True): seeds |= extractor.supplemental_callable_seeds(body, base) @@ -665,7 +686,7 @@ def prepare(profile, disc, records, output, views=None): singles[source['name']] = record for source in sources: if source['spec']['method'] in ('fixed_address_files', 'fixed_address_extents', 'packed_sector_members', 'sector_extent_members', 'aligned_lzss_banks', 'indexed_lzss_members', 'tagged_relocated_files', 'counted_relocated_members') and source['name'] not in singles: - record = make_fixed_record(source, disc) + record = make_fixed_record(source, disc, sources, bool(profile.get('cross_image_call_roots'))) singles[source['name']] = record records.append(record) # Loader-established exports remain required even when heuristic discovery diff --git a/tools/aot_overlay_spike/extract_generic.py b/tools/aot_overlay_spike/extract_generic.py index 075aaca7..81d073fc 100644 --- a/tools/aot_overlay_spike/extract_generic.py +++ b/tools/aot_overlay_spike/extract_generic.py @@ -216,10 +216,13 @@ def jal_targets(data): ts.add(0x80000000 | ((w & 0x03FFFFFF) << 2)) return ts -def direct_jal_roots(data, base): - """Locally bounded call targets with independent boundary evidence.""" +def direct_jal_roots(data, base, targets=None): + """Call targets inside [base, base+len) with independent boundary evidence. + + `targets` are the jal targets to consider: this image's own by default; an + AOT profile can pass the targets other images call into this one.""" lo,hi=base,base+len(data) - return sorted(t for t in jal_targets(data) + return sorted(t for t in (jal_targets(data) if targets is None else targets) if lo <= t < hi and (t&3)==0 and not _dense_local_pointer_table(data,base,t-base) and (_callable_boundary(data,t-base) or diff --git a/tools/tests/test_aot_overlay_pipeline.py b/tools/tests/test_aot_overlay_pipeline.py index 49c3fca7..46bf98d5 100644 --- a/tools/tests/test_aot_overlay_pipeline.py +++ b/tools/tests/test_aot_overlay_pipeline.py @@ -1110,6 +1110,44 @@ class PreparedImageTests(unittest.TestCase): self.assertEqual(inventory['source_images'][0]['sha256'], sha(self.CODE)) +class CrossImageRootsTests(unittest.TestCase): + """Calls made from one image into another are entries of the callee image.""" + + @staticmethod + def image(name, base, words): + body = b''.join(struct.pack('> 2) + self.caller = self.image('CALLER', 0x80100000, [jal(self.target), nop, ret, nop]) + + def test_a_target_only_another_image_calls_becomes_a_root(self): + engine, caller = self.engine, self.caller + self.assertEqual(pipeline.foreign_call_roots(engine, [engine, caller]), [self.target]) + self.assertEqual(pipeline.foreign_call_roots(caller, [engine, caller]), []) + local = pipeline.make_fixed_record(engine, None, [engine, caller], False) + cross = pipeline.make_fixed_record(engine, None, [engine, caller], True) + self.assertNotIn(hex(self.target), local.get('static_discovery_entry_pcs', [])) + self.assertIn(hex(self.target), cross['static_discovery_entry_pcs']) + + def test_an_address_several_images_occupy_names_no_one_of_them(self): + engine = self.engine + twin = self.image('TWIN', 0x80010000, [0x03E00008, 0, 0, 0, 0x27BDFFE8, 0, 0x03E00008, 0]) + self.assertEqual(pipeline.foreign_call_roots(engine, [engine, twin, self.caller]), []) + + def test_a_target_without_boundary_evidence_is_not_a_root(self): + data = self.image('DATA', 0x80010000, [0x11111111] * 8) + self.assertEqual(pipeline.foreign_call_roots(data, [data, self.caller]), []) + + class StaticGenerationTest(unittest.TestCase): """The `static` action as `psxrecomp_cli.py generate` drives it."""