From f2c9fb33f3c477b824d9769cdd5e687be719ea3f Mon Sep 17 00:00:00 2001 From: Matt Stanley <1379tech@gmail.com> Date: Mon, 19 Oct 2026 12:28:54 -0700 Subject: [PATCH] Integrate PR402 retaining segment-aware identities and guarding v3 capture evidence --- docs/internal/OVERLAY_ROOT_ENRICHMENT.md | 32 +- recompiler/CMakeLists.txt | 10 + recompiler/src/main_psx.cpp | 27 ++ runtime/CMakeLists.txt | 14 + runtime/src/overlay_capture.c | 13 +- .../tests/test_overlay_capture_guard_word.cpp | 200 +++++++++ tools/compile_overlays.py | 150 ++++++- .../tests/test_overlay_guard_word_requests.py | 389 ++++++++++++++++++ 8 files changed, 813 insertions(+), 22 deletions(-) create mode 100644 runtime/tests/test_overlay_capture_guard_word.cpp create mode 100644 tools/tests/test_overlay_guard_word_requests.py diff --git a/docs/internal/OVERLAY_ROOT_ENRICHMENT.md b/docs/internal/OVERLAY_ROOT_ENRICHMENT.md index 61f5c267..df56e367 100644 --- a/docs/internal/OVERLAY_ROOT_ENRICHMENT.md +++ b/docs/internal/OVERLAY_ROOT_ENRICHMENT.md @@ -173,6 +173,37 @@ Measured: only the store/MMIO attribution hashes move (code that changed between native and interpreted); writes and cycles are identical. +## Guard-word requests (beads-eio.3.190) + +The `no-func-ids` failures were requests for a capture's trailing +delay-slot guard word. The capture writer appends one word past a +dirty-page run so a branch at the run's last word has its delay slot; that +word is the first word of the next page. It is readable, but the +recompiler's analysis ends before it, so a fragment requested there comes +back with an empty manifest. On Ace Combat 3 two page-boundary resume +points (0x800D1000, 0x800EA000) were requested this way once rooting gave +their images a host whose walk falls through into the guard word. + +Now nothing names the guard word: + +- `classify_overlay_seeds` excludes every kind of evidence there + (dispatch, executed, captured and static entries, TOML entries, alias + recipes, derived call targets) as `GUARD_WORD`, audited and counted. It + is never a seed, an alias or a fragment demand. +- Fragment jobs, hosted cross-variant selection, fragment batches and + static mode hold the same rule, so no path can request it. A request + that still reaches `compile_fragment_batch` is reported as a failure + (`guard-word-request`), not spent on a recompile. +- The recompiler warns when it drops a requested seed at or past Analysis + End, and when the header entry point is there; `compile_overlays` copies + the warning into the compile log. +- The capture writer records executed and dispatch PCs for the page run + only, not for the guard word. Older captures still carry such PCs; the + classifier handles them. + +No coverage is lost: the neighbour variant that holds the word as code +serves it. + ## Gate used, and what is left The gate the four titles passed is **no regression against master**: @@ -193,7 +224,6 @@ Not done: - Step 3 as a widening. The evidence rules were made stricter (image-local proof, cross-producer calls excluded), not extended with new categories. -- Step 4, the `no-func-ids` root cause. - Re-measuring Ace Combat 3. #386's AC3 numbers (rootless images 250 to 1, capacity skips 3,028 to 0) were taken without the guard; the stricter evidence rule may root fewer of those images. diff --git a/recompiler/CMakeLists.txt b/recompiler/CMakeLists.txt index f03bf6a8..660f670f 100644 --- a/recompiler/CMakeLists.txt +++ b/recompiler/CMakeLists.txt @@ -974,6 +974,16 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_overlay_guard_word_analysis_bound.py --recompiler $ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) + # Request half of the same contract: no seed, alias or fragment demand + # may name the guard word (classifier, fragment jobs, hosted selection, + # fragment batches), and the recompiler warns when it drops a seed + # past Analysis End. Bead beads-eio.3.190 (AC3 fragment no-func-ids). + add_test( + NAME overlay_guard_word_requests + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/../tools/tests/test_overlay_guard_word_requests.py + --recompiler $ + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..) # Producer half of the same contract: the capture writer declares its # guard bytes and compile_overlays puts them in the PS-EXE tag the # recompiler reads. diff --git a/recompiler/src/main_psx.cpp b/recompiler/src/main_psx.cpp index 61ade585..fe51133b 100644 --- a/recompiler/src/main_psx.cpp +++ b/recompiler/src/main_psx.cpp @@ -588,6 +588,8 @@ static int psxrecomp_game_main(int argc, char** argv) { if (!exe->header.entry_in_range()) { fmt::print(stderr, "⚠ Warning: Entry point 0x{:08X} is outside loaded code range!\n\n", exe->header.initial_pc); + } else if (exe->header.initial_pc >= exe->analysis_end_address()) { + fmt::print("WARNING: entry point 0x{:08X} is at or past Analysis End 0x{:08X} (delay-slot guard bytes, not analysable); it cannot produce a function\n", exe->header.initial_pc, exe->analysis_end_address()); } // Config code sites are code identities, and code identity is the full @@ -817,6 +819,8 @@ static int psxrecomp_game_main(int argc, char** argv) { if (ef.is_open()) { const uint32_t seed_lo = exe->header.load_address; const uint32_t seed_hi = exe->analysis_end_address(); + std::vector dropped_past_analysis_end; + std::vector dropped_outside_image; // The directive lines below name code by PC too, so they follow // the seed rule (§5.3): an address that names this image's bytes // in another segment is refused with its link-segment spelling, @@ -965,6 +969,8 @@ static int psxrecomp_game_main(int argc, char** argv) { if (phys < (seed_lo & PSXRecomp::kPhysMask) || phys >= (seed_lo & PSXRecomp::kPhysMask) + (seed_hi - seed_lo)) { ++outside_seeds; + if (exe->contains_phys(addr)) dropped_past_analysis_end.push_back(addr); + else dropped_outside_image.push_back(addr); } else if (!PSXRecomp::maps_physical(addr)) { // Only KUSEG below 0x20000000, KSEG0 and KSEG1 alias the // image's bytes; a variant in any other segment would @@ -992,6 +998,27 @@ static int psxrecomp_game_main(int argc, char** argv) { exact_entries.push_back(addr); } } + auto list_seeds = [](const std::vector& seeds) { + std::string text; + for (size_t i = 0; i < seeds.size() && i < 16; i++) + text += fmt::format("{}0x{:08X}", i ? ", " : "", seeds[i]); + if (seeds.size() > 16) text += ", ..."; + return text; + }; + if (!dropped_past_analysis_end.empty()) { + fmt::print("WARNING: dropped {} requested seed(s) at or past " + "Analysis End 0x{:08X} (delay-slot guard bytes, " + "not analysable): {}\n", + dropped_past_analysis_end.size(), seed_hi, + list_seeds(dropped_past_analysis_end)); + } + if (!dropped_outside_image.empty()) { + fmt::print("WARNING: dropped {} requested seed(s) outside the " + "image 0x{:08X}..0x{:08X}: {}\n", + dropped_outside_image.size(), seed_lo, + exe->end_address(), + list_seeds(dropped_outside_image)); + } std::sort(producer_ranges.begin(), producer_ranges.end()); for (size_t i = 1; i < producer_ranges.size(); i++) { if (producer_ranges[i - 1].second > producer_ranges[i].first) { diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index d06a90f0..f36008b9 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -1194,6 +1194,20 @@ if(BUILD_TESTING) target_link_libraries(overlay_capture_retry_test PRIVATE ${PSX_SDL_LIBRARIES}) add_test(NAME overlay_capture_retry_test COMMAND overlay_capture_retry_test) + # PC evidence stays inside the dirty-page run: nothing recorded in the + # trailing delay-slot guard word (bead beads-eio.3.190). + add_executable(overlay_capture_guard_word_test + tests/test_overlay_capture_guard_word.cpp + src/overlay_capture.c) + target_include_directories(overlay_capture_guard_word_test PRIVATE + include ${PSX_SDL_INCLUDE_DIRS}) + target_compile_definitions(overlay_capture_guard_word_test PRIVATE + SDL_MAIN_HANDLED $<$:PSX_SDL3=1>) + target_link_libraries(overlay_capture_guard_word_test PRIVATE + ${PSX_SDL_LIBRARIES}) + add_test(NAME overlay_capture_guard_word_test + COMMAND overlay_capture_guard_word_test) + add_executable(overlay_path_canon_test tests/test_overlay_path_canon.c src/overlay_path_canon.c) diff --git a/runtime/src/overlay_capture.c b/runtime/src/overlay_capture.c index d525bfd3..cb17790b 100644 --- a/runtime/src/overlay_capture.c +++ b/runtime/src/overlay_capture.c @@ -366,10 +366,13 @@ static void write_json_window(FILE *f, uint32_t win_lo_page, uint32_t virt = PSX_OVERLAY_CODE_SEGMENT | psx_ram_canonical_offset(phys); in_run = 0; + /* The readable delay-slot guard is not executable evidence. */ + const uint32_t pc_hi = phys + size - guard_bytes; + /* Seeds: only per-PC interpreter hits — execution-verified. */ int nexec = 0; int ndisp = 0; - for (uint32_t ep = phys; ep < phys + size; ep += 4u) { + for (uint32_t ep = phys; ep < pc_hi; ep += 4u) { uint32_t wi = ep >> 2; if ((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u) ndisp++; @@ -414,7 +417,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page, fprintf(f, " \"executed_pcs\": ["); int emitted_exec = 0; - for (uint32_t ep = phys; ep < phys + size; ep += 4u) { + for (uint32_t ep = phys; ep < pc_hi; ep += 4u) { uint32_t wi = ep >> 2; if (!((exec_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u)) continue; @@ -426,7 +429,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page, fprintf(f, " \"dispatch_entry_pcs\": ["); int emitted_dispatch = 0; - for (uint32_t ep = phys; ep < phys + size; ep += 4u) { + for (uint32_t ep = phys; ep < pc_hi; ep += 4u) { uint32_t wi = ep >> 2; if (!((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u)) continue; @@ -448,7 +451,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page, int emitted_seg = 0; fprintf(f, "%s\"%s\": [", seg ? ", " : "", psx_code_segment_name(seg)); - for (uint32_t ep = phys; ep < phys + size; ep += 4u) { + for (uint32_t ep = phys; ep < pc_hi; ep += 4u) { uint32_t wi = ep >> 2; if (!((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u) || !((seg_bitmap[wi >> 5] >> (wi & 31u)) & 1u)) @@ -464,7 +467,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page, fprintf(f, " \"seeds\": ["); emitted_dispatch = 0; - for (uint32_t ep = phys; ep < phys + size; ep += 4u) { + for (uint32_t ep = phys; ep < pc_hi; ep += 4u) { uint32_t wi = ep >> 2; if (!((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u)) continue; diff --git a/runtime/tests/test_overlay_capture_guard_word.cpp b/runtime/tests/test_overlay_capture_guard_word.cpp new file mode 100644 index 00000000..120db59e --- /dev/null +++ b/runtime/tests/test_overlay_capture_guard_word.cpp @@ -0,0 +1,200 @@ +/* The capture writer records PC evidence for the dirty-page run only, never + * for the trailing delay-slot guard word it appends (bead beads-eio.3.190). + * + * write_json_window grows a run by one guard word so a branch at the run's + * last word has its delay slot. That word is the first word of the NEXT page. + * The writer used to walk phys..phys+size for executed_pcs / + * dispatch_entry_pcs / seeds AFTER adding the guard, so a PC recorded in the + * next page's first word was attributed to this run. compile_overlays then + * requested it as a fragment and the recompiler (whose analysis ends before + * the guard word) returned an empty manifest: Ace Combat 3 `no-func-ids` at + * 0x800D1000 / 0x800EA000. + * + * Two shapes, both through the real writer (overlay_capture_write_json): + * A. A dispatch-only PC in the guard word of an ordinary run. + * B. Two executed runs on either side of the kernel/boot capture-window + * boundary (0x10000). The kernel run's guard word is the boot run's + * first word, executed: it must be listed by the boot run and not by the + * kernel run. */ +#include "code_provider.h" +#include "dirty_ram_interp.h" +#include "overlay_capture.h" + +#include "psx_sdl.h" + +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +uint32_t g_dirty_ram_exec_pc_bitmap[DIRTY_RAM_EXEC_BITMAP_WORDS]{}; +uint32_t g_dirty_ram_dispatch_pc_bitmap[DIRTY_RAM_EXEC_BITMAP_WORDS]{}; +uint32_t g_dirty_ram_exec_page_bitmap[DIRTY_RAM_EXEC_PAGE_BITMAP_WORDS]{}; +uint64_t g_dirty_ram_insns_run = 0; +uint64_t g_dirty_window_dispatches = 0; +uint64_t s_frame_count = 0; +uint32_t g_overlay_region_floor = 0x00010000u; +} + +namespace { + +uint8_t g_ram[2u * 1024u * 1024u]{}; +uint32_t g_dirty_pages[DIRTY_RAM_EXEC_PAGE_BITMAP_WORDS]{}; + +int provider_available() { return 0; } +int provider_request() { return 0; } +int provider_busy() { return 0; } + +const CodeProvider kProvider = { + "capture-guard-word-test", provider_available, provider_request, + provider_busy, nullptr, +}; + +void set_bit(uint32_t *bitmap, uint32_t phys) { + const uint32_t word = phys >> 2; + bitmap[word >> 5] |= 1u << (word & 31u); +} + +void set_exec(uint32_t phys) { + set_bit(g_dirty_ram_exec_pc_bitmap, phys); + set_bit(g_dirty_ram_dispatch_pc_bitmap, phys); + const uint32_t page = phys >> 12; + g_dirty_ram_exec_page_bitmap[page >> 5] |= 1u << (page & 31u); + g_dirty_pages[page >> 5] |= 1u << (page & 31u); +} + +std::string read_all(const std::filesystem::path &path) { + std::ifstream input(path, std::ios::binary); + return std::string(std::istreambuf_iterator(input), + std::istreambuf_iterator()); +} + +/* The JSON object of the region whose load_addr is `virt`, or "". */ +std::string region(const std::string &json, uint32_t virt) { + char key[64]; + std::snprintf(key, sizeof(key), "\"load_addr\": \"0x%08X\"", virt); + const size_t at = json.find(key); + if (at == std::string::npos) return std::string(); + const size_t open = json.rfind('{', at); + const size_t close = json.find('}', at); + return json.substr(open, close - open + 1); +} + +/* The array text of `field` inside one region object. */ +std::string field(const std::string &obj, const char *name) { + const std::string key = std::string("\"") + name + "\": ["; + const size_t at = obj.find(key); + if (at == std::string::npos) return std::string(); + const size_t start = at + key.size(); + return obj.substr(start, obj.find(']', start) - start); +} + +bool lists(const std::string &obj, const char *name, uint32_t virt) { + char pc[16]; + std::snprintf(pc, sizeof(pc), "\"0x%08X\"", virt); + return field(obj, name).find(pc) != std::string::npos; +} + +int failures = 0; + +void expect(bool ok, const char *what) { + if (!ok) { + std::fprintf(stderr, "FAIL: %s\n", what); + ++failures; + } +} + +} // namespace + +extern "C" uint8_t *memory_get_ram_ptr(void) { return g_ram; } +extern "C" uint32_t dirty_ram_get_bitmap_word_count(void) { + return DIRTY_RAM_EXEC_PAGE_BITMAP_WORDS; +} +extern "C" uint32_t dirty_ram_get_bitmap_word(uint32_t index) { + return index < DIRTY_RAM_EXEC_PAGE_BITMAP_WORDS ? g_dirty_pages[index] : 0u; +} +extern "C" int cdrom_load_in_progress(void) { return 0; } +extern "C" int fntrace_is_game_started(void) { return 1; } +extern "C" void overlay_loader_check_cache(uint32_t, uint32_t, + const uint8_t *) {} +extern "C" int overlay_loader_registered_count(void) { return 0; } +extern "C" const CodeProvider *code_provider_active(void) { return &kProvider; } +extern "C" uint32_t crc32_compute(const uint8_t *data, size_t size) { + uint32_t hash = 2166136261u; + for (size_t i = 0; i < size; ++i) hash = (hash ^ data[i]) * 16777619u; + return hash ? hash : 1u; +} + +int main() { + if (psx_sdl_init(0) != 0) { + std::fprintf(stderr, "FAIL: SDL_Init: %s\n", SDL_GetError()); + return 1; + } + const auto root = std::filesystem::temp_directory_path() / + ("psxrecomp-capture-guard-" + std::to_string( + static_cast(SDL_GetPerformanceCounter()))); + std::error_code ignored; + std::filesystem::remove_all(root, ignored); + std::filesystem::create_directories(root); + const auto capture = root / "overlay_captures.json"; + overlay_capture_set_path(capture.string().c_str()); + overlay_capture_set_enabled(1); + /* The writer activates on the first post-handoff DMA. */ + overlay_capture_on_dma(0x30000u, 4u, &g_ram[0x30000]); + + /* A: run [0x30000, 0x31000) + guard word 0x31000, dispatch-only there. */ + set_exec(0x30000u); + set_exec(0x30FFCu); + set_bit(g_dirty_ram_dispatch_pc_bitmap, 0x31000u); + /* B: kernel run [0xF000, 0x10000) + guard 0x10000; boot run + * [0x10000, 0x11000) whose first word is executed. */ + set_exec(0x0F000u); + set_exec(0x0FFFCu); + set_exec(0x10000u); + set_exec(0x10FFCu); + + overlay_capture_write_json(); + overlay_capture_wait_pending(); + const std::string json = read_all(capture); + + const std::string a = region(json, 0x80030000u); + expect(!a.empty(), "A: region 0x80030000 written"); + expect(a.find("\"size\": 4100") != std::string::npos, + "A: run still carries its guard word"); + expect(a.find("\"guard_bytes\": 4") != std::string::npos, + "A: guard word declared"); + expect(lists(a, "dispatch_entry_pcs", 0x80030FFCu), + "A: last word of the run is listed"); + expect(!lists(a, "dispatch_entry_pcs", 0x80031000u), + "A: guard-word PC not a dispatch entry of the run"); + expect(!lists(a, "seeds", 0x80031000u), + "A: guard-word PC not a seed of the run"); + expect(region(json, 0x80031000u).empty(), + "A: a dispatch-only guard word does not make a region"); + + const std::string kernel = region(json, 0x8000F000u); + const std::string boot = region(json, 0x80010000u); + expect(!kernel.empty() && !boot.empty(), "B: both window runs written"); + expect(kernel.find("\"guard_bytes\": 4") != std::string::npos, + "B: kernel run carries the cross-window guard word"); + expect(lists(kernel, "executed_pcs", 0x8000FFFCu), + "B: kernel run lists its last word"); + expect(!lists(kernel, "executed_pcs", 0x80010000u), + "B: kernel run does not list the boot run's first word"); + expect(!lists(kernel, "dispatch_entry_pcs", 0x80010000u), + "B: nor as a dispatch entry"); + expect(lists(boot, "executed_pcs", 0x80010000u) && + lists(boot, "dispatch_entry_pcs", 0x80010000u), + "B: the boot run still owns its first word"); + + overlay_capture_wait_pending(); + std::filesystem::remove_all(root, ignored); + SDL_Quit(); + if (failures) return 1; + std::puts("PASS: capture writer records no PC in a run's guard word"); + return 0; +} diff --git a/tools/compile_overlays.py b/tools/compile_overlays.py index e3e3e0b2..04e93631 100644 --- a/tools/compile_overlays.py +++ b/tools/compile_overlays.py @@ -2461,7 +2461,23 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, """ lo = load_addr hi = load_addr + size - region = lambda a: lo <= a < hi and (a & 3) == 0 + # Trailing delay-slot guard words (capture_guard_bytes) are readable as a + # delay slot but are not code of this image: the word after them is not + # in it, and the recompiler's analysis ends before them. Evidence there + # (a resume PC at a page boundary, a PC attributed to the run by an older + # capture writer) is excluded as GUARD_WORD and never requested; the + # neighbour variant that holds the word as code serves it. + fragment_hi = hi - capture_guard_bytes(cap, size) + region = lambda a: lo <= a < fragment_hi and (a & 3) == 0 + guard_word = lambda a: fragment_hi <= a < hi and (a & 3) == 0 + guard_word_excluded = set() + + def exclude_outside(addr: int) -> None: + if guard_word(addr): + guard_word_excluded.add(addr) + excluded[addr] = 'GUARD_WORD' + else: + excluded[addr] = 'UNKNOWN' if root_enrichment is None: root_enrichment = root_enrichment_default() if cap.get(ROOT_ENRICHMENT_OFF_KEY): @@ -2555,7 +2571,6 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, # validated dispatch evidence separately so executed entries can still use # the isolated, audited fragment path without truncating a shared host. dispatch_fragment_demands = set() - fragment_hi = hi - capture_guard_bytes(cap, size) game_text = _game_text_range(toml_doc) # Every walk in this classification is a pure function of (entry, cap): @@ -2626,7 +2641,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, def include(addr: int, reason: str): if not region(addr): - excluded[addr] = 'UNKNOWN' + exclude_outside(addr) return # Composite padding has bytes but no producer identity. A root there # would otherwise get entry_producer=None and could walk through every @@ -2690,7 +2705,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, include(addr, 'FUNCTION_POINTER_TARGET') for addr in static_discovery_entries: if not region(addr): - excluded[addr] = 'UNKNOWN' + exclude_outside(addr) elif (_callable_legacy_seed(data, load_addr, addr) or plausible_callable_target(data, load_addr, size, addr, hi)): include(addr, 'STATIC_DISCOVERY_ROOT') @@ -2721,6 +2736,8 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, for addr, _range_lo, _range_hi in static_alias_ranges: if region(addr): included[addr] = 'DISPATCH_INTERIOR' + else: + exclude_outside(addr) # Walk roots: callable entries only. DISPATCH_INTERIOR addresses are NOT # roots — as roots they would hard-cap (truncate) the sibling walk that @@ -2813,6 +2830,11 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, if ((target in initial_known and target not in droppable) or target in suppressed_derived): return False + if guard_word(target): + # A call or branch whose target is this image's guard + # word: the callee is in the next page, not this image. + guard_word_excluded.add(target) + return False if _in_delay_slot(data, load_addr, target): delay_slot_rejected.setdefault(target, 'DERIVED') return False @@ -2973,10 +2995,22 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, for addr in delay_slot_rejected: if addr not in included: excluded[addr] = 'DELAY_SLOT' - candidates = {a for a in (executed_pcs | dispatch_entry_pcs | - captured_function_entries | static_discovery_entries | - legacy_seeds | toml_entries) - if region(a)} + evidence = (executed_pcs | dispatch_entry_pcs | captured_function_entries | + static_discovery_entries | legacy_seeds | toml_entries) + candidates = {a for a in evidence if region(a)} + for addr in sorted(a for a in evidence if guard_word(a)): + exclude_outside(addr) + for addr in guard_word_excluded: + excluded[addr] = 'GUARD_WORD' + # Invariant: no seed, alias or fragment demand in the guard word. The + # recompiler's analysis ends before it, so any request there comes back + # with an empty manifest (beads-eio.3.190). + guard_leaks = sorted(a for a in (set(included) | dispatch_fragment_demands) + if not region(a)) + if guard_leaks: + raise RuntimeError( + 'BUG: classifier kept evidence outside the analysable image: ' + + ', '.join(f'0x{a:08X}' for a in guard_leaks)) for addr in sorted(candidates - set(included) - set(delay_slot_rejected)): if addr in all_branch_targets or addr in jump_table_targets: excluded[addr] = 'BRANCH_TARGET_ONLY' @@ -3030,7 +3064,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, # the label outside every current code interval; aliases already inside # a guarded body remain candidates for the future exact-resume pass. 'static_interval_fragment_demands': { - entry for entry, _lo, _hi in static_alias_ranges + entry for entry, _lo, _hi in static_alias_ranges if region(entry) }, 'rejected_cross_producer_calls': rejected_cross_producer_calls, 'accepted_cross_producer_calls': accepted_cross_producer_calls, @@ -3045,6 +3079,10 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int, 'interior_hosts': interior_hosts, 'enrichment_shadowed': enrichment_shadowed, 'delay_slot_rejected': delay_slot_rejected, + # Evidence at or past the analysable end (the trailing guard word). + 'analysable_hi': fragment_hi, + 'guard_word_excluded': guard_word_excluded | { + a for a, r in excluded.items() if r == 'GUARD_WORD'}, } # Interior entries carry the 'interior' marker so the recompiler emits # them as overlapping aliases, never as walk roots. Promoted kernel @@ -3079,6 +3117,24 @@ def report_recompiler_guard_absorptions(stdout: str) -> int: return total +RECOMPILER_SEED_DROP_LINE = re.compile( + r'^WARNING: (?:dropped \d+ requested seed\(s\)|entry point 0x[0-9A-F]{8} ' + r'is at or past Analysis End).*$', re.M) + + +def report_recompiler_seed_drops(stdout: str) -> int: + """Surface requested seeds the recompiler could not use. + + A seed at or past Analysis End (the delay-slot guard word) or outside the + image produces no function; when it was the whole request the result is + an empty manifest. The classifier never writes one, so a line here means + a request path broke that invariant (beads-eio.3.190).""" + lines = [m.group(0) for m in RECOMPILER_SEED_DROP_LINE.finditer(stdout or '')] + for line in lines: + print(f' recompiler {line}') + return len(lines) + + def print_seed_audit(audit: dict) -> None: print(f'Overlay {audit["load_addr"]:08X}_{audit["crc32"]:08X}') print(f'Region: {audit["lo"]:08X}..{audit["hi"] - 1:08X}') @@ -3122,11 +3178,16 @@ def print_seed_audit(audit: dict) -> None: print(f'branch_targets_excluded: {audit["branch_targets_excluded_count"]}') print(f'observed_only_excluded: {audit["excluded_counts"].get("OBSERVED_PC_ONLY", 0)}') print(f'unknown_excluded: {audit["excluded_counts"].get("UNKNOWN", 0)}') + print(f'guard_word_excluded: {len(audit.get("guard_word_excluded", ()))}') for addr in sorted(audit['included_reasons']): print(f' {addr:08X} {audit["included_reasons"][addr]}') for addr in sorted(audit['excluded_reasons']): reason = audit['excluded_reasons'][addr] - if reason in ('BRANCH_TARGET_ONLY', 'OBSERVED_PC_ONLY', 'UNKNOWN'): + if reason == 'GUARD_WORD': + print(f' {addr:08X} excluded: GUARD_WORD (delay-slot guard ' + f'past analysable end 0x{audit["analysable_hi"]:08X}; ' + f'never requested)') + elif reason in ('BRANCH_TARGET_ONLY', 'OBSERVED_PC_ONLY', 'UNKNOWN'): recovery = ('; isolated fragment demand retained' if addr in unhosted_dispatch else '') print(f' {addr:08X} excluded: {reason}{recovery}') @@ -4927,6 +4988,32 @@ def make_interior_fragment_job(phys_addr: int, load_addr: int, size: int, if phys_addr <= (a & 0x1FFFFFFF) < region_hi and segment_of(a) == segment_of(load_addr) } + # Trailing delay-slot guard words in `data` (readable, not discoverable). + # Stated by the capture writer where the field exists, reconstructed from + # the page-run format otherwise. No demand of any kind may name one: the + # recompiler's analysis ends before it and returns an empty manifest. + guard_bytes = capture_guard_bytes(capture, size, + f'region 0x{load_addr:08X}') + analysable_hi = region_hi - guard_bytes + guard_word_excluded = set() + + def analysable(demands: set) -> set: + kept = {a for a in demands if (a & 0x1FFFFFFF) < analysable_hi} + guard_word_excluded.update(demands - kept) + return kept + + interiors = analysable(interiors) + dispatch_roots = analysable(dispatch_roots) + observed_dispatch = analysable(observed_dispatch) + static_exact_demands = analysable(static_exact_demands) + hosted_donor_demands = analysable(hosted_donor_demands) + static_interval_demands = analysable(static_interval_demands) + forced = analysable(forced) + for addr in sorted(guard_word_excluded): + print(f' fragment demand 0x{addr:08X} excluded: GUARD_WORD ' + f'(past analysable end 0x{analysable_hi | 0x80000000:08X} of ' + f'region 0x{phys_addr:08X}; never requested)') + static_demands = static_exact_demands | static_interval_demands if not (((interiors or dispatch_roots) and executed) or observed_dispatch or static_demands or forced): return None @@ -4935,11 +5022,8 @@ def make_interior_fragment_job(phys_addr: int, load_addr: int, size: int, 'load_addr': load_addr, 'size': size, 'data': data, - # Trailing delay-slot guard words in `data` (readable, not - # discoverable). Stated by the capture writer where the field exists, - # reconstructed from the page-run format otherwise. - 'guard_bytes': capture_guard_bytes( - capture, size, f'region 0x{load_addr:08X}'), + 'guard_bytes': guard_bytes, + 'guard_word_excluded': guard_word_excluded, 'candidates': (interiors | dispatch_roots | observed_dispatch | static_demands | forced), 'executed': executed, @@ -4991,6 +5075,7 @@ def merge_fragment_jobs_by_recipe(jobs: list[dict], 'candidates', 'executed', 'static_demands', 'static_exact_demands', 'hosted_donor_demands', 'static_interval_demands', 'forced', 'guard_demoted_aliases', + 'guard_word_excluded', ) for job in jobs: key = ( @@ -5079,6 +5164,9 @@ def select_hosted_interior_demands(job: dict, donor_recipes: dict, donor_items = donor_items[:HOSTED_NOMINATION_CAP] lo = job['load_addr'] & 0x1FFFFFFF hi = lo + job['size'] + # A sibling recipe may hold as code the word that is this recipe's + # delay-slot guard; this recipe cannot analyse it. + analysable_hi = hi - job.get('guard_bytes', 0) local_reasons = { _canonical_guest_addr(addr): set(reasons) for addr, reasons in job.get('included_reasons', {}).items() @@ -5126,6 +5214,9 @@ def select_hosted_interior_demands(job: dict, donor_recipes: dict, if (target & 3) != 0 or not (lo <= target_phys < hi): rejected['target_outside'] += 1 continue + if target_phys >= analysable_hi: + rejected['target_guard_word'] += 1 + continue target_producer = _producer_for_job_addr(job, target) if target_producer is None: rejected['target_no_producer'] += 1 @@ -5587,6 +5678,17 @@ def compile_fragment_batch(requested_entries, data: bytes, load_addr: int, }) if not requested: return None, 'no-requested-entries' + # Every demand path filters the delay-slot guard word out before it gets + # here (classify_overlay_seeds, make_interior_fragment_job, hosted + # selection). One that does not is a bug, reported as a failure rather + # than spent on a recompile that can only return an empty manifest. + analysable_hi = (load_addr & 0x1FFFFFFF) + size - guard_bytes + guard_requests = sorted(entry for entry in requested + if (entry & 0x1FFFFFFF) >= analysable_hi) + if guard_requests: + return None, ('guard-word-request (BUG: demand past analysable end ' + f'0x{analysable_hi | 0x80000000:08X}): ' + + ', '.join(f'0x{entry:08X}' for entry in guard_requests)) if hosted_owners is not None: hosted_specs = { _canonical_guest_addr(entry): spec @@ -5652,6 +5754,7 @@ def compile_fragment_batch(requested_entries, data: bytes, load_addr: int, if r.returncode != 0: return None, f'recompiler-error: {(r.stderr or r.stdout or "").strip()}' report_recompiler_guard_absorptions(r.stdout) + report_recompiler_seed_drops(r.stdout) full_c = ranges_src = None for fn in os.listdir(out_dir_tmp): if fn.endswith('_full.c'): @@ -7153,7 +7256,20 @@ def _static_capture_job_in_segment(cap, args, toml, forced_interiors, result['outcome'] = 'fail' result['fail'] = (cls, detail) + # The delay-slot guard word is not code of this image and the recompiler + # cannot analyse it (beads-eio.3.190): never a static demand either. + analysable_hi = phys_addr + size - guard_bytes + for captured_entry in sorted( + _parse_addr_list(cap.get('dispatch_entry_pcs', [])) | + {forced for forced in forced_interiors + if phys_addr <= (forced & 0x1FFFFFFF) < phys_addr + size}): + if (captured_entry & 0x1FFFFFFF) >= analysable_hi: + print(f' demand 0x{captured_entry:08X} excluded: GUARD_WORD ' + f'(past analysable end 0x{analysable_hi | 0x80000000:08X}; ' + f'never requested)') for captured_entry in _parse_addr_list(cap.get('dispatch_entry_pcs', [])): + if (captured_entry & 0x1FFFFFFF) >= analysable_hi: + continue entry = _canonical_guest_addr(captured_entry) key = (entry, crc32) result['requested_entries'].add(key) @@ -7166,7 +7282,7 @@ def _static_capture_job_in_segment(cap, args, toml, forced_interiors, region_hi = phys_addr + size for forced_entry in forced_interiors: forced_phys = forced_entry & 0x1FFFFFFF - if (phys_addr <= forced_phys < region_hi and + if (phys_addr <= forced_phys < analysable_hi and segment_of(forced_entry) == segment_of(load_addr)): entry = forced_entry key = (entry, crc32) @@ -7229,6 +7345,7 @@ def _static_capture_job_in_segment(cap, args, toml, forced_interiors, fail('recompiler', r.stderr or r.stdout) return report_recompiler_guard_absorptions(r.stdout) + report_recompiler_seed_drops(r.stdout) stem = os.path.basename(psx_path) full_c = os.path.join(out_dir_tmp, stem + '_full.c') @@ -7901,6 +8018,7 @@ def main(): stats.add_fail(_label, 'recompiler', r.stderr or r.stdout) return report_recompiler_guard_absorptions(r.stdout) + report_recompiler_seed_drops(r.stdout) # Find the generated _full.c stem = os.path.basename(psx_path) diff --git a/tools/tests/test_overlay_guard_word_requests.py b/tools/tests/test_overlay_guard_word_requests.py new file mode 100644 index 00000000..ded79ce5 --- /dev/null +++ b/tools/tests/test_overlay_guard_word_requests.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +"""No overlay demand may name a capture's trailing delay-slot guard word. + +overlay_capture.c appends one guard word past a dirty-page run so a branch at +the run's last word has its delay slot. The word is the first word of the +NEXT page: readable, never code of this image, and the recompiler's analysis +ends before it (PS-EXE analysis-bound tag). + +Measured on Ace Combat 3 (bead beads-eio.3.190): two dispatch entries at +load+size-4 (page-boundary resume points, 0x800D1000 and 0x800EA000) were +classified DISPATCH_INTERIOR because a host walk falls through into the guard +word, then requested as isolated fragments. The recompiler dropped the seed +silently (past Analysis End) and returned an empty manifest: SHARD FAIL +`fragment: no-func-ids`. The fix is on the request side, not a relabel: + + 1. classify_overlay_seeds excludes every kind of evidence in the guard word + as GUARD_WORD (audited, counted, never a seed or a demand); + 2. make_interior_fragment_job, hosted selection and compile_fragment_batch + hold the same invariant, so no path can request it; + 3. the recompiler warns when it drops a requested seed past Analysis End; + 4. the capture writer no longer records PCs in the guard word (pinned by + runtime/tests/test_overlay_capture_guard_word.cpp). + +Usage: test_overlay_guard_word_requests.py [--recompiler ] +""" +import argparse +import contextlib +import io +import struct +import subprocess +import sys +import tempfile +import types +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'tools')) +import compile_overlays as CO # noqa: E402 + +RECOMPILER = None + +LOAD = 0x800D0000 +PAGE = 0x1000 +SIZE = PAGE + 4 # one dirty page plus the guard word +GUARD = LOAD + PAGE # 0x800D1000: first word of the next page +LAST = GUARD - 4 # last analysable word +NOP = 0x00000000 +JR_RA = 0x03E00008 +FRAME = 0x27BDFFE0 # addiu sp,sp,-32 +ADDIU_S5 = 0x26B50014 # addiu s5,s5,0x14 (AC3 0x800D0FFC) +ADDIU_V0 = 0x24420001 # addiu v0,v0,1 + + +def jal(target): + return 0x0C000000 | ((target >> 2) & 0x03FFFFFF) + + +def image(words: dict, size: int = SIZE, load: int = LOAD) -> bytes: + data = bytearray(size) + for addr, word in words.items(): + struct.pack_into(' bytes: + """A host function whose straight-line body runs to the page end and + falls through into the guard word, as at AC3 0x800CF000 / 0x800E8000.""" + words = {LOAD + 0xF00: FRAME} + for addr in range(LOAD + 0xF04, GUARD, 4): + words[addr] = ADDIU_V0 + words[LAST] = ADDIU_S5 + words[GUARD] = ADDIU_V0 # next page's code, captured as the guard + words.update(extra or {}) + return image(words) + + +def capture(**fields): + cap = {'schema': 'psxrecomp overlay capture v2', 'guard_bytes': 4, + 'function_entry_pcs': [], 'dispatch_entry_pcs': []} + for key, value in fields.items(): + cap[key] = ([f'0x{a:08X}' for a in value] + if isinstance(value, (list, set, tuple)) else value) + return cap + + +def classify(data, toml=None, load=LOAD, **fields): + return CO.classify_overlay_seeds(capture(**fields), data, load, len(data), + 0, toml or {}, root_enrichment=True) + + +def seed_addrs(seeds): + out = set() + for seed in seeds: + parts = seed.split() + if parts and parts[0] not in ('producer_range', 'cross_call_allow'): + out.add(int(parts[-1], 16)) + return out + + +def quiet(fn, *a, **kw): + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + result = fn(*a, **kw) + return result, buf.getvalue() + + +class ClassifierTests(unittest.TestCase): + HOST = LOAD + 0xF00 + + def assert_guard_excluded(self, seeds, audit, addr=GUARD): + self.assertEqual(audit['excluded_reasons'].get(addr), 'GUARD_WORD') + self.assertIn(addr, audit['guard_word_excluded']) + self.assertNotIn(addr, audit['included_reasons']) + self.assertNotIn(addr, seed_addrs(seeds)) + self.assertNotIn(addr, audit['dispatch_fragment_demands']) + self.assertNotIn(addr, audit['static_exact_fragment_demands']) + self.assertNotIn(addr, audit['static_interval_fragment_demands']) + self.assertNotIn(addr, audit['interior_hosts']) + self.assertEqual(audit['analysable_hi'], GUARD) + + def test_ac3_dispatch_entry_in_guard_word_is_never_requested(self): + data = ac3_shape() + # The host walk really does visit the guard word by fallthrough: + # that is what made it a hosted DISPATCH_INTERIOR before the fix. + walk = CO._walk_overlay_function(data, LOAD, SIZE, self.HOST, + LOAD + SIZE) + self.assertIn(GUARD, walk['visited']) + seeds, audit = classify(data, function_entry_pcs=[self.HOST], + dispatch_entry_pcs=[self.HOST, GUARD], + executed_pcs=[self.HOST, GUARD]) + self.assert_guard_excluded(seeds, audit) + self.assertIn(self.HOST, seed_addrs(seeds)) + job, log = quiet(CO.make_interior_fragment_job, + LOAD & 0x1FFFFFFF, LOAD, SIZE, data, audit, set(), + capture()) + if job is not None: + self.assertNotIn(GUARD, job['candidates']) + self.assertNotIn(GUARD, CO.select_fragment_orphans( + job['candidates'], job['executed'], job['forced'], + job['static_exact_demands'], job['static_interval_demands'], + set(), [])) + out = io.StringIO() + with contextlib.redirect_stdout(out): + CO.print_seed_audit(audit) + self.assertIn('guard_word_excluded: 1', out.getvalue()) + self.assertIn(f'{GUARD:08X} excluded: GUARD_WORD', out.getvalue()) + + def test_same_pc_is_code_in_the_neighbour_variant(self): + # 0x800D1000 is ordinary analysable code in the image loaded there, + # so excluding it from the guard-word image loses no coverage. + data = image({GUARD: FRAME, GUARD + 4: ADDIU_V0, GUARD + 8: JR_RA, + GUARD + 12: NOP}, load=GUARD) + seeds, audit = classify(data, load=GUARD, dispatch_entry_pcs=[GUARD], + executed_pcs=[GUARD]) + self.assertIn(GUARD, seed_addrs(seeds)) + self.assertFalse(audit['guard_word_excluded']) + + def test_last_analysable_word_is_unaffected(self): + data = ac3_shape() + _seeds, audit = classify(data, function_entry_pcs=[self.HOST], + dispatch_entry_pcs=[LAST], + executed_pcs=[LAST]) + self.assertEqual(audit['included_reasons'].get(LAST), + 'DISPATCH_INTERIOR') + self.assertIn(LAST, audit['dispatch_fragment_demands']) + self.assertFalse(audit['guard_word_excluded']) + + def test_executed_only_pc_in_guard_word(self): + seeds, audit = classify(ac3_shape(), function_entry_pcs=[self.HOST], + executed_pcs=[self.HOST, GUARD]) + self.assert_guard_excluded(seeds, audit) + + def test_every_other_evidence_source_in_guard_word(self): + data = ac3_shape() + cases = { + 'function_entry_pcs': dict(function_entry_pcs=[self.HOST, GUARD]), + 'static_discovery_entry_pcs': dict( + function_entry_pcs=[self.HOST], + static_discovery_entry_pcs=[GUARD]), + 'static_dispatch_entry_pcs': dict( + function_entry_pcs=[self.HOST], + dispatch_entry_pcs=[GUARD], static_dispatch_entry_pcs=[GUARD]), + 'legacy seeds': dict(function_entry_pcs=[self.HOST], + seeds=[GUARD]), + } + for name, fields in cases.items(): + with self.subTest(name): + seeds, audit = classify(data, **fields) + self.assert_guard_excluded(seeds, audit) + toml = {'overlays': [{'load_addr': f'0x{LOAD:08X}', + 'entries': [f'0x{GUARD:08X}']}]} + seeds, audit = classify(data, toml=toml, + function_entry_pcs=[self.HOST]) + self.assert_guard_excluded(seeds, audit) + alias = capture(function_entry_pcs=[self.HOST]) + alias['static_alias_ranges'] = [{ + 'entry': f'0x{GUARD:08X}', 'start': f'0x{self.HOST:08X}', + 'end': f'0x{LOAD + SIZE:08X}'}] + seeds, audit = CO.classify_overlay_seeds(alias, data, LOAD, SIZE, 0, + {}, root_enrichment=True) + self.assert_guard_excluded(seeds, audit) + + def test_call_into_guard_word_is_not_derived(self): + caller = LOAD + 0x100 + data = ac3_shape({caller: FRAME, caller + 4: jal(GUARD), + caller + 8: NOP, caller + 12: JR_RA, + caller + 16: NOP}) + walk = CO._walk_overlay_function(data, LOAD, SIZE, caller, + LOAD + SIZE) + self.assertIn(GUARD, walk['direct_jals']) + seeds, audit = classify(data, function_entry_pcs=[caller, self.HOST]) + self.assert_guard_excluded(seeds, audit) + + def test_declared_zero_guard_keeps_the_last_word_analysable(self): + # The exclusion follows the writer's declaration, not the size. + data = ac3_shape({GUARD: JR_RA}) + cap = capture(function_entry_pcs=[self.HOST], + dispatch_entry_pcs=[GUARD], executed_pcs=[GUARD]) + cap['guard_bytes'] = 0 + _seeds, audit = CO.classify_overlay_seeds(cap, data, LOAD, SIZE, 0, + {}, root_enrichment=True) + self.assertFalse(audit['guard_word_excluded']) + self.assertNotEqual(audit['excluded_reasons'].get(GUARD), + 'GUARD_WORD') + + +class RequestInvariantTests(unittest.TestCase): + """Paths that could still name the guard word hold the same rule.""" + + def audit(self, **sets): + audit = {'included_reasons': {}, 'executed_pcs': set(), + 'dispatch_fragment_demands': set(), + 'static_exact_fragment_demands': set(), + 'cross_variant_hosted_demands': set(), + 'static_interval_fragment_demands': set(), + 'producer_ranges': [], 'accepted_cross_producer_calls': set()} + audit.update(sets) + return audit + + def test_fragment_job_drops_forced_and_leaked_demands(self): + data = ac3_shape() + audit = self.audit( + executed_pcs={LAST, GUARD}, + dispatch_fragment_demands={GUARD}, + static_exact_fragment_demands={GUARD, LAST}, + static_interval_fragment_demands={GUARD}, + cross_variant_hosted_demands={GUARD}, + included_reasons={GUARD: 'DISPATCH_INTERIOR', + LAST: 'STATIC_DISCOVERY_ROOT'}) + job, log = quiet(CO.make_interior_fragment_job, + LOAD & 0x1FFFFFFF, LOAD, SIZE, data, audit, + {GUARD, LAST}, capture()) + for field in ('candidates', 'static_demands', 'static_exact_demands', + 'hosted_donor_demands', 'static_interval_demands', + 'forced'): + self.assertNotIn(GUARD, job[field], field) + self.assertIn(LAST, job['candidates']) + self.assertIn(LAST, job['forced']) + self.assertEqual(job['guard_word_excluded'], {GUARD}) + self.assertIn(f'fragment demand 0x{GUARD:08X} excluded: GUARD_WORD', + log) + self.assertEqual(CO.partition_strong_root_demands( + job['static_exact_demands'], job['executed'], job['forced'], + job['static_interval_demands'], set()), ([], [LAST])) + + def test_hosted_selection_rejects_a_guard_word_target(self): + job = {'load_addr': LOAD, 'size': SIZE, 'guard_bytes': 4, + 'included_reasons': {}, 'producer_ranges': ()} + audit = {} + selected = CO.select_hosted_interior_demands( + job, {GUARD: True}, [], audit) + self.assertEqual(selected, {}) + self.assertEqual(audit.get('target_guard_word'), 1) + + def test_fragment_batch_refuses_a_guard_word_request(self): + args = types.SimpleNamespace(recompiler=None, game_toml=None) + frag_ids, status = CO.compile_fragment_batch( + {GUARD}, ac3_shape(), LOAD, SIZE, LOAD & 0x1FFFFFFF, '', args, + {}, {}, guard_bytes=4) + self.assertIsNone(frag_ids) + self.assertTrue(status.startswith('guard-word-request'), status) + self.assertIn(f'0x{GUARD:08X}', status) + + +class RecompilerWarningTests(unittest.TestCase): + """The recompiler states it dropped a requested seed past Analysis End.""" + + def run_recompiler(self, entry, seed_lines): + data = ac3_shape({LAST: JR_RA, GUARD: NOP}) + with tempfile.TemporaryDirectory() as tmp: + psx = Path(tmp) / 'frag.psx' + psx.write_bytes(CO.make_psxexe(LOAD, entry, data, guard_bytes=4)) + seeds = Path(tmp) / 'seeds.txt' + seeds.write_text(''.join(line + '\n' for line in seed_lines)) + out = Path(tmp) / 'out' + out.mkdir() + return subprocess.run( + [RECOMPILER, str(psx), '--seeds', str(seeds), + '--out-dir', str(out), '--overlay'], + capture_output=True, text=True, cwd=str(ROOT)) + + def setUp(self): + if not RECOMPILER: + self.skipTest('pass --recompiler (ctest always does)') + + def test_seed_past_analysis_end_is_reported(self): + host = LOAD + 0xF00 + proc = self.run_recompiler(host, [f'dispatch_root 0x{host:08X}', + f'dispatch_root 0x{GUARD:08X}', + f'interior 0x{GUARD:08X}']) + self.assertEqual(proc.returncode, 0, proc.stderr or proc.stdout) + self.assertIn( + f'WARNING: dropped 2 requested seed(s) at or past Analysis End ' + f'0x{GUARD:08X}', proc.stdout) + self.assertNotIn('WARNING: entry point', proc.stdout) + # compile_overlays copies the warning into the compile log. + count, log = quiet(CO.report_recompiler_seed_drops, proc.stdout) + self.assertEqual(count, 1) + self.assertIn(f'recompiler WARNING: dropped 2 requested seed(s)', log) + + def test_entry_point_past_analysis_end_is_reported(self): + proc = self.run_recompiler(GUARD, [f'dispatch_root 0x{GUARD:08X}']) + self.assertIn(f'WARNING: entry point 0x{GUARD:08X} is at or past ' + f'Analysis End 0x{GUARD:08X}', proc.stdout) + self.assertIn('WARNING: dropped 1 requested seed(s) at or past ' + 'Analysis End', proc.stdout) + + def test_in_range_seeds_warn_nothing(self): + host = LOAD + 0xF00 + proc = self.run_recompiler(host, [f'dispatch_root 0x{host:08X}', + f'interior 0x{LAST:08X}']) + self.assertEqual(proc.returncode, 0, proc.stderr or proc.stdout) + self.assertNotIn('WARNING: dropped', proc.stdout) + self.assertNotIn('WARNING: entry point', proc.stdout) + + +class StaticModeTests(unittest.TestCase): + """--static binds captured dispatch entries to demands before the + classifier runs; the guard word must not become one there either.""" + + def setUp(self): + if not RECOMPILER: + self.skipTest('pass --recompiler (ctest always does)') + + def test_static_mode_never_demands_the_guard_word(self): + host = LOAD + 0xF00 + data = ac3_shape({LAST: JR_RA, GUARD: NOP}) + rec = capture(function_entry_pcs=[host], + dispatch_entry_pcs=[host, GUARD], + executed_pcs=[host, GUARD]) + rec.update({'load_addr': f'0x{LOAD:08X}', 'size': SIZE, + 'bytes_b64': __import__('base64').b64encode(data).decode()}) + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + (tmp / 'captures.json').write_text(__import__('json').dumps([rec])) + header = bytearray(0x800) + header[:8] = b'PS-X EXE' + struct.pack_into('