Integrate PR402 retaining segment-aware identities and guarding v3 capture evidence

This commit is contained in:
Matt Stanley
2026-10-19 12:28:54 -07:00
parent 4ff7ed4d6e
commit f2c9fb33f3
8 changed files with 813 additions and 22 deletions
+31 -1
View File
@@ -173,6 +173,37 @@ Measured:
only the store/MMIO attribution hashes move (code that changed between
native and interpreted); writes and cycles are identical.
## Guard-word requests (beads-eio.3.190)
The `no-func-ids` failures were requests for a capture's trailing
delay-slot guard word. The capture writer appends one word past a
dirty-page run so a branch at the run's last word has its delay slot; that
word is the first word of the next page. It is readable, but the
recompiler's analysis ends before it, so a fragment requested there comes
back with an empty manifest. On Ace Combat 3 two page-boundary resume
points (0x800D1000, 0x800EA000) were requested this way once rooting gave
their images a host whose walk falls through into the guard word.
Now nothing names the guard word:
- `classify_overlay_seeds` excludes every kind of evidence there
(dispatch, executed, captured and static entries, TOML entries, alias
recipes, derived call targets) as `GUARD_WORD`, audited and counted. It
is never a seed, an alias or a fragment demand.
- Fragment jobs, hosted cross-variant selection, fragment batches and
static mode hold the same rule, so no path can request it. A request
that still reaches `compile_fragment_batch` is reported as a failure
(`guard-word-request`), not spent on a recompile.
- The recompiler warns when it drops a requested seed at or past Analysis
End, and when the header entry point is there; `compile_overlays` copies
the warning into the compile log.
- The capture writer records executed and dispatch PCs for the page run
only, not for the guard word. Older captures still carry such PCs; the
classifier handles them.
No coverage is lost: the neighbour variant that holds the word as code
serves it.
## Gate used, and what is left
The gate the four titles passed is **no regression against master**:
@@ -193,7 +224,6 @@ Not done:
- Step 3 as a widening. The evidence rules were made stricter (image-local
proof, cross-producer calls excluded), not extended with new categories.
- Step 4, the `no-func-ids` root cause.
- Re-measuring Ace Combat 3. #386's AC3 numbers (rootless images 250 to 1,
capacity skips 3,028 to 0) were taken without the guard; the stricter
evidence rule may root fewer of those images.
+10
View File
@@ -974,6 +974,16 @@ if(BUILD_TESTING)
${CMAKE_CURRENT_SOURCE_DIR}/tests/test_overlay_guard_word_analysis_bound.py
--recompiler $<TARGET_FILE:psxrecomp-game>
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..)
# Request half of the same contract: no seed, alias or fragment demand
# may name the guard word (classifier, fragment jobs, hosted selection,
# fragment batches), and the recompiler warns when it drops a seed
# past Analysis End. Bead beads-eio.3.190 (AC3 fragment no-func-ids).
add_test(
NAME overlay_guard_word_requests
COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/../tools/tests/test_overlay_guard_word_requests.py
--recompiler $<TARGET_FILE:psxrecomp-game>
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..)
# Producer half of the same contract: the capture writer declares its
# guard bytes and compile_overlays puts them in the PS-EXE tag the
# recompiler reads.
+27
View File
@@ -588,6 +588,8 @@ static int psxrecomp_game_main(int argc, char** argv) {
if (!exe->header.entry_in_range()) {
fmt::print(stderr, "⚠ Warning: Entry point 0x{:08X} is outside loaded code range!\n\n",
exe->header.initial_pc);
} else if (exe->header.initial_pc >= exe->analysis_end_address()) {
fmt::print("WARNING: entry point 0x{:08X} is at or past Analysis End 0x{:08X} (delay-slot guard bytes, not analysable); it cannot produce a function\n", exe->header.initial_pc, exe->analysis_end_address());
}
// Config code sites are code identities, and code identity is the full
@@ -817,6 +819,8 @@ static int psxrecomp_game_main(int argc, char** argv) {
if (ef.is_open()) {
const uint32_t seed_lo = exe->header.load_address;
const uint32_t seed_hi = exe->analysis_end_address();
std::vector<uint32_t> dropped_past_analysis_end;
std::vector<uint32_t> dropped_outside_image;
// The directive lines below name code by PC too, so they follow
// the seed rule (§5.3): an address that names this image's bytes
// in another segment is refused with its link-segment spelling,
@@ -965,6 +969,8 @@ static int psxrecomp_game_main(int argc, char** argv) {
if (phys < (seed_lo & PSXRecomp::kPhysMask) ||
phys >= (seed_lo & PSXRecomp::kPhysMask) + (seed_hi - seed_lo)) {
++outside_seeds;
if (exe->contains_phys(addr)) dropped_past_analysis_end.push_back(addr);
else dropped_outside_image.push_back(addr);
} else if (!PSXRecomp::maps_physical(addr)) {
// Only KUSEG below 0x20000000, KSEG0 and KSEG1 alias the
// image's bytes; a variant in any other segment would
@@ -992,6 +998,27 @@ static int psxrecomp_game_main(int argc, char** argv) {
exact_entries.push_back(addr);
}
}
auto list_seeds = [](const std::vector<uint32_t>& seeds) {
std::string text;
for (size_t i = 0; i < seeds.size() && i < 16; i++)
text += fmt::format("{}0x{:08X}", i ? ", " : "", seeds[i]);
if (seeds.size() > 16) text += ", ...";
return text;
};
if (!dropped_past_analysis_end.empty()) {
fmt::print("WARNING: dropped {} requested seed(s) at or past "
"Analysis End 0x{:08X} (delay-slot guard bytes, "
"not analysable): {}\n",
dropped_past_analysis_end.size(), seed_hi,
list_seeds(dropped_past_analysis_end));
}
if (!dropped_outside_image.empty()) {
fmt::print("WARNING: dropped {} requested seed(s) outside the "
"image 0x{:08X}..0x{:08X}: {}\n",
dropped_outside_image.size(), seed_lo,
exe->end_address(),
list_seeds(dropped_outside_image));
}
std::sort(producer_ranges.begin(), producer_ranges.end());
for (size_t i = 1; i < producer_ranges.size(); i++) {
if (producer_ranges[i - 1].second > producer_ranges[i].first) {
+14
View File
@@ -1194,6 +1194,20 @@ if(BUILD_TESTING)
target_link_libraries(overlay_capture_retry_test PRIVATE ${PSX_SDL_LIBRARIES})
add_test(NAME overlay_capture_retry_test COMMAND overlay_capture_retry_test)
# PC evidence stays inside the dirty-page run: nothing recorded in the
# trailing delay-slot guard word (bead beads-eio.3.190).
add_executable(overlay_capture_guard_word_test
tests/test_overlay_capture_guard_word.cpp
src/overlay_capture.c)
target_include_directories(overlay_capture_guard_word_test PRIVATE
include ${PSX_SDL_INCLUDE_DIRS})
target_compile_definitions(overlay_capture_guard_word_test PRIVATE
SDL_MAIN_HANDLED $<$<BOOL:${PSX_SDL3}>:PSX_SDL3=1>)
target_link_libraries(overlay_capture_guard_word_test PRIVATE
${PSX_SDL_LIBRARIES})
add_test(NAME overlay_capture_guard_word_test
COMMAND overlay_capture_guard_word_test)
add_executable(overlay_path_canon_test
tests/test_overlay_path_canon.c
src/overlay_path_canon.c)
+8 -5
View File
@@ -366,10 +366,13 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
uint32_t virt = PSX_OVERLAY_CODE_SEGMENT | psx_ram_canonical_offset(phys);
in_run = 0;
/* The readable delay-slot guard is not executable evidence. */
const uint32_t pc_hi = phys + size - guard_bytes;
/* Seeds: only per-PC interpreter hits — execution-verified. */
int nexec = 0;
int ndisp = 0;
for (uint32_t ep = phys; ep < phys + size; ep += 4u) {
for (uint32_t ep = phys; ep < pc_hi; ep += 4u) {
uint32_t wi = ep >> 2;
if ((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u)
ndisp++;
@@ -414,7 +417,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
fprintf(f, " \"executed_pcs\": [");
int emitted_exec = 0;
for (uint32_t ep = phys; ep < phys + size; ep += 4u) {
for (uint32_t ep = phys; ep < pc_hi; ep += 4u) {
uint32_t wi = ep >> 2;
if (!((exec_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u))
continue;
@@ -426,7 +429,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
fprintf(f, " \"dispatch_entry_pcs\": [");
int emitted_dispatch = 0;
for (uint32_t ep = phys; ep < phys + size; ep += 4u) {
for (uint32_t ep = phys; ep < pc_hi; ep += 4u) {
uint32_t wi = ep >> 2;
if (!((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u))
continue;
@@ -448,7 +451,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
int emitted_seg = 0;
fprintf(f, "%s\"%s\": [", seg ? ", " : "",
psx_code_segment_name(seg));
for (uint32_t ep = phys; ep < phys + size; ep += 4u) {
for (uint32_t ep = phys; ep < pc_hi; ep += 4u) {
uint32_t wi = ep >> 2;
if (!((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u) ||
!((seg_bitmap[wi >> 5] >> (wi & 31u)) & 1u))
@@ -464,7 +467,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
fprintf(f, " \"seeds\": [");
emitted_dispatch = 0;
for (uint32_t ep = phys; ep < phys + size; ep += 4u) {
for (uint32_t ep = phys; ep < pc_hi; ep += 4u) {
uint32_t wi = ep >> 2;
if (!((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u))
continue;
@@ -0,0 +1,200 @@
/* The capture writer records PC evidence for the dirty-page run only, never
* for the trailing delay-slot guard word it appends (bead beads-eio.3.190).
*
* write_json_window grows a run by one guard word so a branch at the run's
* last word has its delay slot. That word is the first word of the NEXT page.
* The writer used to walk phys..phys+size for executed_pcs /
* dispatch_entry_pcs / seeds AFTER adding the guard, so a PC recorded in the
* next page's first word was attributed to this run. compile_overlays then
* requested it as a fragment and the recompiler (whose analysis ends before
* the guard word) returned an empty manifest: Ace Combat 3 `no-func-ids` at
* 0x800D1000 / 0x800EA000.
*
* Two shapes, both through the real writer (overlay_capture_write_json):
* A. A dispatch-only PC in the guard word of an ordinary run.
* B. Two executed runs on either side of the kernel/boot capture-window
* boundary (0x10000). The kernel run's guard word is the boot run's
* first word, executed: it must be listed by the boot run and not by the
* kernel run. */
#include "code_provider.h"
#include "dirty_ram_interp.h"
#include "overlay_capture.h"
#include "psx_sdl.h"
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <iterator>
#include <string>
extern "C" {
uint32_t g_dirty_ram_exec_pc_bitmap[DIRTY_RAM_EXEC_BITMAP_WORDS]{};
uint32_t g_dirty_ram_dispatch_pc_bitmap[DIRTY_RAM_EXEC_BITMAP_WORDS]{};
uint32_t g_dirty_ram_exec_page_bitmap[DIRTY_RAM_EXEC_PAGE_BITMAP_WORDS]{};
uint64_t g_dirty_ram_insns_run = 0;
uint64_t g_dirty_window_dispatches = 0;
uint64_t s_frame_count = 0;
uint32_t g_overlay_region_floor = 0x00010000u;
}
namespace {
uint8_t g_ram[2u * 1024u * 1024u]{};
uint32_t g_dirty_pages[DIRTY_RAM_EXEC_PAGE_BITMAP_WORDS]{};
int provider_available() { return 0; }
int provider_request() { return 0; }
int provider_busy() { return 0; }
const CodeProvider kProvider = {
"capture-guard-word-test", provider_available, provider_request,
provider_busy, nullptr,
};
void set_bit(uint32_t *bitmap, uint32_t phys) {
const uint32_t word = phys >> 2;
bitmap[word >> 5] |= 1u << (word & 31u);
}
void set_exec(uint32_t phys) {
set_bit(g_dirty_ram_exec_pc_bitmap, phys);
set_bit(g_dirty_ram_dispatch_pc_bitmap, phys);
const uint32_t page = phys >> 12;
g_dirty_ram_exec_page_bitmap[page >> 5] |= 1u << (page & 31u);
g_dirty_pages[page >> 5] |= 1u << (page & 31u);
}
std::string read_all(const std::filesystem::path &path) {
std::ifstream input(path, std::ios::binary);
return std::string(std::istreambuf_iterator<char>(input),
std::istreambuf_iterator<char>());
}
/* The JSON object of the region whose load_addr is `virt`, or "". */
std::string region(const std::string &json, uint32_t virt) {
char key[64];
std::snprintf(key, sizeof(key), "\"load_addr\": \"0x%08X\"", virt);
const size_t at = json.find(key);
if (at == std::string::npos) return std::string();
const size_t open = json.rfind('{', at);
const size_t close = json.find('}', at);
return json.substr(open, close - open + 1);
}
/* The array text of `field` inside one region object. */
std::string field(const std::string &obj, const char *name) {
const std::string key = std::string("\"") + name + "\": [";
const size_t at = obj.find(key);
if (at == std::string::npos) return std::string();
const size_t start = at + key.size();
return obj.substr(start, obj.find(']', start) - start);
}
bool lists(const std::string &obj, const char *name, uint32_t virt) {
char pc[16];
std::snprintf(pc, sizeof(pc), "\"0x%08X\"", virt);
return field(obj, name).find(pc) != std::string::npos;
}
int failures = 0;
void expect(bool ok, const char *what) {
if (!ok) {
std::fprintf(stderr, "FAIL: %s\n", what);
++failures;
}
}
} // namespace
extern "C" uint8_t *memory_get_ram_ptr(void) { return g_ram; }
extern "C" uint32_t dirty_ram_get_bitmap_word_count(void) {
return DIRTY_RAM_EXEC_PAGE_BITMAP_WORDS;
}
extern "C" uint32_t dirty_ram_get_bitmap_word(uint32_t index) {
return index < DIRTY_RAM_EXEC_PAGE_BITMAP_WORDS ? g_dirty_pages[index] : 0u;
}
extern "C" int cdrom_load_in_progress(void) { return 0; }
extern "C" int fntrace_is_game_started(void) { return 1; }
extern "C" void overlay_loader_check_cache(uint32_t, uint32_t,
const uint8_t *) {}
extern "C" int overlay_loader_registered_count(void) { return 0; }
extern "C" const CodeProvider *code_provider_active(void) { return &kProvider; }
extern "C" uint32_t crc32_compute(const uint8_t *data, size_t size) {
uint32_t hash = 2166136261u;
for (size_t i = 0; i < size; ++i) hash = (hash ^ data[i]) * 16777619u;
return hash ? hash : 1u;
}
int main() {
if (psx_sdl_init(0) != 0) {
std::fprintf(stderr, "FAIL: SDL_Init: %s\n", SDL_GetError());
return 1;
}
const auto root = std::filesystem::temp_directory_path() /
("psxrecomp-capture-guard-" + std::to_string(
static_cast<unsigned long long>(SDL_GetPerformanceCounter())));
std::error_code ignored;
std::filesystem::remove_all(root, ignored);
std::filesystem::create_directories(root);
const auto capture = root / "overlay_captures.json";
overlay_capture_set_path(capture.string().c_str());
overlay_capture_set_enabled(1);
/* The writer activates on the first post-handoff DMA. */
overlay_capture_on_dma(0x30000u, 4u, &g_ram[0x30000]);
/* A: run [0x30000, 0x31000) + guard word 0x31000, dispatch-only there. */
set_exec(0x30000u);
set_exec(0x30FFCu);
set_bit(g_dirty_ram_dispatch_pc_bitmap, 0x31000u);
/* B: kernel run [0xF000, 0x10000) + guard 0x10000; boot run
* [0x10000, 0x11000) whose first word is executed. */
set_exec(0x0F000u);
set_exec(0x0FFFCu);
set_exec(0x10000u);
set_exec(0x10FFCu);
overlay_capture_write_json();
overlay_capture_wait_pending();
const std::string json = read_all(capture);
const std::string a = region(json, 0x80030000u);
expect(!a.empty(), "A: region 0x80030000 written");
expect(a.find("\"size\": 4100") != std::string::npos,
"A: run still carries its guard word");
expect(a.find("\"guard_bytes\": 4") != std::string::npos,
"A: guard word declared");
expect(lists(a, "dispatch_entry_pcs", 0x80030FFCu),
"A: last word of the run is listed");
expect(!lists(a, "dispatch_entry_pcs", 0x80031000u),
"A: guard-word PC not a dispatch entry of the run");
expect(!lists(a, "seeds", 0x80031000u),
"A: guard-word PC not a seed of the run");
expect(region(json, 0x80031000u).empty(),
"A: a dispatch-only guard word does not make a region");
const std::string kernel = region(json, 0x8000F000u);
const std::string boot = region(json, 0x80010000u);
expect(!kernel.empty() && !boot.empty(), "B: both window runs written");
expect(kernel.find("\"guard_bytes\": 4") != std::string::npos,
"B: kernel run carries the cross-window guard word");
expect(lists(kernel, "executed_pcs", 0x8000FFFCu),
"B: kernel run lists its last word");
expect(!lists(kernel, "executed_pcs", 0x80010000u),
"B: kernel run does not list the boot run's first word");
expect(!lists(kernel, "dispatch_entry_pcs", 0x80010000u),
"B: nor as a dispatch entry");
expect(lists(boot, "executed_pcs", 0x80010000u) &&
lists(boot, "dispatch_entry_pcs", 0x80010000u),
"B: the boot run still owns its first word");
overlay_capture_wait_pending();
std::filesystem::remove_all(root, ignored);
SDL_Quit();
if (failures) return 1;
std::puts("PASS: capture writer records no PC in a run's guard word");
return 0;
}
+134 -16
View File
@@ -2461,7 +2461,23 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
"""
lo = load_addr
hi = load_addr + size
region = lambda a: lo <= a < hi and (a & 3) == 0
# Trailing delay-slot guard words (capture_guard_bytes) are readable as a
# delay slot but are not code of this image: the word after them is not
# in it, and the recompiler's analysis ends before them. Evidence there
# (a resume PC at a page boundary, a PC attributed to the run by an older
# capture writer) is excluded as GUARD_WORD and never requested; the
# neighbour variant that holds the word as code serves it.
fragment_hi = hi - capture_guard_bytes(cap, size)
region = lambda a: lo <= a < fragment_hi and (a & 3) == 0
guard_word = lambda a: fragment_hi <= a < hi and (a & 3) == 0
guard_word_excluded = set()
def exclude_outside(addr: int) -> None:
if guard_word(addr):
guard_word_excluded.add(addr)
excluded[addr] = 'GUARD_WORD'
else:
excluded[addr] = 'UNKNOWN'
if root_enrichment is None:
root_enrichment = root_enrichment_default()
if cap.get(ROOT_ENRICHMENT_OFF_KEY):
@@ -2555,7 +2571,6 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
# validated dispatch evidence separately so executed entries can still use
# the isolated, audited fragment path without truncating a shared host.
dispatch_fragment_demands = set()
fragment_hi = hi - capture_guard_bytes(cap, size)
game_text = _game_text_range(toml_doc)
# Every walk in this classification is a pure function of (entry, cap):
@@ -2626,7 +2641,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
def include(addr: int, reason: str):
if not region(addr):
excluded[addr] = 'UNKNOWN'
exclude_outside(addr)
return
# Composite padding has bytes but no producer identity. A root there
# would otherwise get entry_producer=None and could walk through every
@@ -2690,7 +2705,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
include(addr, 'FUNCTION_POINTER_TARGET')
for addr in static_discovery_entries:
if not region(addr):
excluded[addr] = 'UNKNOWN'
exclude_outside(addr)
elif (_callable_legacy_seed(data, load_addr, addr) or
plausible_callable_target(data, load_addr, size, addr, hi)):
include(addr, 'STATIC_DISCOVERY_ROOT')
@@ -2721,6 +2736,8 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
for addr, _range_lo, _range_hi in static_alias_ranges:
if region(addr):
included[addr] = 'DISPATCH_INTERIOR'
else:
exclude_outside(addr)
# Walk roots: callable entries only. DISPATCH_INTERIOR addresses are NOT
# roots — as roots they would hard-cap (truncate) the sibling walk that
@@ -2813,6 +2830,11 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
if ((target in initial_known and target not in droppable) or
target in suppressed_derived):
return False
if guard_word(target):
# A call or branch whose target is this image's guard
# word: the callee is in the next page, not this image.
guard_word_excluded.add(target)
return False
if _in_delay_slot(data, load_addr, target):
delay_slot_rejected.setdefault(target, 'DERIVED')
return False
@@ -2973,10 +2995,22 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
for addr in delay_slot_rejected:
if addr not in included:
excluded[addr] = 'DELAY_SLOT'
candidates = {a for a in (executed_pcs | dispatch_entry_pcs |
captured_function_entries | static_discovery_entries |
legacy_seeds | toml_entries)
if region(a)}
evidence = (executed_pcs | dispatch_entry_pcs | captured_function_entries |
static_discovery_entries | legacy_seeds | toml_entries)
candidates = {a for a in evidence if region(a)}
for addr in sorted(a for a in evidence if guard_word(a)):
exclude_outside(addr)
for addr in guard_word_excluded:
excluded[addr] = 'GUARD_WORD'
# Invariant: no seed, alias or fragment demand in the guard word. The
# recompiler's analysis ends before it, so any request there comes back
# with an empty manifest (beads-eio.3.190).
guard_leaks = sorted(a for a in (set(included) | dispatch_fragment_demands)
if not region(a))
if guard_leaks:
raise RuntimeError(
'BUG: classifier kept evidence outside the analysable image: ' +
', '.join(f'0x{a:08X}' for a in guard_leaks))
for addr in sorted(candidates - set(included) - set(delay_slot_rejected)):
if addr in all_branch_targets or addr in jump_table_targets:
excluded[addr] = 'BRANCH_TARGET_ONLY'
@@ -3030,7 +3064,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
# the label outside every current code interval; aliases already inside
# a guarded body remain candidates for the future exact-resume pass.
'static_interval_fragment_demands': {
entry for entry, _lo, _hi in static_alias_ranges
entry for entry, _lo, _hi in static_alias_ranges if region(entry)
},
'rejected_cross_producer_calls': rejected_cross_producer_calls,
'accepted_cross_producer_calls': accepted_cross_producer_calls,
@@ -3045,6 +3079,10 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
'interior_hosts': interior_hosts,
'enrichment_shadowed': enrichment_shadowed,
'delay_slot_rejected': delay_slot_rejected,
# Evidence at or past the analysable end (the trailing guard word).
'analysable_hi': fragment_hi,
'guard_word_excluded': guard_word_excluded | {
a for a, r in excluded.items() if r == 'GUARD_WORD'},
}
# Interior entries carry the 'interior' marker so the recompiler emits
# them as overlapping aliases, never as walk roots. Promoted kernel
@@ -3079,6 +3117,24 @@ def report_recompiler_guard_absorptions(stdout: str) -> int:
return total
RECOMPILER_SEED_DROP_LINE = re.compile(
r'^WARNING: (?:dropped \d+ requested seed\(s\)|entry point 0x[0-9A-F]{8} '
r'is at or past Analysis End).*$', re.M)
def report_recompiler_seed_drops(stdout: str) -> int:
"""Surface requested seeds the recompiler could not use.
A seed at or past Analysis End (the delay-slot guard word) or outside the
image produces no function; when it was the whole request the result is
an empty manifest. The classifier never writes one, so a line here means
a request path broke that invariant (beads-eio.3.190)."""
lines = [m.group(0) for m in RECOMPILER_SEED_DROP_LINE.finditer(stdout or '')]
for line in lines:
print(f' recompiler {line}')
return len(lines)
def print_seed_audit(audit: dict) -> None:
print(f'Overlay {audit["load_addr"]:08X}_{audit["crc32"]:08X}')
print(f'Region: {audit["lo"]:08X}..{audit["hi"] - 1:08X}')
@@ -3122,11 +3178,16 @@ def print_seed_audit(audit: dict) -> None:
print(f'branch_targets_excluded: {audit["branch_targets_excluded_count"]}')
print(f'observed_only_excluded: {audit["excluded_counts"].get("OBSERVED_PC_ONLY", 0)}')
print(f'unknown_excluded: {audit["excluded_counts"].get("UNKNOWN", 0)}')
print(f'guard_word_excluded: {len(audit.get("guard_word_excluded", ()))}')
for addr in sorted(audit['included_reasons']):
print(f' {addr:08X} {audit["included_reasons"][addr]}')
for addr in sorted(audit['excluded_reasons']):
reason = audit['excluded_reasons'][addr]
if reason in ('BRANCH_TARGET_ONLY', 'OBSERVED_PC_ONLY', 'UNKNOWN'):
if reason == 'GUARD_WORD':
print(f' {addr:08X} excluded: GUARD_WORD (delay-slot guard '
f'past analysable end 0x{audit["analysable_hi"]:08X}; '
f'never requested)')
elif reason in ('BRANCH_TARGET_ONLY', 'OBSERVED_PC_ONLY', 'UNKNOWN'):
recovery = ('; isolated fragment demand retained'
if addr in unhosted_dispatch else '')
print(f' {addr:08X} excluded: {reason}{recovery}')
@@ -4927,6 +4988,32 @@ def make_interior_fragment_job(phys_addr: int, load_addr: int, size: int,
if phys_addr <= (a & 0x1FFFFFFF) < region_hi and
segment_of(a) == segment_of(load_addr)
}
# Trailing delay-slot guard words in `data` (readable, not discoverable).
# Stated by the capture writer where the field exists, reconstructed from
# the page-run format otherwise. No demand of any kind may name one: the
# recompiler's analysis ends before it and returns an empty manifest.
guard_bytes = capture_guard_bytes(capture, size,
f'region 0x{load_addr:08X}')
analysable_hi = region_hi - guard_bytes
guard_word_excluded = set()
def analysable(demands: set) -> set:
kept = {a for a in demands if (a & 0x1FFFFFFF) < analysable_hi}
guard_word_excluded.update(demands - kept)
return kept
interiors = analysable(interiors)
dispatch_roots = analysable(dispatch_roots)
observed_dispatch = analysable(observed_dispatch)
static_exact_demands = analysable(static_exact_demands)
hosted_donor_demands = analysable(hosted_donor_demands)
static_interval_demands = analysable(static_interval_demands)
forced = analysable(forced)
for addr in sorted(guard_word_excluded):
print(f' fragment demand 0x{addr:08X} excluded: GUARD_WORD '
f'(past analysable end 0x{analysable_hi | 0x80000000:08X} of '
f'region 0x{phys_addr:08X}; never requested)')
static_demands = static_exact_demands | static_interval_demands
if not (((interiors or dispatch_roots) and executed) or observed_dispatch or
static_demands or forced):
return None
@@ -4935,11 +5022,8 @@ def make_interior_fragment_job(phys_addr: int, load_addr: int, size: int,
'load_addr': load_addr,
'size': size,
'data': data,
# Trailing delay-slot guard words in `data` (readable, not
# discoverable). Stated by the capture writer where the field exists,
# reconstructed from the page-run format otherwise.
'guard_bytes': capture_guard_bytes(
capture, size, f'region 0x{load_addr:08X}'),
'guard_bytes': guard_bytes,
'guard_word_excluded': guard_word_excluded,
'candidates': (interiors | dispatch_roots | observed_dispatch |
static_demands | forced),
'executed': executed,
@@ -4991,6 +5075,7 @@ def merge_fragment_jobs_by_recipe(jobs: list[dict],
'candidates', 'executed', 'static_demands',
'static_exact_demands', 'hosted_donor_demands',
'static_interval_demands', 'forced', 'guard_demoted_aliases',
'guard_word_excluded',
)
for job in jobs:
key = (
@@ -5079,6 +5164,9 @@ def select_hosted_interior_demands(job: dict, donor_recipes: dict,
donor_items = donor_items[:HOSTED_NOMINATION_CAP]
lo = job['load_addr'] & 0x1FFFFFFF
hi = lo + job['size']
# A sibling recipe may hold as code the word that is this recipe's
# delay-slot guard; this recipe cannot analyse it.
analysable_hi = hi - job.get('guard_bytes', 0)
local_reasons = {
_canonical_guest_addr(addr): set(reasons)
for addr, reasons in job.get('included_reasons', {}).items()
@@ -5126,6 +5214,9 @@ def select_hosted_interior_demands(job: dict, donor_recipes: dict,
if (target & 3) != 0 or not (lo <= target_phys < hi):
rejected['target_outside'] += 1
continue
if target_phys >= analysable_hi:
rejected['target_guard_word'] += 1
continue
target_producer = _producer_for_job_addr(job, target)
if target_producer is None:
rejected['target_no_producer'] += 1
@@ -5587,6 +5678,17 @@ def compile_fragment_batch(requested_entries, data: bytes, load_addr: int,
})
if not requested:
return None, 'no-requested-entries'
# Every demand path filters the delay-slot guard word out before it gets
# here (classify_overlay_seeds, make_interior_fragment_job, hosted
# selection). One that does not is a bug, reported as a failure rather
# than spent on a recompile that can only return an empty manifest.
analysable_hi = (load_addr & 0x1FFFFFFF) + size - guard_bytes
guard_requests = sorted(entry for entry in requested
if (entry & 0x1FFFFFFF) >= analysable_hi)
if guard_requests:
return None, ('guard-word-request (BUG: demand past analysable end '
f'0x{analysable_hi | 0x80000000:08X}): ' +
', '.join(f'0x{entry:08X}' for entry in guard_requests))
if hosted_owners is not None:
hosted_specs = {
_canonical_guest_addr(entry): spec
@@ -5652,6 +5754,7 @@ def compile_fragment_batch(requested_entries, data: bytes, load_addr: int,
if r.returncode != 0:
return None, f'recompiler-error: {(r.stderr or r.stdout or "").strip()}'
report_recompiler_guard_absorptions(r.stdout)
report_recompiler_seed_drops(r.stdout)
full_c = ranges_src = None
for fn in os.listdir(out_dir_tmp):
if fn.endswith('_full.c'):
@@ -7153,7 +7256,20 @@ def _static_capture_job_in_segment(cap, args, toml, forced_interiors,
result['outcome'] = 'fail'
result['fail'] = (cls, detail)
# The delay-slot guard word is not code of this image and the recompiler
# cannot analyse it (beads-eio.3.190): never a static demand either.
analysable_hi = phys_addr + size - guard_bytes
for captured_entry in sorted(
_parse_addr_list(cap.get('dispatch_entry_pcs', [])) |
{forced for forced in forced_interiors
if phys_addr <= (forced & 0x1FFFFFFF) < phys_addr + size}):
if (captured_entry & 0x1FFFFFFF) >= analysable_hi:
print(f' demand 0x{captured_entry:08X} excluded: GUARD_WORD '
f'(past analysable end 0x{analysable_hi | 0x80000000:08X}; '
f'never requested)')
for captured_entry in _parse_addr_list(cap.get('dispatch_entry_pcs', [])):
if (captured_entry & 0x1FFFFFFF) >= analysable_hi:
continue
entry = _canonical_guest_addr(captured_entry)
key = (entry, crc32)
result['requested_entries'].add(key)
@@ -7166,7 +7282,7 @@ def _static_capture_job_in_segment(cap, args, toml, forced_interiors,
region_hi = phys_addr + size
for forced_entry in forced_interiors:
forced_phys = forced_entry & 0x1FFFFFFF
if (phys_addr <= forced_phys < region_hi and
if (phys_addr <= forced_phys < analysable_hi and
segment_of(forced_entry) == segment_of(load_addr)):
entry = forced_entry
key = (entry, crc32)
@@ -7229,6 +7345,7 @@ def _static_capture_job_in_segment(cap, args, toml, forced_interiors,
fail('recompiler', r.stderr or r.stdout)
return
report_recompiler_guard_absorptions(r.stdout)
report_recompiler_seed_drops(r.stdout)
stem = os.path.basename(psx_path)
full_c = os.path.join(out_dir_tmp, stem + '_full.c')
@@ -7901,6 +8018,7 @@ def main():
stats.add_fail(_label, 'recompiler', r.stderr or r.stdout)
return
report_recompiler_guard_absorptions(r.stdout)
report_recompiler_seed_drops(r.stdout)
# Find the generated _full.c
stem = os.path.basename(psx_path)
@@ -0,0 +1,389 @@
#!/usr/bin/env python3
"""No overlay demand may name a capture's trailing delay-slot guard word.
overlay_capture.c appends one guard word past a dirty-page run so a branch at
the run's last word has its delay slot. The word is the first word of the
NEXT page: readable, never code of this image, and the recompiler's analysis
ends before it (PS-EXE analysis-bound tag).
Measured on Ace Combat 3 (bead beads-eio.3.190): two dispatch entries at
load+size-4 (page-boundary resume points, 0x800D1000 and 0x800EA000) were
classified DISPATCH_INTERIOR because a host walk falls through into the guard
word, then requested as isolated fragments. The recompiler dropped the seed
silently (past Analysis End) and returned an empty manifest: SHARD FAIL
`fragment: no-func-ids`. The fix is on the request side, not a relabel:
1. classify_overlay_seeds excludes every kind of evidence in the guard word
as GUARD_WORD (audited, counted, never a seed or a demand);
2. make_interior_fragment_job, hosted selection and compile_fragment_batch
hold the same invariant, so no path can request it;
3. the recompiler warns when it drops a requested seed past Analysis End;
4. the capture writer no longer records PCs in the guard word (pinned by
runtime/tests/test_overlay_capture_guard_word.cpp).
Usage: test_overlay_guard_word_requests.py [--recompiler <exe>]
"""
import argparse
import contextlib
import io
import struct
import subprocess
import sys
import tempfile
import types
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'tools'))
import compile_overlays as CO # noqa: E402
RECOMPILER = None
LOAD = 0x800D0000
PAGE = 0x1000
SIZE = PAGE + 4 # one dirty page plus the guard word
GUARD = LOAD + PAGE # 0x800D1000: first word of the next page
LAST = GUARD - 4 # last analysable word
NOP = 0x00000000
JR_RA = 0x03E00008
FRAME = 0x27BDFFE0 # addiu sp,sp,-32
ADDIU_S5 = 0x26B50014 # addiu s5,s5,0x14 (AC3 0x800D0FFC)
ADDIU_V0 = 0x24420001 # addiu v0,v0,1
def jal(target):
return 0x0C000000 | ((target >> 2) & 0x03FFFFFF)
def image(words: dict, size: int = SIZE, load: int = LOAD) -> bytes:
data = bytearray(size)
for addr, word in words.items():
struct.pack_into('<I', data, addr - load, word)
return bytes(data)
def ac3_shape(extra=None) -> bytes:
"""A host function whose straight-line body runs to the page end and
falls through into the guard word, as at AC3 0x800CF000 / 0x800E8000."""
words = {LOAD + 0xF00: FRAME}
for addr in range(LOAD + 0xF04, GUARD, 4):
words[addr] = ADDIU_V0
words[LAST] = ADDIU_S5
words[GUARD] = ADDIU_V0 # next page's code, captured as the guard
words.update(extra or {})
return image(words)
def capture(**fields):
cap = {'schema': 'psxrecomp overlay capture v2', 'guard_bytes': 4,
'function_entry_pcs': [], 'dispatch_entry_pcs': []}
for key, value in fields.items():
cap[key] = ([f'0x{a:08X}' for a in value]
if isinstance(value, (list, set, tuple)) else value)
return cap
def classify(data, toml=None, load=LOAD, **fields):
return CO.classify_overlay_seeds(capture(**fields), data, load, len(data),
0, toml or {}, root_enrichment=True)
def seed_addrs(seeds):
out = set()
for seed in seeds:
parts = seed.split()
if parts and parts[0] not in ('producer_range', 'cross_call_allow'):
out.add(int(parts[-1], 16))
return out
def quiet(fn, *a, **kw):
buf = io.StringIO()
with contextlib.redirect_stdout(buf):
result = fn(*a, **kw)
return result, buf.getvalue()
class ClassifierTests(unittest.TestCase):
HOST = LOAD + 0xF00
def assert_guard_excluded(self, seeds, audit, addr=GUARD):
self.assertEqual(audit['excluded_reasons'].get(addr), 'GUARD_WORD')
self.assertIn(addr, audit['guard_word_excluded'])
self.assertNotIn(addr, audit['included_reasons'])
self.assertNotIn(addr, seed_addrs(seeds))
self.assertNotIn(addr, audit['dispatch_fragment_demands'])
self.assertNotIn(addr, audit['static_exact_fragment_demands'])
self.assertNotIn(addr, audit['static_interval_fragment_demands'])
self.assertNotIn(addr, audit['interior_hosts'])
self.assertEqual(audit['analysable_hi'], GUARD)
def test_ac3_dispatch_entry_in_guard_word_is_never_requested(self):
data = ac3_shape()
# The host walk really does visit the guard word by fallthrough:
# that is what made it a hosted DISPATCH_INTERIOR before the fix.
walk = CO._walk_overlay_function(data, LOAD, SIZE, self.HOST,
LOAD + SIZE)
self.assertIn(GUARD, walk['visited'])
seeds, audit = classify(data, function_entry_pcs=[self.HOST],
dispatch_entry_pcs=[self.HOST, GUARD],
executed_pcs=[self.HOST, GUARD])
self.assert_guard_excluded(seeds, audit)
self.assertIn(self.HOST, seed_addrs(seeds))
job, log = quiet(CO.make_interior_fragment_job,
LOAD & 0x1FFFFFFF, LOAD, SIZE, data, audit, set(),
capture())
if job is not None:
self.assertNotIn(GUARD, job['candidates'])
self.assertNotIn(GUARD, CO.select_fragment_orphans(
job['candidates'], job['executed'], job['forced'],
job['static_exact_demands'], job['static_interval_demands'],
set(), []))
out = io.StringIO()
with contextlib.redirect_stdout(out):
CO.print_seed_audit(audit)
self.assertIn('guard_word_excluded: 1', out.getvalue())
self.assertIn(f'{GUARD:08X} excluded: GUARD_WORD', out.getvalue())
def test_same_pc_is_code_in_the_neighbour_variant(self):
# 0x800D1000 is ordinary analysable code in the image loaded there,
# so excluding it from the guard-word image loses no coverage.
data = image({GUARD: FRAME, GUARD + 4: ADDIU_V0, GUARD + 8: JR_RA,
GUARD + 12: NOP}, load=GUARD)
seeds, audit = classify(data, load=GUARD, dispatch_entry_pcs=[GUARD],
executed_pcs=[GUARD])
self.assertIn(GUARD, seed_addrs(seeds))
self.assertFalse(audit['guard_word_excluded'])
def test_last_analysable_word_is_unaffected(self):
data = ac3_shape()
_seeds, audit = classify(data, function_entry_pcs=[self.HOST],
dispatch_entry_pcs=[LAST],
executed_pcs=[LAST])
self.assertEqual(audit['included_reasons'].get(LAST),
'DISPATCH_INTERIOR')
self.assertIn(LAST, audit['dispatch_fragment_demands'])
self.assertFalse(audit['guard_word_excluded'])
def test_executed_only_pc_in_guard_word(self):
seeds, audit = classify(ac3_shape(), function_entry_pcs=[self.HOST],
executed_pcs=[self.HOST, GUARD])
self.assert_guard_excluded(seeds, audit)
def test_every_other_evidence_source_in_guard_word(self):
data = ac3_shape()
cases = {
'function_entry_pcs': dict(function_entry_pcs=[self.HOST, GUARD]),
'static_discovery_entry_pcs': dict(
function_entry_pcs=[self.HOST],
static_discovery_entry_pcs=[GUARD]),
'static_dispatch_entry_pcs': dict(
function_entry_pcs=[self.HOST],
dispatch_entry_pcs=[GUARD], static_dispatch_entry_pcs=[GUARD]),
'legacy seeds': dict(function_entry_pcs=[self.HOST],
seeds=[GUARD]),
}
for name, fields in cases.items():
with self.subTest(name):
seeds, audit = classify(data, **fields)
self.assert_guard_excluded(seeds, audit)
toml = {'overlays': [{'load_addr': f'0x{LOAD:08X}',
'entries': [f'0x{GUARD:08X}']}]}
seeds, audit = classify(data, toml=toml,
function_entry_pcs=[self.HOST])
self.assert_guard_excluded(seeds, audit)
alias = capture(function_entry_pcs=[self.HOST])
alias['static_alias_ranges'] = [{
'entry': f'0x{GUARD:08X}', 'start': f'0x{self.HOST:08X}',
'end': f'0x{LOAD + SIZE:08X}'}]
seeds, audit = CO.classify_overlay_seeds(alias, data, LOAD, SIZE, 0,
{}, root_enrichment=True)
self.assert_guard_excluded(seeds, audit)
def test_call_into_guard_word_is_not_derived(self):
caller = LOAD + 0x100
data = ac3_shape({caller: FRAME, caller + 4: jal(GUARD),
caller + 8: NOP, caller + 12: JR_RA,
caller + 16: NOP})
walk = CO._walk_overlay_function(data, LOAD, SIZE, caller,
LOAD + SIZE)
self.assertIn(GUARD, walk['direct_jals'])
seeds, audit = classify(data, function_entry_pcs=[caller, self.HOST])
self.assert_guard_excluded(seeds, audit)
def test_declared_zero_guard_keeps_the_last_word_analysable(self):
# The exclusion follows the writer's declaration, not the size.
data = ac3_shape({GUARD: JR_RA})
cap = capture(function_entry_pcs=[self.HOST],
dispatch_entry_pcs=[GUARD], executed_pcs=[GUARD])
cap['guard_bytes'] = 0
_seeds, audit = CO.classify_overlay_seeds(cap, data, LOAD, SIZE, 0,
{}, root_enrichment=True)
self.assertFalse(audit['guard_word_excluded'])
self.assertNotEqual(audit['excluded_reasons'].get(GUARD),
'GUARD_WORD')
class RequestInvariantTests(unittest.TestCase):
"""Paths that could still name the guard word hold the same rule."""
def audit(self, **sets):
audit = {'included_reasons': {}, 'executed_pcs': set(),
'dispatch_fragment_demands': set(),
'static_exact_fragment_demands': set(),
'cross_variant_hosted_demands': set(),
'static_interval_fragment_demands': set(),
'producer_ranges': [], 'accepted_cross_producer_calls': set()}
audit.update(sets)
return audit
def test_fragment_job_drops_forced_and_leaked_demands(self):
data = ac3_shape()
audit = self.audit(
executed_pcs={LAST, GUARD},
dispatch_fragment_demands={GUARD},
static_exact_fragment_demands={GUARD, LAST},
static_interval_fragment_demands={GUARD},
cross_variant_hosted_demands={GUARD},
included_reasons={GUARD: 'DISPATCH_INTERIOR',
LAST: 'STATIC_DISCOVERY_ROOT'})
job, log = quiet(CO.make_interior_fragment_job,
LOAD & 0x1FFFFFFF, LOAD, SIZE, data, audit,
{GUARD, LAST}, capture())
for field in ('candidates', 'static_demands', 'static_exact_demands',
'hosted_donor_demands', 'static_interval_demands',
'forced'):
self.assertNotIn(GUARD, job[field], field)
self.assertIn(LAST, job['candidates'])
self.assertIn(LAST, job['forced'])
self.assertEqual(job['guard_word_excluded'], {GUARD})
self.assertIn(f'fragment demand 0x{GUARD:08X} excluded: GUARD_WORD',
log)
self.assertEqual(CO.partition_strong_root_demands(
job['static_exact_demands'], job['executed'], job['forced'],
job['static_interval_demands'], set()), ([], [LAST]))
def test_hosted_selection_rejects_a_guard_word_target(self):
job = {'load_addr': LOAD, 'size': SIZE, 'guard_bytes': 4,
'included_reasons': {}, 'producer_ranges': ()}
audit = {}
selected = CO.select_hosted_interior_demands(
job, {GUARD: True}, [], audit)
self.assertEqual(selected, {})
self.assertEqual(audit.get('target_guard_word'), 1)
def test_fragment_batch_refuses_a_guard_word_request(self):
args = types.SimpleNamespace(recompiler=None, game_toml=None)
frag_ids, status = CO.compile_fragment_batch(
{GUARD}, ac3_shape(), LOAD, SIZE, LOAD & 0x1FFFFFFF, '', args,
{}, {}, guard_bytes=4)
self.assertIsNone(frag_ids)
self.assertTrue(status.startswith('guard-word-request'), status)
self.assertIn(f'0x{GUARD:08X}', status)
class RecompilerWarningTests(unittest.TestCase):
"""The recompiler states it dropped a requested seed past Analysis End."""
def run_recompiler(self, entry, seed_lines):
data = ac3_shape({LAST: JR_RA, GUARD: NOP})
with tempfile.TemporaryDirectory() as tmp:
psx = Path(tmp) / 'frag.psx'
psx.write_bytes(CO.make_psxexe(LOAD, entry, data, guard_bytes=4))
seeds = Path(tmp) / 'seeds.txt'
seeds.write_text(''.join(line + '\n' for line in seed_lines))
out = Path(tmp) / 'out'
out.mkdir()
return subprocess.run(
[RECOMPILER, str(psx), '--seeds', str(seeds),
'--out-dir', str(out), '--overlay'],
capture_output=True, text=True, cwd=str(ROOT))
def setUp(self):
if not RECOMPILER:
self.skipTest('pass --recompiler (ctest always does)')
def test_seed_past_analysis_end_is_reported(self):
host = LOAD + 0xF00
proc = self.run_recompiler(host, [f'dispatch_root 0x{host:08X}',
f'dispatch_root 0x{GUARD:08X}',
f'interior 0x{GUARD:08X}'])
self.assertEqual(proc.returncode, 0, proc.stderr or proc.stdout)
self.assertIn(
f'WARNING: dropped 2 requested seed(s) at or past Analysis End '
f'0x{GUARD:08X}', proc.stdout)
self.assertNotIn('WARNING: entry point', proc.stdout)
# compile_overlays copies the warning into the compile log.
count, log = quiet(CO.report_recompiler_seed_drops, proc.stdout)
self.assertEqual(count, 1)
self.assertIn(f'recompiler WARNING: dropped 2 requested seed(s)', log)
def test_entry_point_past_analysis_end_is_reported(self):
proc = self.run_recompiler(GUARD, [f'dispatch_root 0x{GUARD:08X}'])
self.assertIn(f'WARNING: entry point 0x{GUARD:08X} is at or past '
f'Analysis End 0x{GUARD:08X}', proc.stdout)
self.assertIn('WARNING: dropped 1 requested seed(s) at or past '
'Analysis End', proc.stdout)
def test_in_range_seeds_warn_nothing(self):
host = LOAD + 0xF00
proc = self.run_recompiler(host, [f'dispatch_root 0x{host:08X}',
f'interior 0x{LAST:08X}'])
self.assertEqual(proc.returncode, 0, proc.stderr or proc.stdout)
self.assertNotIn('WARNING: dropped', proc.stdout)
self.assertNotIn('WARNING: entry point', proc.stdout)
class StaticModeTests(unittest.TestCase):
"""--static binds captured dispatch entries to demands before the
classifier runs; the guard word must not become one there either."""
def setUp(self):
if not RECOMPILER:
self.skipTest('pass --recompiler (ctest always does)')
def test_static_mode_never_demands_the_guard_word(self):
host = LOAD + 0xF00
data = ac3_shape({LAST: JR_RA, GUARD: NOP})
rec = capture(function_entry_pcs=[host],
dispatch_entry_pcs=[host, GUARD],
executed_pcs=[host, GUARD])
rec.update({'load_addr': f'0x{LOAD:08X}', 'size': SIZE,
'bytes_b64': __import__('base64').b64encode(data).decode()})
with tempfile.TemporaryDirectory() as tmp:
tmp = Path(tmp)
(tmp / 'captures.json').write_text(__import__('json').dumps([rec]))
header = bytearray(0x800)
header[:8] = b'PS-X EXE'
struct.pack_into('<I', header, 0x18, 0x80010000)
struct.pack_into('<I', header, 0x1C, 0x10)
(tmp / 'MAIN.EXE').write_bytes(bytes(header) + bytes(0x10))
(tmp / 'game.toml').write_text(
'[game]\nid = "TEST-00000"\nname = "x"\nexe = "MAIN.EXE"\n'
'load_address = "0x80010000"\nentry_pc = "0x80010000"\n'
'text_size = "0x10"\nstack_base = "0x801FFFF0"\n\n'
'[recompiler]\nseeds = "seeds.txt"\nout_dir = "generated"\n')
proc = subprocess.run(
[sys.executable, str(ROOT / 'tools' / 'compile_overlays.py'),
'--static', '--captures', str(tmp / 'captures.json'),
'--out-dir', str(tmp / 'out'),
'--game-toml', str(tmp / 'game.toml'),
'--recompiler', RECOMPILER, '--project-root', str(ROOT),
'--runtime-include', str(ROOT / 'runtime' / 'include'),
'--force-interior', f'0x{GUARD:08X}'],
capture_output=True, text=True, timeout=300, cwd=str(ROOT))
out = proc.stdout + proc.stderr
self.assertEqual(proc.returncode, 0, out[-3000:])
self.assertIn(f'demand 0x{GUARD:08X} excluded: GUARD_WORD', out)
self.assertNotIn('recompiler WARNING', out)
self.assertNotIn('no-func-ids', out)
self.assertNotIn('SHARD FAIL', out)
if __name__ == '__main__':
ap = argparse.ArgumentParser()
ap.add_argument('--recompiler')
args, rest = ap.parse_known_args()
RECOMPILER = args.recompiler
unittest.main(argv=[sys.argv[0]] + rest)