mirror of
https://github.com/RetroPortingToolKit/psxrecomp
synced 2026-10-26 02:11:36 -04:00
Integrate PR402 retaining segment-aware identities and guarding v3 capture evidence
This commit is contained in:
@@ -173,6 +173,37 @@ Measured:
|
||||
only the store/MMIO attribution hashes move (code that changed between
|
||||
native and interpreted); writes and cycles are identical.
|
||||
|
||||
## Guard-word requests (beads-eio.3.190)
|
||||
|
||||
The `no-func-ids` failures were requests for a capture's trailing
|
||||
delay-slot guard word. The capture writer appends one word past a
|
||||
dirty-page run so a branch at the run's last word has its delay slot; that
|
||||
word is the first word of the next page. It is readable, but the
|
||||
recompiler's analysis ends before it, so a fragment requested there comes
|
||||
back with an empty manifest. On Ace Combat 3 two page-boundary resume
|
||||
points (0x800D1000, 0x800EA000) were requested this way once rooting gave
|
||||
their images a host whose walk falls through into the guard word.
|
||||
|
||||
Now nothing names the guard word:
|
||||
|
||||
- `classify_overlay_seeds` excludes every kind of evidence there
|
||||
(dispatch, executed, captured and static entries, TOML entries, alias
|
||||
recipes, derived call targets) as `GUARD_WORD`, audited and counted. It
|
||||
is never a seed, an alias or a fragment demand.
|
||||
- Fragment jobs, hosted cross-variant selection, fragment batches and
|
||||
static mode hold the same rule, so no path can request it. A request
|
||||
that still reaches `compile_fragment_batch` is reported as a failure
|
||||
(`guard-word-request`), not spent on a recompile.
|
||||
- The recompiler warns when it drops a requested seed at or past Analysis
|
||||
End, and when the header entry point is there; `compile_overlays` copies
|
||||
the warning into the compile log.
|
||||
- The capture writer records executed and dispatch PCs for the page run
|
||||
only, not for the guard word. Older captures still carry such PCs; the
|
||||
classifier handles them.
|
||||
|
||||
No coverage is lost: the neighbour variant that holds the word as code
|
||||
serves it.
|
||||
|
||||
## Gate used, and what is left
|
||||
|
||||
The gate the four titles passed is **no regression against master**:
|
||||
@@ -193,7 +224,6 @@ Not done:
|
||||
|
||||
- Step 3 as a widening. The evidence rules were made stricter (image-local
|
||||
proof, cross-producer calls excluded), not extended with new categories.
|
||||
- Step 4, the `no-func-ids` root cause.
|
||||
- Re-measuring Ace Combat 3. #386's AC3 numbers (rootless images 250 to 1,
|
||||
capacity skips 3,028 to 0) were taken without the guard; the stricter
|
||||
evidence rule may root fewer of those images.
|
||||
|
||||
@@ -974,6 +974,16 @@ if(BUILD_TESTING)
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tests/test_overlay_guard_word_analysis_bound.py
|
||||
--recompiler $<TARGET_FILE:psxrecomp-game>
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..)
|
||||
# Request half of the same contract: no seed, alias or fragment demand
|
||||
# may name the guard word (classifier, fragment jobs, hosted selection,
|
||||
# fragment batches), and the recompiler warns when it drops a seed
|
||||
# past Analysis End. Bead beads-eio.3.190 (AC3 fragment no-func-ids).
|
||||
add_test(
|
||||
NAME overlay_guard_word_requests
|
||||
COMMAND ${Python3_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../tools/tests/test_overlay_guard_word_requests.py
|
||||
--recompiler $<TARGET_FILE:psxrecomp-game>
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..)
|
||||
# Producer half of the same contract: the capture writer declares its
|
||||
# guard bytes and compile_overlays puts them in the PS-EXE tag the
|
||||
# recompiler reads.
|
||||
|
||||
@@ -588,6 +588,8 @@ static int psxrecomp_game_main(int argc, char** argv) {
|
||||
if (!exe->header.entry_in_range()) {
|
||||
fmt::print(stderr, "⚠ Warning: Entry point 0x{:08X} is outside loaded code range!\n\n",
|
||||
exe->header.initial_pc);
|
||||
} else if (exe->header.initial_pc >= exe->analysis_end_address()) {
|
||||
fmt::print("WARNING: entry point 0x{:08X} is at or past Analysis End 0x{:08X} (delay-slot guard bytes, not analysable); it cannot produce a function\n", exe->header.initial_pc, exe->analysis_end_address());
|
||||
}
|
||||
|
||||
// Config code sites are code identities, and code identity is the full
|
||||
@@ -817,6 +819,8 @@ static int psxrecomp_game_main(int argc, char** argv) {
|
||||
if (ef.is_open()) {
|
||||
const uint32_t seed_lo = exe->header.load_address;
|
||||
const uint32_t seed_hi = exe->analysis_end_address();
|
||||
std::vector<uint32_t> dropped_past_analysis_end;
|
||||
std::vector<uint32_t> dropped_outside_image;
|
||||
// The directive lines below name code by PC too, so they follow
|
||||
// the seed rule (§5.3): an address that names this image's bytes
|
||||
// in another segment is refused with its link-segment spelling,
|
||||
@@ -965,6 +969,8 @@ static int psxrecomp_game_main(int argc, char** argv) {
|
||||
if (phys < (seed_lo & PSXRecomp::kPhysMask) ||
|
||||
phys >= (seed_lo & PSXRecomp::kPhysMask) + (seed_hi - seed_lo)) {
|
||||
++outside_seeds;
|
||||
if (exe->contains_phys(addr)) dropped_past_analysis_end.push_back(addr);
|
||||
else dropped_outside_image.push_back(addr);
|
||||
} else if (!PSXRecomp::maps_physical(addr)) {
|
||||
// Only KUSEG below 0x20000000, KSEG0 and KSEG1 alias the
|
||||
// image's bytes; a variant in any other segment would
|
||||
@@ -992,6 +998,27 @@ static int psxrecomp_game_main(int argc, char** argv) {
|
||||
exact_entries.push_back(addr);
|
||||
}
|
||||
}
|
||||
auto list_seeds = [](const std::vector<uint32_t>& seeds) {
|
||||
std::string text;
|
||||
for (size_t i = 0; i < seeds.size() && i < 16; i++)
|
||||
text += fmt::format("{}0x{:08X}", i ? ", " : "", seeds[i]);
|
||||
if (seeds.size() > 16) text += ", ...";
|
||||
return text;
|
||||
};
|
||||
if (!dropped_past_analysis_end.empty()) {
|
||||
fmt::print("WARNING: dropped {} requested seed(s) at or past "
|
||||
"Analysis End 0x{:08X} (delay-slot guard bytes, "
|
||||
"not analysable): {}\n",
|
||||
dropped_past_analysis_end.size(), seed_hi,
|
||||
list_seeds(dropped_past_analysis_end));
|
||||
}
|
||||
if (!dropped_outside_image.empty()) {
|
||||
fmt::print("WARNING: dropped {} requested seed(s) outside the "
|
||||
"image 0x{:08X}..0x{:08X}: {}\n",
|
||||
dropped_outside_image.size(), seed_lo,
|
||||
exe->end_address(),
|
||||
list_seeds(dropped_outside_image));
|
||||
}
|
||||
std::sort(producer_ranges.begin(), producer_ranges.end());
|
||||
for (size_t i = 1; i < producer_ranges.size(); i++) {
|
||||
if (producer_ranges[i - 1].second > producer_ranges[i].first) {
|
||||
|
||||
@@ -1194,6 +1194,20 @@ if(BUILD_TESTING)
|
||||
target_link_libraries(overlay_capture_retry_test PRIVATE ${PSX_SDL_LIBRARIES})
|
||||
add_test(NAME overlay_capture_retry_test COMMAND overlay_capture_retry_test)
|
||||
|
||||
# PC evidence stays inside the dirty-page run: nothing recorded in the
|
||||
# trailing delay-slot guard word (bead beads-eio.3.190).
|
||||
add_executable(overlay_capture_guard_word_test
|
||||
tests/test_overlay_capture_guard_word.cpp
|
||||
src/overlay_capture.c)
|
||||
target_include_directories(overlay_capture_guard_word_test PRIVATE
|
||||
include ${PSX_SDL_INCLUDE_DIRS})
|
||||
target_compile_definitions(overlay_capture_guard_word_test PRIVATE
|
||||
SDL_MAIN_HANDLED $<$<BOOL:${PSX_SDL3}>:PSX_SDL3=1>)
|
||||
target_link_libraries(overlay_capture_guard_word_test PRIVATE
|
||||
${PSX_SDL_LIBRARIES})
|
||||
add_test(NAME overlay_capture_guard_word_test
|
||||
COMMAND overlay_capture_guard_word_test)
|
||||
|
||||
add_executable(overlay_path_canon_test
|
||||
tests/test_overlay_path_canon.c
|
||||
src/overlay_path_canon.c)
|
||||
|
||||
@@ -366,10 +366,13 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
|
||||
uint32_t virt = PSX_OVERLAY_CODE_SEGMENT | psx_ram_canonical_offset(phys);
|
||||
in_run = 0;
|
||||
|
||||
/* The readable delay-slot guard is not executable evidence. */
|
||||
const uint32_t pc_hi = phys + size - guard_bytes;
|
||||
|
||||
/* Seeds: only per-PC interpreter hits — execution-verified. */
|
||||
int nexec = 0;
|
||||
int ndisp = 0;
|
||||
for (uint32_t ep = phys; ep < phys + size; ep += 4u) {
|
||||
for (uint32_t ep = phys; ep < pc_hi; ep += 4u) {
|
||||
uint32_t wi = ep >> 2;
|
||||
if ((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u)
|
||||
ndisp++;
|
||||
@@ -414,7 +417,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
|
||||
|
||||
fprintf(f, " \"executed_pcs\": [");
|
||||
int emitted_exec = 0;
|
||||
for (uint32_t ep = phys; ep < phys + size; ep += 4u) {
|
||||
for (uint32_t ep = phys; ep < pc_hi; ep += 4u) {
|
||||
uint32_t wi = ep >> 2;
|
||||
if (!((exec_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u))
|
||||
continue;
|
||||
@@ -426,7 +429,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
|
||||
|
||||
fprintf(f, " \"dispatch_entry_pcs\": [");
|
||||
int emitted_dispatch = 0;
|
||||
for (uint32_t ep = phys; ep < phys + size; ep += 4u) {
|
||||
for (uint32_t ep = phys; ep < pc_hi; ep += 4u) {
|
||||
uint32_t wi = ep >> 2;
|
||||
if (!((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u))
|
||||
continue;
|
||||
@@ -448,7 +451,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
|
||||
int emitted_seg = 0;
|
||||
fprintf(f, "%s\"%s\": [", seg ? ", " : "",
|
||||
psx_code_segment_name(seg));
|
||||
for (uint32_t ep = phys; ep < phys + size; ep += 4u) {
|
||||
for (uint32_t ep = phys; ep < pc_hi; ep += 4u) {
|
||||
uint32_t wi = ep >> 2;
|
||||
if (!((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u) ||
|
||||
!((seg_bitmap[wi >> 5] >> (wi & 31u)) & 1u))
|
||||
@@ -464,7 +467,7 @@ static void write_json_window(FILE *f, uint32_t win_lo_page,
|
||||
|
||||
fprintf(f, " \"seeds\": [");
|
||||
emitted_dispatch = 0;
|
||||
for (uint32_t ep = phys; ep < phys + size; ep += 4u) {
|
||||
for (uint32_t ep = phys; ep < pc_hi; ep += 4u) {
|
||||
uint32_t wi = ep >> 2;
|
||||
if (!((dispatch_pc_bitmap[wi >> 5] >> (wi & 31u)) & 1u))
|
||||
continue;
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
/* The capture writer records PC evidence for the dirty-page run only, never
|
||||
* for the trailing delay-slot guard word it appends (bead beads-eio.3.190).
|
||||
*
|
||||
* write_json_window grows a run by one guard word so a branch at the run's
|
||||
* last word has its delay slot. That word is the first word of the NEXT page.
|
||||
* The writer used to walk phys..phys+size for executed_pcs /
|
||||
* dispatch_entry_pcs / seeds AFTER adding the guard, so a PC recorded in the
|
||||
* next page's first word was attributed to this run. compile_overlays then
|
||||
* requested it as a fragment and the recompiler (whose analysis ends before
|
||||
* the guard word) returned an empty manifest: Ace Combat 3 `no-func-ids` at
|
||||
* 0x800D1000 / 0x800EA000.
|
||||
*
|
||||
* Two shapes, both through the real writer (overlay_capture_write_json):
|
||||
* A. A dispatch-only PC in the guard word of an ordinary run.
|
||||
* B. Two executed runs on either side of the kernel/boot capture-window
|
||||
* boundary (0x10000). The kernel run's guard word is the boot run's
|
||||
* first word, executed: it must be listed by the boot run and not by the
|
||||
* kernel run. */
|
||||
#include "code_provider.h"
|
||||
#include "dirty_ram_interp.h"
|
||||
#include "overlay_capture.h"
|
||||
|
||||
#include "psx_sdl.h"
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iterator>
|
||||
#include <string>
|
||||
|
||||
extern "C" {
|
||||
uint32_t g_dirty_ram_exec_pc_bitmap[DIRTY_RAM_EXEC_BITMAP_WORDS]{};
|
||||
uint32_t g_dirty_ram_dispatch_pc_bitmap[DIRTY_RAM_EXEC_BITMAP_WORDS]{};
|
||||
uint32_t g_dirty_ram_exec_page_bitmap[DIRTY_RAM_EXEC_PAGE_BITMAP_WORDS]{};
|
||||
uint64_t g_dirty_ram_insns_run = 0;
|
||||
uint64_t g_dirty_window_dispatches = 0;
|
||||
uint64_t s_frame_count = 0;
|
||||
uint32_t g_overlay_region_floor = 0x00010000u;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
uint8_t g_ram[2u * 1024u * 1024u]{};
|
||||
uint32_t g_dirty_pages[DIRTY_RAM_EXEC_PAGE_BITMAP_WORDS]{};
|
||||
|
||||
int provider_available() { return 0; }
|
||||
int provider_request() { return 0; }
|
||||
int provider_busy() { return 0; }
|
||||
|
||||
const CodeProvider kProvider = {
|
||||
"capture-guard-word-test", provider_available, provider_request,
|
||||
provider_busy, nullptr,
|
||||
};
|
||||
|
||||
void set_bit(uint32_t *bitmap, uint32_t phys) {
|
||||
const uint32_t word = phys >> 2;
|
||||
bitmap[word >> 5] |= 1u << (word & 31u);
|
||||
}
|
||||
|
||||
void set_exec(uint32_t phys) {
|
||||
set_bit(g_dirty_ram_exec_pc_bitmap, phys);
|
||||
set_bit(g_dirty_ram_dispatch_pc_bitmap, phys);
|
||||
const uint32_t page = phys >> 12;
|
||||
g_dirty_ram_exec_page_bitmap[page >> 5] |= 1u << (page & 31u);
|
||||
g_dirty_pages[page >> 5] |= 1u << (page & 31u);
|
||||
}
|
||||
|
||||
std::string read_all(const std::filesystem::path &path) {
|
||||
std::ifstream input(path, std::ios::binary);
|
||||
return std::string(std::istreambuf_iterator<char>(input),
|
||||
std::istreambuf_iterator<char>());
|
||||
}
|
||||
|
||||
/* The JSON object of the region whose load_addr is `virt`, or "". */
|
||||
std::string region(const std::string &json, uint32_t virt) {
|
||||
char key[64];
|
||||
std::snprintf(key, sizeof(key), "\"load_addr\": \"0x%08X\"", virt);
|
||||
const size_t at = json.find(key);
|
||||
if (at == std::string::npos) return std::string();
|
||||
const size_t open = json.rfind('{', at);
|
||||
const size_t close = json.find('}', at);
|
||||
return json.substr(open, close - open + 1);
|
||||
}
|
||||
|
||||
/* The array text of `field` inside one region object. */
|
||||
std::string field(const std::string &obj, const char *name) {
|
||||
const std::string key = std::string("\"") + name + "\": [";
|
||||
const size_t at = obj.find(key);
|
||||
if (at == std::string::npos) return std::string();
|
||||
const size_t start = at + key.size();
|
||||
return obj.substr(start, obj.find(']', start) - start);
|
||||
}
|
||||
|
||||
bool lists(const std::string &obj, const char *name, uint32_t virt) {
|
||||
char pc[16];
|
||||
std::snprintf(pc, sizeof(pc), "\"0x%08X\"", virt);
|
||||
return field(obj, name).find(pc) != std::string::npos;
|
||||
}
|
||||
|
||||
int failures = 0;
|
||||
|
||||
void expect(bool ok, const char *what) {
|
||||
if (!ok) {
|
||||
std::fprintf(stderr, "FAIL: %s\n", what);
|
||||
++failures;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
extern "C" uint8_t *memory_get_ram_ptr(void) { return g_ram; }
|
||||
extern "C" uint32_t dirty_ram_get_bitmap_word_count(void) {
|
||||
return DIRTY_RAM_EXEC_PAGE_BITMAP_WORDS;
|
||||
}
|
||||
extern "C" uint32_t dirty_ram_get_bitmap_word(uint32_t index) {
|
||||
return index < DIRTY_RAM_EXEC_PAGE_BITMAP_WORDS ? g_dirty_pages[index] : 0u;
|
||||
}
|
||||
extern "C" int cdrom_load_in_progress(void) { return 0; }
|
||||
extern "C" int fntrace_is_game_started(void) { return 1; }
|
||||
extern "C" void overlay_loader_check_cache(uint32_t, uint32_t,
|
||||
const uint8_t *) {}
|
||||
extern "C" int overlay_loader_registered_count(void) { return 0; }
|
||||
extern "C" const CodeProvider *code_provider_active(void) { return &kProvider; }
|
||||
extern "C" uint32_t crc32_compute(const uint8_t *data, size_t size) {
|
||||
uint32_t hash = 2166136261u;
|
||||
for (size_t i = 0; i < size; ++i) hash = (hash ^ data[i]) * 16777619u;
|
||||
return hash ? hash : 1u;
|
||||
}
|
||||
|
||||
int main() {
|
||||
if (psx_sdl_init(0) != 0) {
|
||||
std::fprintf(stderr, "FAIL: SDL_Init: %s\n", SDL_GetError());
|
||||
return 1;
|
||||
}
|
||||
const auto root = std::filesystem::temp_directory_path() /
|
||||
("psxrecomp-capture-guard-" + std::to_string(
|
||||
static_cast<unsigned long long>(SDL_GetPerformanceCounter())));
|
||||
std::error_code ignored;
|
||||
std::filesystem::remove_all(root, ignored);
|
||||
std::filesystem::create_directories(root);
|
||||
const auto capture = root / "overlay_captures.json";
|
||||
overlay_capture_set_path(capture.string().c_str());
|
||||
overlay_capture_set_enabled(1);
|
||||
/* The writer activates on the first post-handoff DMA. */
|
||||
overlay_capture_on_dma(0x30000u, 4u, &g_ram[0x30000]);
|
||||
|
||||
/* A: run [0x30000, 0x31000) + guard word 0x31000, dispatch-only there. */
|
||||
set_exec(0x30000u);
|
||||
set_exec(0x30FFCu);
|
||||
set_bit(g_dirty_ram_dispatch_pc_bitmap, 0x31000u);
|
||||
/* B: kernel run [0xF000, 0x10000) + guard 0x10000; boot run
|
||||
* [0x10000, 0x11000) whose first word is executed. */
|
||||
set_exec(0x0F000u);
|
||||
set_exec(0x0FFFCu);
|
||||
set_exec(0x10000u);
|
||||
set_exec(0x10FFCu);
|
||||
|
||||
overlay_capture_write_json();
|
||||
overlay_capture_wait_pending();
|
||||
const std::string json = read_all(capture);
|
||||
|
||||
const std::string a = region(json, 0x80030000u);
|
||||
expect(!a.empty(), "A: region 0x80030000 written");
|
||||
expect(a.find("\"size\": 4100") != std::string::npos,
|
||||
"A: run still carries its guard word");
|
||||
expect(a.find("\"guard_bytes\": 4") != std::string::npos,
|
||||
"A: guard word declared");
|
||||
expect(lists(a, "dispatch_entry_pcs", 0x80030FFCu),
|
||||
"A: last word of the run is listed");
|
||||
expect(!lists(a, "dispatch_entry_pcs", 0x80031000u),
|
||||
"A: guard-word PC not a dispatch entry of the run");
|
||||
expect(!lists(a, "seeds", 0x80031000u),
|
||||
"A: guard-word PC not a seed of the run");
|
||||
expect(region(json, 0x80031000u).empty(),
|
||||
"A: a dispatch-only guard word does not make a region");
|
||||
|
||||
const std::string kernel = region(json, 0x8000F000u);
|
||||
const std::string boot = region(json, 0x80010000u);
|
||||
expect(!kernel.empty() && !boot.empty(), "B: both window runs written");
|
||||
expect(kernel.find("\"guard_bytes\": 4") != std::string::npos,
|
||||
"B: kernel run carries the cross-window guard word");
|
||||
expect(lists(kernel, "executed_pcs", 0x8000FFFCu),
|
||||
"B: kernel run lists its last word");
|
||||
expect(!lists(kernel, "executed_pcs", 0x80010000u),
|
||||
"B: kernel run does not list the boot run's first word");
|
||||
expect(!lists(kernel, "dispatch_entry_pcs", 0x80010000u),
|
||||
"B: nor as a dispatch entry");
|
||||
expect(lists(boot, "executed_pcs", 0x80010000u) &&
|
||||
lists(boot, "dispatch_entry_pcs", 0x80010000u),
|
||||
"B: the boot run still owns its first word");
|
||||
|
||||
overlay_capture_wait_pending();
|
||||
std::filesystem::remove_all(root, ignored);
|
||||
SDL_Quit();
|
||||
if (failures) return 1;
|
||||
std::puts("PASS: capture writer records no PC in a run's guard word");
|
||||
return 0;
|
||||
}
|
||||
+134
-16
@@ -2461,7 +2461,23 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
|
||||
"""
|
||||
lo = load_addr
|
||||
hi = load_addr + size
|
||||
region = lambda a: lo <= a < hi and (a & 3) == 0
|
||||
# Trailing delay-slot guard words (capture_guard_bytes) are readable as a
|
||||
# delay slot but are not code of this image: the word after them is not
|
||||
# in it, and the recompiler's analysis ends before them. Evidence there
|
||||
# (a resume PC at a page boundary, a PC attributed to the run by an older
|
||||
# capture writer) is excluded as GUARD_WORD and never requested; the
|
||||
# neighbour variant that holds the word as code serves it.
|
||||
fragment_hi = hi - capture_guard_bytes(cap, size)
|
||||
region = lambda a: lo <= a < fragment_hi and (a & 3) == 0
|
||||
guard_word = lambda a: fragment_hi <= a < hi and (a & 3) == 0
|
||||
guard_word_excluded = set()
|
||||
|
||||
def exclude_outside(addr: int) -> None:
|
||||
if guard_word(addr):
|
||||
guard_word_excluded.add(addr)
|
||||
excluded[addr] = 'GUARD_WORD'
|
||||
else:
|
||||
excluded[addr] = 'UNKNOWN'
|
||||
if root_enrichment is None:
|
||||
root_enrichment = root_enrichment_default()
|
||||
if cap.get(ROOT_ENRICHMENT_OFF_KEY):
|
||||
@@ -2555,7 +2571,6 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
|
||||
# validated dispatch evidence separately so executed entries can still use
|
||||
# the isolated, audited fragment path without truncating a shared host.
|
||||
dispatch_fragment_demands = set()
|
||||
fragment_hi = hi - capture_guard_bytes(cap, size)
|
||||
game_text = _game_text_range(toml_doc)
|
||||
|
||||
# Every walk in this classification is a pure function of (entry, cap):
|
||||
@@ -2626,7 +2641,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
|
||||
|
||||
def include(addr: int, reason: str):
|
||||
if not region(addr):
|
||||
excluded[addr] = 'UNKNOWN'
|
||||
exclude_outside(addr)
|
||||
return
|
||||
# Composite padding has bytes but no producer identity. A root there
|
||||
# would otherwise get entry_producer=None and could walk through every
|
||||
@@ -2690,7 +2705,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
|
||||
include(addr, 'FUNCTION_POINTER_TARGET')
|
||||
for addr in static_discovery_entries:
|
||||
if not region(addr):
|
||||
excluded[addr] = 'UNKNOWN'
|
||||
exclude_outside(addr)
|
||||
elif (_callable_legacy_seed(data, load_addr, addr) or
|
||||
plausible_callable_target(data, load_addr, size, addr, hi)):
|
||||
include(addr, 'STATIC_DISCOVERY_ROOT')
|
||||
@@ -2721,6 +2736,8 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
|
||||
for addr, _range_lo, _range_hi in static_alias_ranges:
|
||||
if region(addr):
|
||||
included[addr] = 'DISPATCH_INTERIOR'
|
||||
else:
|
||||
exclude_outside(addr)
|
||||
|
||||
# Walk roots: callable entries only. DISPATCH_INTERIOR addresses are NOT
|
||||
# roots — as roots they would hard-cap (truncate) the sibling walk that
|
||||
@@ -2813,6 +2830,11 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
|
||||
if ((target in initial_known and target not in droppable) or
|
||||
target in suppressed_derived):
|
||||
return False
|
||||
if guard_word(target):
|
||||
# A call or branch whose target is this image's guard
|
||||
# word: the callee is in the next page, not this image.
|
||||
guard_word_excluded.add(target)
|
||||
return False
|
||||
if _in_delay_slot(data, load_addr, target):
|
||||
delay_slot_rejected.setdefault(target, 'DERIVED')
|
||||
return False
|
||||
@@ -2973,10 +2995,22 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
|
||||
for addr in delay_slot_rejected:
|
||||
if addr not in included:
|
||||
excluded[addr] = 'DELAY_SLOT'
|
||||
candidates = {a for a in (executed_pcs | dispatch_entry_pcs |
|
||||
captured_function_entries | static_discovery_entries |
|
||||
legacy_seeds | toml_entries)
|
||||
if region(a)}
|
||||
evidence = (executed_pcs | dispatch_entry_pcs | captured_function_entries |
|
||||
static_discovery_entries | legacy_seeds | toml_entries)
|
||||
candidates = {a for a in evidence if region(a)}
|
||||
for addr in sorted(a for a in evidence if guard_word(a)):
|
||||
exclude_outside(addr)
|
||||
for addr in guard_word_excluded:
|
||||
excluded[addr] = 'GUARD_WORD'
|
||||
# Invariant: no seed, alias or fragment demand in the guard word. The
|
||||
# recompiler's analysis ends before it, so any request there comes back
|
||||
# with an empty manifest (beads-eio.3.190).
|
||||
guard_leaks = sorted(a for a in (set(included) | dispatch_fragment_demands)
|
||||
if not region(a))
|
||||
if guard_leaks:
|
||||
raise RuntimeError(
|
||||
'BUG: classifier kept evidence outside the analysable image: ' +
|
||||
', '.join(f'0x{a:08X}' for a in guard_leaks))
|
||||
for addr in sorted(candidates - set(included) - set(delay_slot_rejected)):
|
||||
if addr in all_branch_targets or addr in jump_table_targets:
|
||||
excluded[addr] = 'BRANCH_TARGET_ONLY'
|
||||
@@ -3030,7 +3064,7 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
|
||||
# the label outside every current code interval; aliases already inside
|
||||
# a guarded body remain candidates for the future exact-resume pass.
|
||||
'static_interval_fragment_demands': {
|
||||
entry for entry, _lo, _hi in static_alias_ranges
|
||||
entry for entry, _lo, _hi in static_alias_ranges if region(entry)
|
||||
},
|
||||
'rejected_cross_producer_calls': rejected_cross_producer_calls,
|
||||
'accepted_cross_producer_calls': accepted_cross_producer_calls,
|
||||
@@ -3045,6 +3079,10 @@ def classify_overlay_seeds(cap: dict, data: bytes, load_addr: int, size: int,
|
||||
'interior_hosts': interior_hosts,
|
||||
'enrichment_shadowed': enrichment_shadowed,
|
||||
'delay_slot_rejected': delay_slot_rejected,
|
||||
# Evidence at or past the analysable end (the trailing guard word).
|
||||
'analysable_hi': fragment_hi,
|
||||
'guard_word_excluded': guard_word_excluded | {
|
||||
a for a, r in excluded.items() if r == 'GUARD_WORD'},
|
||||
}
|
||||
# Interior entries carry the 'interior' marker so the recompiler emits
|
||||
# them as overlapping aliases, never as walk roots. Promoted kernel
|
||||
@@ -3079,6 +3117,24 @@ def report_recompiler_guard_absorptions(stdout: str) -> int:
|
||||
return total
|
||||
|
||||
|
||||
RECOMPILER_SEED_DROP_LINE = re.compile(
|
||||
r'^WARNING: (?:dropped \d+ requested seed\(s\)|entry point 0x[0-9A-F]{8} '
|
||||
r'is at or past Analysis End).*$', re.M)
|
||||
|
||||
|
||||
def report_recompiler_seed_drops(stdout: str) -> int:
|
||||
"""Surface requested seeds the recompiler could not use.
|
||||
|
||||
A seed at or past Analysis End (the delay-slot guard word) or outside the
|
||||
image produces no function; when it was the whole request the result is
|
||||
an empty manifest. The classifier never writes one, so a line here means
|
||||
a request path broke that invariant (beads-eio.3.190)."""
|
||||
lines = [m.group(0) for m in RECOMPILER_SEED_DROP_LINE.finditer(stdout or '')]
|
||||
for line in lines:
|
||||
print(f' recompiler {line}')
|
||||
return len(lines)
|
||||
|
||||
|
||||
def print_seed_audit(audit: dict) -> None:
|
||||
print(f'Overlay {audit["load_addr"]:08X}_{audit["crc32"]:08X}')
|
||||
print(f'Region: {audit["lo"]:08X}..{audit["hi"] - 1:08X}')
|
||||
@@ -3122,11 +3178,16 @@ def print_seed_audit(audit: dict) -> None:
|
||||
print(f'branch_targets_excluded: {audit["branch_targets_excluded_count"]}')
|
||||
print(f'observed_only_excluded: {audit["excluded_counts"].get("OBSERVED_PC_ONLY", 0)}')
|
||||
print(f'unknown_excluded: {audit["excluded_counts"].get("UNKNOWN", 0)}')
|
||||
print(f'guard_word_excluded: {len(audit.get("guard_word_excluded", ()))}')
|
||||
for addr in sorted(audit['included_reasons']):
|
||||
print(f' {addr:08X} {audit["included_reasons"][addr]}')
|
||||
for addr in sorted(audit['excluded_reasons']):
|
||||
reason = audit['excluded_reasons'][addr]
|
||||
if reason in ('BRANCH_TARGET_ONLY', 'OBSERVED_PC_ONLY', 'UNKNOWN'):
|
||||
if reason == 'GUARD_WORD':
|
||||
print(f' {addr:08X} excluded: GUARD_WORD (delay-slot guard '
|
||||
f'past analysable end 0x{audit["analysable_hi"]:08X}; '
|
||||
f'never requested)')
|
||||
elif reason in ('BRANCH_TARGET_ONLY', 'OBSERVED_PC_ONLY', 'UNKNOWN'):
|
||||
recovery = ('; isolated fragment demand retained'
|
||||
if addr in unhosted_dispatch else '')
|
||||
print(f' {addr:08X} excluded: {reason}{recovery}')
|
||||
@@ -4927,6 +4988,32 @@ def make_interior_fragment_job(phys_addr: int, load_addr: int, size: int,
|
||||
if phys_addr <= (a & 0x1FFFFFFF) < region_hi and
|
||||
segment_of(a) == segment_of(load_addr)
|
||||
}
|
||||
# Trailing delay-slot guard words in `data` (readable, not discoverable).
|
||||
# Stated by the capture writer where the field exists, reconstructed from
|
||||
# the page-run format otherwise. No demand of any kind may name one: the
|
||||
# recompiler's analysis ends before it and returns an empty manifest.
|
||||
guard_bytes = capture_guard_bytes(capture, size,
|
||||
f'region 0x{load_addr:08X}')
|
||||
analysable_hi = region_hi - guard_bytes
|
||||
guard_word_excluded = set()
|
||||
|
||||
def analysable(demands: set) -> set:
|
||||
kept = {a for a in demands if (a & 0x1FFFFFFF) < analysable_hi}
|
||||
guard_word_excluded.update(demands - kept)
|
||||
return kept
|
||||
|
||||
interiors = analysable(interiors)
|
||||
dispatch_roots = analysable(dispatch_roots)
|
||||
observed_dispatch = analysable(observed_dispatch)
|
||||
static_exact_demands = analysable(static_exact_demands)
|
||||
hosted_donor_demands = analysable(hosted_donor_demands)
|
||||
static_interval_demands = analysable(static_interval_demands)
|
||||
forced = analysable(forced)
|
||||
for addr in sorted(guard_word_excluded):
|
||||
print(f' fragment demand 0x{addr:08X} excluded: GUARD_WORD '
|
||||
f'(past analysable end 0x{analysable_hi | 0x80000000:08X} of '
|
||||
f'region 0x{phys_addr:08X}; never requested)')
|
||||
static_demands = static_exact_demands | static_interval_demands
|
||||
if not (((interiors or dispatch_roots) and executed) or observed_dispatch or
|
||||
static_demands or forced):
|
||||
return None
|
||||
@@ -4935,11 +5022,8 @@ def make_interior_fragment_job(phys_addr: int, load_addr: int, size: int,
|
||||
'load_addr': load_addr,
|
||||
'size': size,
|
||||
'data': data,
|
||||
# Trailing delay-slot guard words in `data` (readable, not
|
||||
# discoverable). Stated by the capture writer where the field exists,
|
||||
# reconstructed from the page-run format otherwise.
|
||||
'guard_bytes': capture_guard_bytes(
|
||||
capture, size, f'region 0x{load_addr:08X}'),
|
||||
'guard_bytes': guard_bytes,
|
||||
'guard_word_excluded': guard_word_excluded,
|
||||
'candidates': (interiors | dispatch_roots | observed_dispatch |
|
||||
static_demands | forced),
|
||||
'executed': executed,
|
||||
@@ -4991,6 +5075,7 @@ def merge_fragment_jobs_by_recipe(jobs: list[dict],
|
||||
'candidates', 'executed', 'static_demands',
|
||||
'static_exact_demands', 'hosted_donor_demands',
|
||||
'static_interval_demands', 'forced', 'guard_demoted_aliases',
|
||||
'guard_word_excluded',
|
||||
)
|
||||
for job in jobs:
|
||||
key = (
|
||||
@@ -5079,6 +5164,9 @@ def select_hosted_interior_demands(job: dict, donor_recipes: dict,
|
||||
donor_items = donor_items[:HOSTED_NOMINATION_CAP]
|
||||
lo = job['load_addr'] & 0x1FFFFFFF
|
||||
hi = lo + job['size']
|
||||
# A sibling recipe may hold as code the word that is this recipe's
|
||||
# delay-slot guard; this recipe cannot analyse it.
|
||||
analysable_hi = hi - job.get('guard_bytes', 0)
|
||||
local_reasons = {
|
||||
_canonical_guest_addr(addr): set(reasons)
|
||||
for addr, reasons in job.get('included_reasons', {}).items()
|
||||
@@ -5126,6 +5214,9 @@ def select_hosted_interior_demands(job: dict, donor_recipes: dict,
|
||||
if (target & 3) != 0 or not (lo <= target_phys < hi):
|
||||
rejected['target_outside'] += 1
|
||||
continue
|
||||
if target_phys >= analysable_hi:
|
||||
rejected['target_guard_word'] += 1
|
||||
continue
|
||||
target_producer = _producer_for_job_addr(job, target)
|
||||
if target_producer is None:
|
||||
rejected['target_no_producer'] += 1
|
||||
@@ -5587,6 +5678,17 @@ def compile_fragment_batch(requested_entries, data: bytes, load_addr: int,
|
||||
})
|
||||
if not requested:
|
||||
return None, 'no-requested-entries'
|
||||
# Every demand path filters the delay-slot guard word out before it gets
|
||||
# here (classify_overlay_seeds, make_interior_fragment_job, hosted
|
||||
# selection). One that does not is a bug, reported as a failure rather
|
||||
# than spent on a recompile that can only return an empty manifest.
|
||||
analysable_hi = (load_addr & 0x1FFFFFFF) + size - guard_bytes
|
||||
guard_requests = sorted(entry for entry in requested
|
||||
if (entry & 0x1FFFFFFF) >= analysable_hi)
|
||||
if guard_requests:
|
||||
return None, ('guard-word-request (BUG: demand past analysable end '
|
||||
f'0x{analysable_hi | 0x80000000:08X}): ' +
|
||||
', '.join(f'0x{entry:08X}' for entry in guard_requests))
|
||||
if hosted_owners is not None:
|
||||
hosted_specs = {
|
||||
_canonical_guest_addr(entry): spec
|
||||
@@ -5652,6 +5754,7 @@ def compile_fragment_batch(requested_entries, data: bytes, load_addr: int,
|
||||
if r.returncode != 0:
|
||||
return None, f'recompiler-error: {(r.stderr or r.stdout or "").strip()}'
|
||||
report_recompiler_guard_absorptions(r.stdout)
|
||||
report_recompiler_seed_drops(r.stdout)
|
||||
full_c = ranges_src = None
|
||||
for fn in os.listdir(out_dir_tmp):
|
||||
if fn.endswith('_full.c'):
|
||||
@@ -7153,7 +7256,20 @@ def _static_capture_job_in_segment(cap, args, toml, forced_interiors,
|
||||
result['outcome'] = 'fail'
|
||||
result['fail'] = (cls, detail)
|
||||
|
||||
# The delay-slot guard word is not code of this image and the recompiler
|
||||
# cannot analyse it (beads-eio.3.190): never a static demand either.
|
||||
analysable_hi = phys_addr + size - guard_bytes
|
||||
for captured_entry in sorted(
|
||||
_parse_addr_list(cap.get('dispatch_entry_pcs', [])) |
|
||||
{forced for forced in forced_interiors
|
||||
if phys_addr <= (forced & 0x1FFFFFFF) < phys_addr + size}):
|
||||
if (captured_entry & 0x1FFFFFFF) >= analysable_hi:
|
||||
print(f' demand 0x{captured_entry:08X} excluded: GUARD_WORD '
|
||||
f'(past analysable end 0x{analysable_hi | 0x80000000:08X}; '
|
||||
f'never requested)')
|
||||
for captured_entry in _parse_addr_list(cap.get('dispatch_entry_pcs', [])):
|
||||
if (captured_entry & 0x1FFFFFFF) >= analysable_hi:
|
||||
continue
|
||||
entry = _canonical_guest_addr(captured_entry)
|
||||
key = (entry, crc32)
|
||||
result['requested_entries'].add(key)
|
||||
@@ -7166,7 +7282,7 @@ def _static_capture_job_in_segment(cap, args, toml, forced_interiors,
|
||||
region_hi = phys_addr + size
|
||||
for forced_entry in forced_interiors:
|
||||
forced_phys = forced_entry & 0x1FFFFFFF
|
||||
if (phys_addr <= forced_phys < region_hi and
|
||||
if (phys_addr <= forced_phys < analysable_hi and
|
||||
segment_of(forced_entry) == segment_of(load_addr)):
|
||||
entry = forced_entry
|
||||
key = (entry, crc32)
|
||||
@@ -7229,6 +7345,7 @@ def _static_capture_job_in_segment(cap, args, toml, forced_interiors,
|
||||
fail('recompiler', r.stderr or r.stdout)
|
||||
return
|
||||
report_recompiler_guard_absorptions(r.stdout)
|
||||
report_recompiler_seed_drops(r.stdout)
|
||||
|
||||
stem = os.path.basename(psx_path)
|
||||
full_c = os.path.join(out_dir_tmp, stem + '_full.c')
|
||||
@@ -7901,6 +8018,7 @@ def main():
|
||||
stats.add_fail(_label, 'recompiler', r.stderr or r.stdout)
|
||||
return
|
||||
report_recompiler_guard_absorptions(r.stdout)
|
||||
report_recompiler_seed_drops(r.stdout)
|
||||
|
||||
# Find the generated _full.c
|
||||
stem = os.path.basename(psx_path)
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
#!/usr/bin/env python3
|
||||
"""No overlay demand may name a capture's trailing delay-slot guard word.
|
||||
|
||||
overlay_capture.c appends one guard word past a dirty-page run so a branch at
|
||||
the run's last word has its delay slot. The word is the first word of the
|
||||
NEXT page: readable, never code of this image, and the recompiler's analysis
|
||||
ends before it (PS-EXE analysis-bound tag).
|
||||
|
||||
Measured on Ace Combat 3 (bead beads-eio.3.190): two dispatch entries at
|
||||
load+size-4 (page-boundary resume points, 0x800D1000 and 0x800EA000) were
|
||||
classified DISPATCH_INTERIOR because a host walk falls through into the guard
|
||||
word, then requested as isolated fragments. The recompiler dropped the seed
|
||||
silently (past Analysis End) and returned an empty manifest: SHARD FAIL
|
||||
`fragment: no-func-ids`. The fix is on the request side, not a relabel:
|
||||
|
||||
1. classify_overlay_seeds excludes every kind of evidence in the guard word
|
||||
as GUARD_WORD (audited, counted, never a seed or a demand);
|
||||
2. make_interior_fragment_job, hosted selection and compile_fragment_batch
|
||||
hold the same invariant, so no path can request it;
|
||||
3. the recompiler warns when it drops a requested seed past Analysis End;
|
||||
4. the capture writer no longer records PCs in the guard word (pinned by
|
||||
runtime/tests/test_overlay_capture_guard_word.cpp).
|
||||
|
||||
Usage: test_overlay_guard_word_requests.py [--recompiler <exe>]
|
||||
"""
|
||||
import argparse
|
||||
import contextlib
|
||||
import io
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import compile_overlays as CO # noqa: E402
|
||||
|
||||
RECOMPILER = None
|
||||
|
||||
LOAD = 0x800D0000
|
||||
PAGE = 0x1000
|
||||
SIZE = PAGE + 4 # one dirty page plus the guard word
|
||||
GUARD = LOAD + PAGE # 0x800D1000: first word of the next page
|
||||
LAST = GUARD - 4 # last analysable word
|
||||
NOP = 0x00000000
|
||||
JR_RA = 0x03E00008
|
||||
FRAME = 0x27BDFFE0 # addiu sp,sp,-32
|
||||
ADDIU_S5 = 0x26B50014 # addiu s5,s5,0x14 (AC3 0x800D0FFC)
|
||||
ADDIU_V0 = 0x24420001 # addiu v0,v0,1
|
||||
|
||||
|
||||
def jal(target):
|
||||
return 0x0C000000 | ((target >> 2) & 0x03FFFFFF)
|
||||
|
||||
|
||||
def image(words: dict, size: int = SIZE, load: int = LOAD) -> bytes:
|
||||
data = bytearray(size)
|
||||
for addr, word in words.items():
|
||||
struct.pack_into('<I', data, addr - load, word)
|
||||
return bytes(data)
|
||||
|
||||
|
||||
def ac3_shape(extra=None) -> bytes:
|
||||
"""A host function whose straight-line body runs to the page end and
|
||||
falls through into the guard word, as at AC3 0x800CF000 / 0x800E8000."""
|
||||
words = {LOAD + 0xF00: FRAME}
|
||||
for addr in range(LOAD + 0xF04, GUARD, 4):
|
||||
words[addr] = ADDIU_V0
|
||||
words[LAST] = ADDIU_S5
|
||||
words[GUARD] = ADDIU_V0 # next page's code, captured as the guard
|
||||
words.update(extra or {})
|
||||
return image(words)
|
||||
|
||||
|
||||
def capture(**fields):
|
||||
cap = {'schema': 'psxrecomp overlay capture v2', 'guard_bytes': 4,
|
||||
'function_entry_pcs': [], 'dispatch_entry_pcs': []}
|
||||
for key, value in fields.items():
|
||||
cap[key] = ([f'0x{a:08X}' for a in value]
|
||||
if isinstance(value, (list, set, tuple)) else value)
|
||||
return cap
|
||||
|
||||
|
||||
def classify(data, toml=None, load=LOAD, **fields):
|
||||
return CO.classify_overlay_seeds(capture(**fields), data, load, len(data),
|
||||
0, toml or {}, root_enrichment=True)
|
||||
|
||||
|
||||
def seed_addrs(seeds):
|
||||
out = set()
|
||||
for seed in seeds:
|
||||
parts = seed.split()
|
||||
if parts and parts[0] not in ('producer_range', 'cross_call_allow'):
|
||||
out.add(int(parts[-1], 16))
|
||||
return out
|
||||
|
||||
|
||||
def quiet(fn, *a, **kw):
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
result = fn(*a, **kw)
|
||||
return result, buf.getvalue()
|
||||
|
||||
|
||||
class ClassifierTests(unittest.TestCase):
|
||||
HOST = LOAD + 0xF00
|
||||
|
||||
def assert_guard_excluded(self, seeds, audit, addr=GUARD):
|
||||
self.assertEqual(audit['excluded_reasons'].get(addr), 'GUARD_WORD')
|
||||
self.assertIn(addr, audit['guard_word_excluded'])
|
||||
self.assertNotIn(addr, audit['included_reasons'])
|
||||
self.assertNotIn(addr, seed_addrs(seeds))
|
||||
self.assertNotIn(addr, audit['dispatch_fragment_demands'])
|
||||
self.assertNotIn(addr, audit['static_exact_fragment_demands'])
|
||||
self.assertNotIn(addr, audit['static_interval_fragment_demands'])
|
||||
self.assertNotIn(addr, audit['interior_hosts'])
|
||||
self.assertEqual(audit['analysable_hi'], GUARD)
|
||||
|
||||
def test_ac3_dispatch_entry_in_guard_word_is_never_requested(self):
|
||||
data = ac3_shape()
|
||||
# The host walk really does visit the guard word by fallthrough:
|
||||
# that is what made it a hosted DISPATCH_INTERIOR before the fix.
|
||||
walk = CO._walk_overlay_function(data, LOAD, SIZE, self.HOST,
|
||||
LOAD + SIZE)
|
||||
self.assertIn(GUARD, walk['visited'])
|
||||
seeds, audit = classify(data, function_entry_pcs=[self.HOST],
|
||||
dispatch_entry_pcs=[self.HOST, GUARD],
|
||||
executed_pcs=[self.HOST, GUARD])
|
||||
self.assert_guard_excluded(seeds, audit)
|
||||
self.assertIn(self.HOST, seed_addrs(seeds))
|
||||
job, log = quiet(CO.make_interior_fragment_job,
|
||||
LOAD & 0x1FFFFFFF, LOAD, SIZE, data, audit, set(),
|
||||
capture())
|
||||
if job is not None:
|
||||
self.assertNotIn(GUARD, job['candidates'])
|
||||
self.assertNotIn(GUARD, CO.select_fragment_orphans(
|
||||
job['candidates'], job['executed'], job['forced'],
|
||||
job['static_exact_demands'], job['static_interval_demands'],
|
||||
set(), []))
|
||||
out = io.StringIO()
|
||||
with contextlib.redirect_stdout(out):
|
||||
CO.print_seed_audit(audit)
|
||||
self.assertIn('guard_word_excluded: 1', out.getvalue())
|
||||
self.assertIn(f'{GUARD:08X} excluded: GUARD_WORD', out.getvalue())
|
||||
|
||||
def test_same_pc_is_code_in_the_neighbour_variant(self):
|
||||
# 0x800D1000 is ordinary analysable code in the image loaded there,
|
||||
# so excluding it from the guard-word image loses no coverage.
|
||||
data = image({GUARD: FRAME, GUARD + 4: ADDIU_V0, GUARD + 8: JR_RA,
|
||||
GUARD + 12: NOP}, load=GUARD)
|
||||
seeds, audit = classify(data, load=GUARD, dispatch_entry_pcs=[GUARD],
|
||||
executed_pcs=[GUARD])
|
||||
self.assertIn(GUARD, seed_addrs(seeds))
|
||||
self.assertFalse(audit['guard_word_excluded'])
|
||||
|
||||
def test_last_analysable_word_is_unaffected(self):
|
||||
data = ac3_shape()
|
||||
_seeds, audit = classify(data, function_entry_pcs=[self.HOST],
|
||||
dispatch_entry_pcs=[LAST],
|
||||
executed_pcs=[LAST])
|
||||
self.assertEqual(audit['included_reasons'].get(LAST),
|
||||
'DISPATCH_INTERIOR')
|
||||
self.assertIn(LAST, audit['dispatch_fragment_demands'])
|
||||
self.assertFalse(audit['guard_word_excluded'])
|
||||
|
||||
def test_executed_only_pc_in_guard_word(self):
|
||||
seeds, audit = classify(ac3_shape(), function_entry_pcs=[self.HOST],
|
||||
executed_pcs=[self.HOST, GUARD])
|
||||
self.assert_guard_excluded(seeds, audit)
|
||||
|
||||
def test_every_other_evidence_source_in_guard_word(self):
|
||||
data = ac3_shape()
|
||||
cases = {
|
||||
'function_entry_pcs': dict(function_entry_pcs=[self.HOST, GUARD]),
|
||||
'static_discovery_entry_pcs': dict(
|
||||
function_entry_pcs=[self.HOST],
|
||||
static_discovery_entry_pcs=[GUARD]),
|
||||
'static_dispatch_entry_pcs': dict(
|
||||
function_entry_pcs=[self.HOST],
|
||||
dispatch_entry_pcs=[GUARD], static_dispatch_entry_pcs=[GUARD]),
|
||||
'legacy seeds': dict(function_entry_pcs=[self.HOST],
|
||||
seeds=[GUARD]),
|
||||
}
|
||||
for name, fields in cases.items():
|
||||
with self.subTest(name):
|
||||
seeds, audit = classify(data, **fields)
|
||||
self.assert_guard_excluded(seeds, audit)
|
||||
toml = {'overlays': [{'load_addr': f'0x{LOAD:08X}',
|
||||
'entries': [f'0x{GUARD:08X}']}]}
|
||||
seeds, audit = classify(data, toml=toml,
|
||||
function_entry_pcs=[self.HOST])
|
||||
self.assert_guard_excluded(seeds, audit)
|
||||
alias = capture(function_entry_pcs=[self.HOST])
|
||||
alias['static_alias_ranges'] = [{
|
||||
'entry': f'0x{GUARD:08X}', 'start': f'0x{self.HOST:08X}',
|
||||
'end': f'0x{LOAD + SIZE:08X}'}]
|
||||
seeds, audit = CO.classify_overlay_seeds(alias, data, LOAD, SIZE, 0,
|
||||
{}, root_enrichment=True)
|
||||
self.assert_guard_excluded(seeds, audit)
|
||||
|
||||
def test_call_into_guard_word_is_not_derived(self):
|
||||
caller = LOAD + 0x100
|
||||
data = ac3_shape({caller: FRAME, caller + 4: jal(GUARD),
|
||||
caller + 8: NOP, caller + 12: JR_RA,
|
||||
caller + 16: NOP})
|
||||
walk = CO._walk_overlay_function(data, LOAD, SIZE, caller,
|
||||
LOAD + SIZE)
|
||||
self.assertIn(GUARD, walk['direct_jals'])
|
||||
seeds, audit = classify(data, function_entry_pcs=[caller, self.HOST])
|
||||
self.assert_guard_excluded(seeds, audit)
|
||||
|
||||
def test_declared_zero_guard_keeps_the_last_word_analysable(self):
|
||||
# The exclusion follows the writer's declaration, not the size.
|
||||
data = ac3_shape({GUARD: JR_RA})
|
||||
cap = capture(function_entry_pcs=[self.HOST],
|
||||
dispatch_entry_pcs=[GUARD], executed_pcs=[GUARD])
|
||||
cap['guard_bytes'] = 0
|
||||
_seeds, audit = CO.classify_overlay_seeds(cap, data, LOAD, SIZE, 0,
|
||||
{}, root_enrichment=True)
|
||||
self.assertFalse(audit['guard_word_excluded'])
|
||||
self.assertNotEqual(audit['excluded_reasons'].get(GUARD),
|
||||
'GUARD_WORD')
|
||||
|
||||
|
||||
class RequestInvariantTests(unittest.TestCase):
|
||||
"""Paths that could still name the guard word hold the same rule."""
|
||||
|
||||
def audit(self, **sets):
|
||||
audit = {'included_reasons': {}, 'executed_pcs': set(),
|
||||
'dispatch_fragment_demands': set(),
|
||||
'static_exact_fragment_demands': set(),
|
||||
'cross_variant_hosted_demands': set(),
|
||||
'static_interval_fragment_demands': set(),
|
||||
'producer_ranges': [], 'accepted_cross_producer_calls': set()}
|
||||
audit.update(sets)
|
||||
return audit
|
||||
|
||||
def test_fragment_job_drops_forced_and_leaked_demands(self):
|
||||
data = ac3_shape()
|
||||
audit = self.audit(
|
||||
executed_pcs={LAST, GUARD},
|
||||
dispatch_fragment_demands={GUARD},
|
||||
static_exact_fragment_demands={GUARD, LAST},
|
||||
static_interval_fragment_demands={GUARD},
|
||||
cross_variant_hosted_demands={GUARD},
|
||||
included_reasons={GUARD: 'DISPATCH_INTERIOR',
|
||||
LAST: 'STATIC_DISCOVERY_ROOT'})
|
||||
job, log = quiet(CO.make_interior_fragment_job,
|
||||
LOAD & 0x1FFFFFFF, LOAD, SIZE, data, audit,
|
||||
{GUARD, LAST}, capture())
|
||||
for field in ('candidates', 'static_demands', 'static_exact_demands',
|
||||
'hosted_donor_demands', 'static_interval_demands',
|
||||
'forced'):
|
||||
self.assertNotIn(GUARD, job[field], field)
|
||||
self.assertIn(LAST, job['candidates'])
|
||||
self.assertIn(LAST, job['forced'])
|
||||
self.assertEqual(job['guard_word_excluded'], {GUARD})
|
||||
self.assertIn(f'fragment demand 0x{GUARD:08X} excluded: GUARD_WORD',
|
||||
log)
|
||||
self.assertEqual(CO.partition_strong_root_demands(
|
||||
job['static_exact_demands'], job['executed'], job['forced'],
|
||||
job['static_interval_demands'], set()), ([], [LAST]))
|
||||
|
||||
def test_hosted_selection_rejects_a_guard_word_target(self):
|
||||
job = {'load_addr': LOAD, 'size': SIZE, 'guard_bytes': 4,
|
||||
'included_reasons': {}, 'producer_ranges': ()}
|
||||
audit = {}
|
||||
selected = CO.select_hosted_interior_demands(
|
||||
job, {GUARD: True}, [], audit)
|
||||
self.assertEqual(selected, {})
|
||||
self.assertEqual(audit.get('target_guard_word'), 1)
|
||||
|
||||
def test_fragment_batch_refuses_a_guard_word_request(self):
|
||||
args = types.SimpleNamespace(recompiler=None, game_toml=None)
|
||||
frag_ids, status = CO.compile_fragment_batch(
|
||||
{GUARD}, ac3_shape(), LOAD, SIZE, LOAD & 0x1FFFFFFF, '', args,
|
||||
{}, {}, guard_bytes=4)
|
||||
self.assertIsNone(frag_ids)
|
||||
self.assertTrue(status.startswith('guard-word-request'), status)
|
||||
self.assertIn(f'0x{GUARD:08X}', status)
|
||||
|
||||
|
||||
class RecompilerWarningTests(unittest.TestCase):
|
||||
"""The recompiler states it dropped a requested seed past Analysis End."""
|
||||
|
||||
def run_recompiler(self, entry, seed_lines):
|
||||
data = ac3_shape({LAST: JR_RA, GUARD: NOP})
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
psx = Path(tmp) / 'frag.psx'
|
||||
psx.write_bytes(CO.make_psxexe(LOAD, entry, data, guard_bytes=4))
|
||||
seeds = Path(tmp) / 'seeds.txt'
|
||||
seeds.write_text(''.join(line + '\n' for line in seed_lines))
|
||||
out = Path(tmp) / 'out'
|
||||
out.mkdir()
|
||||
return subprocess.run(
|
||||
[RECOMPILER, str(psx), '--seeds', str(seeds),
|
||||
'--out-dir', str(out), '--overlay'],
|
||||
capture_output=True, text=True, cwd=str(ROOT))
|
||||
|
||||
def setUp(self):
|
||||
if not RECOMPILER:
|
||||
self.skipTest('pass --recompiler (ctest always does)')
|
||||
|
||||
def test_seed_past_analysis_end_is_reported(self):
|
||||
host = LOAD + 0xF00
|
||||
proc = self.run_recompiler(host, [f'dispatch_root 0x{host:08X}',
|
||||
f'dispatch_root 0x{GUARD:08X}',
|
||||
f'interior 0x{GUARD:08X}'])
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr or proc.stdout)
|
||||
self.assertIn(
|
||||
f'WARNING: dropped 2 requested seed(s) at or past Analysis End '
|
||||
f'0x{GUARD:08X}', proc.stdout)
|
||||
self.assertNotIn('WARNING: entry point', proc.stdout)
|
||||
# compile_overlays copies the warning into the compile log.
|
||||
count, log = quiet(CO.report_recompiler_seed_drops, proc.stdout)
|
||||
self.assertEqual(count, 1)
|
||||
self.assertIn(f'recompiler WARNING: dropped 2 requested seed(s)', log)
|
||||
|
||||
def test_entry_point_past_analysis_end_is_reported(self):
|
||||
proc = self.run_recompiler(GUARD, [f'dispatch_root 0x{GUARD:08X}'])
|
||||
self.assertIn(f'WARNING: entry point 0x{GUARD:08X} is at or past '
|
||||
f'Analysis End 0x{GUARD:08X}', proc.stdout)
|
||||
self.assertIn('WARNING: dropped 1 requested seed(s) at or past '
|
||||
'Analysis End', proc.stdout)
|
||||
|
||||
def test_in_range_seeds_warn_nothing(self):
|
||||
host = LOAD + 0xF00
|
||||
proc = self.run_recompiler(host, [f'dispatch_root 0x{host:08X}',
|
||||
f'interior 0x{LAST:08X}'])
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr or proc.stdout)
|
||||
self.assertNotIn('WARNING: dropped', proc.stdout)
|
||||
self.assertNotIn('WARNING: entry point', proc.stdout)
|
||||
|
||||
|
||||
class StaticModeTests(unittest.TestCase):
|
||||
"""--static binds captured dispatch entries to demands before the
|
||||
classifier runs; the guard word must not become one there either."""
|
||||
|
||||
def setUp(self):
|
||||
if not RECOMPILER:
|
||||
self.skipTest('pass --recompiler (ctest always does)')
|
||||
|
||||
def test_static_mode_never_demands_the_guard_word(self):
|
||||
host = LOAD + 0xF00
|
||||
data = ac3_shape({LAST: JR_RA, GUARD: NOP})
|
||||
rec = capture(function_entry_pcs=[host],
|
||||
dispatch_entry_pcs=[host, GUARD],
|
||||
executed_pcs=[host, GUARD])
|
||||
rec.update({'load_addr': f'0x{LOAD:08X}', 'size': SIZE,
|
||||
'bytes_b64': __import__('base64').b64encode(data).decode()})
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp = Path(tmp)
|
||||
(tmp / 'captures.json').write_text(__import__('json').dumps([rec]))
|
||||
header = bytearray(0x800)
|
||||
header[:8] = b'PS-X EXE'
|
||||
struct.pack_into('<I', header, 0x18, 0x80010000)
|
||||
struct.pack_into('<I', header, 0x1C, 0x10)
|
||||
(tmp / 'MAIN.EXE').write_bytes(bytes(header) + bytes(0x10))
|
||||
(tmp / 'game.toml').write_text(
|
||||
'[game]\nid = "TEST-00000"\nname = "x"\nexe = "MAIN.EXE"\n'
|
||||
'load_address = "0x80010000"\nentry_pc = "0x80010000"\n'
|
||||
'text_size = "0x10"\nstack_base = "0x801FFFF0"\n\n'
|
||||
'[recompiler]\nseeds = "seeds.txt"\nout_dir = "generated"\n')
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(ROOT / 'tools' / 'compile_overlays.py'),
|
||||
'--static', '--captures', str(tmp / 'captures.json'),
|
||||
'--out-dir', str(tmp / 'out'),
|
||||
'--game-toml', str(tmp / 'game.toml'),
|
||||
'--recompiler', RECOMPILER, '--project-root', str(ROOT),
|
||||
'--runtime-include', str(ROOT / 'runtime' / 'include'),
|
||||
'--force-interior', f'0x{GUARD:08X}'],
|
||||
capture_output=True, text=True, timeout=300, cwd=str(ROOT))
|
||||
out = proc.stdout + proc.stderr
|
||||
self.assertEqual(proc.returncode, 0, out[-3000:])
|
||||
self.assertIn(f'demand 0x{GUARD:08X} excluded: GUARD_WORD', out)
|
||||
self.assertNotIn('recompiler WARNING', out)
|
||||
self.assertNotIn('no-func-ids', out)
|
||||
self.assertNotIn('SHARD FAIL', out)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--recompiler')
|
||||
args, rest = ap.parse_known_args()
|
||||
RECOMPILER = args.recompiler
|
||||
unittest.main(argv=[sys.argv[0]] + rest)
|
||||
Reference in New Issue
Block a user