mirror of
https://github.com/jessicanataliagta/PSPRecomp
synced 2026-09-26 08:41:08 -04:00
Jessica_Natalia
1378 lines
71 KiB
C++
1378 lines
71 KiB
C++
#include "psprecomp/runtime.hpp"
|
|
#include "psprecomp/common.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <chrono>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <iomanip>
|
|
#include <iostream>
|
|
#include <sstream>
|
|
#include <unordered_map>
|
|
#include <vector>
|
|
|
|
namespace psprecomp {
|
|
|
|
// See Runtime::invoke_chained_direct(). Diagnostics deliberately make this
|
|
// sticky: once instrumentation has been requested, correctness matters more
|
|
// than returning to the production fast path later in the process.
|
|
bool g_runtime_chain_observers_active = false;
|
|
std::uint64_t g_runtime_starvation_interval_fast = 0u;
|
|
std::uint64_t g_runtime_thread_switch_generation_fast = 0u;
|
|
|
|
namespace {
|
|
using RuntimePostImportHook = void (*)(Runtime &, AllegrexContext &);
|
|
RuntimePostImportHook g_post_import_hook = nullptr;
|
|
std::int32_t g_runtime_thread_uid = -1;
|
|
std::array<char, 64> g_runtime_thread_name{};
|
|
std::uint32_t g_runtime_dispatch_pc = 0u;
|
|
RuntimeHeartbeatHook g_heartbeat_hook = nullptr;
|
|
std::uint64_t g_heartbeat_interval = 0u;
|
|
RuntimeStarvationHook g_starvation_hook = nullptr;
|
|
std::uint64_t g_starvation_interval = 0u;
|
|
RuntimePreDispatchHook g_pre_dispatch_hook = nullptr;
|
|
RuntimePostDispatchHook g_post_dispatch_hook = nullptr;
|
|
RuntimePreChainedCallHook g_pre_chained_call_hook = nullptr;
|
|
RuntimePostChainedCallHook g_post_chained_call_hook = nullptr;
|
|
}
|
|
|
|
// Execution counter for a handful of guest addresses, armed by
|
|
// PSPRECOMP_COUNT_PC as a comma-separated list.
|
|
//
|
|
// Answering "is this routine ever reached?" needed an instrument that costs
|
|
// nothing: the existing chain tracer pushes a frame on every chained call and
|
|
// perturbed the run enough that it died before reaching the window under
|
|
// investigation. This is a linear scan over at most eight addresses, and it
|
|
// covers both entry paths -- the outer dispatch loop and invoke_chained_call --
|
|
// because a routine reached only through chaining never appears in the outer
|
|
// dispatch profile.
|
|
namespace {
|
|
constexpr std::size_t kMaxCountedPcs = 8u;
|
|
std::array<std::uint32_t, kMaxCountedPcs> g_counted_pcs{};
|
|
std::array<std::uint64_t, kMaxCountedPcs> g_counted_pc_hits{};
|
|
std::size_t g_counted_pc_size = 0u;
|
|
bool g_counted_pcs_loaded = false;
|
|
|
|
void load_counted_pcs() {
|
|
if (g_counted_pcs_loaded) return;
|
|
g_counted_pcs_loaded = true;
|
|
const char *text = std::getenv("PSPRECOMP_COUNT_PC");
|
|
if (text == nullptr || *text == '\0') return;
|
|
const std::string list(text);
|
|
std::size_t begin = 0u;
|
|
while (begin < list.size() && g_counted_pc_size < kMaxCountedPcs) {
|
|
const std::size_t comma = list.find(',', begin);
|
|
const std::string item = list.substr(begin, comma == std::string::npos ? std::string::npos : comma - begin);
|
|
char *end = nullptr;
|
|
const unsigned long value = std::strtoul(item.c_str(), &end, 0);
|
|
if (end != item.c_str() && *end == '\0') g_counted_pcs[g_counted_pc_size++] = static_cast<std::uint32_t>(value);
|
|
if (comma == std::string::npos) break;
|
|
begin = comma + 1u;
|
|
}
|
|
if (g_counted_pc_size != 0u) g_runtime_chain_observers_active = true;
|
|
}
|
|
|
|
inline void count_pc(std::uint32_t pc) noexcept {
|
|
for (std::size_t index = 0u; index < g_counted_pc_size; ++index)
|
|
if (g_counted_pcs[index] == pc) { ++g_counted_pc_hits[index]; return; }
|
|
}
|
|
}
|
|
|
|
void report_counted_pcs() {
|
|
for (std::size_t index = 0u; index < g_counted_pc_size; ++index)
|
|
std::cerr << "[count-pc] pc=" << hex32(g_counted_pcs[index])
|
|
<< " hits=" << g_counted_pc_hits[index] << "\n";
|
|
}
|
|
|
|
void set_runtime_heartbeat_hook(RuntimeHeartbeatHook hook, std::uint64_t interval) noexcept {
|
|
g_heartbeat_hook = interval != 0u ? hook : nullptr;
|
|
g_heartbeat_interval = interval;
|
|
}
|
|
|
|
void set_runtime_starvation_hook(RuntimeStarvationHook hook, std::uint64_t interval) noexcept {
|
|
g_starvation_hook = interval != 0u ? hook : nullptr;
|
|
g_starvation_interval = interval;
|
|
g_runtime_starvation_interval_fast = g_starvation_hook != nullptr ? interval : 0u;
|
|
}
|
|
|
|
void set_runtime_pre_dispatch_hook(RuntimePreDispatchHook hook) noexcept {
|
|
g_pre_dispatch_hook = hook;
|
|
}
|
|
void set_runtime_post_dispatch_hook(RuntimePostDispatchHook hook) noexcept {
|
|
g_post_dispatch_hook = hook;
|
|
}
|
|
void set_runtime_pre_chained_call_hook(RuntimePreChainedCallHook hook) noexcept {
|
|
g_pre_chained_call_hook = hook;
|
|
if (hook != nullptr) g_runtime_chain_observers_active = true;
|
|
}
|
|
void set_runtime_post_chained_call_hook(RuntimePostChainedCallHook hook) noexcept {
|
|
g_post_chained_call_hook = hook;
|
|
if (hook != nullptr) g_runtime_chain_observers_active = true;
|
|
}
|
|
|
|
void set_runtime_post_import_hook(RuntimePostImportHook hook) noexcept { g_post_import_hook = hook; }
|
|
void set_runtime_thread_identity(std::int32_t uid, const std::string &name) noexcept {
|
|
if (uid != g_runtime_thread_uid) ++g_runtime_thread_switch_generation_fast;
|
|
g_runtime_thread_uid = uid;
|
|
g_runtime_thread_name.fill('\0');
|
|
const std::size_t count = std::min(name.size(), g_runtime_thread_name.size() - 1u);
|
|
std::memcpy(g_runtime_thread_name.data(), name.data(), count);
|
|
}
|
|
std::int32_t runtime_thread_uid() noexcept { return g_runtime_thread_uid; }
|
|
const char *runtime_thread_name() noexcept { return g_runtime_thread_name.data(); }
|
|
std::uint32_t runtime_dispatch_pc() noexcept { return g_runtime_dispatch_pc; }
|
|
RuntimeExecutionContextToken capture_runtime_execution_context() noexcept {
|
|
return RuntimeExecutionContextToken{g_runtime_thread_uid, g_runtime_thread_switch_generation_fast};
|
|
}
|
|
bool runtime_execution_context_matches(RuntimeExecutionContextToken token) noexcept {
|
|
return token.thread_uid == g_runtime_thread_uid &&
|
|
token.switch_generation == g_runtime_thread_switch_generation_fast;
|
|
}
|
|
std::uint64_t runtime_thread_switch_generation() noexcept {
|
|
return g_runtime_thread_switch_generation_fast;
|
|
}
|
|
bool runtime_thread_switch_generation_matches(std::uint64_t generation) noexcept {
|
|
return generation == g_runtime_thread_switch_generation_fast;
|
|
}
|
|
|
|
bool g_guest_hotspot_profile_enabled = false;
|
|
std::uint32_t g_guest_hotspot_sample_mask = 0xFFu;
|
|
std::uint64_t g_guest_hotspot_unit_calls[kUnitProfileCapacity]{};
|
|
namespace {
|
|
struct GuestHotspotPcSlot {
|
|
std::uint32_t pc{};
|
|
std::uint16_t unit{};
|
|
std::uint16_t occupied{};
|
|
std::uint64_t samples{};
|
|
std::uint64_t inclusive_sample_ns{};
|
|
};
|
|
std::uint64_t g_guest_hotspot_unit_samples[kUnitProfileCapacity]{};
|
|
std::uint64_t g_guest_hotspot_unit_ns[kUnitProfileCapacity]{};
|
|
std::array<GuestHotspotPcSlot, kGuestHotspotPcCapacity> g_guest_hotspot_pc_slots{};
|
|
std::uint64_t g_guest_hotspot_total_samples = 0u;
|
|
|
|
std::size_t guest_hotspot_hash(std::uint32_t pc) noexcept {
|
|
std::uint32_t x = pc >> 2u;
|
|
x ^= x >> 16u;
|
|
x *= 0x7FEB352Du;
|
|
x ^= x >> 15u;
|
|
return static_cast<std::size_t>(x) & (kGuestHotspotPcCapacity - 1u);
|
|
}
|
|
}
|
|
|
|
void set_guest_hotspot_profile(bool enabled, std::uint32_t sample_shift) noexcept {
|
|
if (sample_shift > 16u) sample_shift = 16u;
|
|
g_guest_hotspot_sample_mask = sample_shift == 0u ? 0u : ((1u << sample_shift) - 1u);
|
|
g_guest_hotspot_profile_enabled = enabled;
|
|
}
|
|
|
|
std::uint64_t guest_hotspot_clock_ns() noexcept {
|
|
return static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(
|
|
std::chrono::steady_clock::now().time_since_epoch()).count());
|
|
}
|
|
|
|
void guest_hotspot_record_sample(std::uint32_t unit, std::uint32_t pc,
|
|
std::uint64_t elapsed_ns) noexcept {
|
|
if (!g_guest_hotspot_profile_enabled || unit >= kUnitProfileCapacity) return;
|
|
++g_guest_hotspot_unit_samples[unit];
|
|
g_guest_hotspot_unit_ns[unit] += elapsed_ns;
|
|
++g_guest_hotspot_total_samples;
|
|
if (pc == 0u) return;
|
|
|
|
std::size_t slot = guest_hotspot_hash(pc);
|
|
for (std::size_t probe = 0u; probe < 12u; ++probe) {
|
|
GuestHotspotPcSlot &entry = g_guest_hotspot_pc_slots[(slot + probe) & (kGuestHotspotPcCapacity - 1u)];
|
|
if (entry.occupied == 0u) {
|
|
entry.pc = pc;
|
|
entry.unit = static_cast<std::uint16_t>(unit);
|
|
entry.occupied = 1u;
|
|
}
|
|
if (entry.pc == pc && entry.unit == unit) {
|
|
++entry.samples;
|
|
entry.inclusive_sample_ns += elapsed_ns;
|
|
return;
|
|
}
|
|
}
|
|
// The table is intentionally fixed and allocation-free. Extremely unlikely
|
|
// collision overflow drops only this sampled PC; unit totals remain exact.
|
|
}
|
|
|
|
GuestHotspotSnapshot consume_guest_hotspot_profile(std::size_t unit_limit, std::size_t pc_limit) {
|
|
GuestHotspotSnapshot out;
|
|
out.sample_stride = g_guest_hotspot_sample_mask + 1u;
|
|
out.total_samples = g_guest_hotspot_total_samples;
|
|
for (std::size_t unit = 0u; unit < kUnitProfileCapacity; ++unit) {
|
|
const std::uint64_t calls = g_guest_hotspot_unit_calls[unit];
|
|
const std::uint64_t samples = g_guest_hotspot_unit_samples[unit];
|
|
const std::uint64_t ns = g_guest_hotspot_unit_ns[unit];
|
|
out.total_unit_calls += calls;
|
|
if (calls != 0u || samples != 0u)
|
|
out.units.push_back(GuestHotspotUnitEntry{static_cast<std::uint32_t>(unit), calls, samples, ns});
|
|
g_guest_hotspot_unit_calls[unit] = 0u;
|
|
g_guest_hotspot_unit_samples[unit] = 0u;
|
|
g_guest_hotspot_unit_ns[unit] = 0u;
|
|
}
|
|
for (GuestHotspotPcSlot &slot : g_guest_hotspot_pc_slots) {
|
|
if (slot.occupied != 0u && slot.samples != 0u)
|
|
out.pcs.push_back(GuestHotspotPcEntry{slot.unit, slot.pc, slot.samples, slot.inclusive_sample_ns});
|
|
slot = {};
|
|
}
|
|
g_guest_hotspot_total_samples = 0u;
|
|
|
|
const auto estimated_cost = [stride = static_cast<std::uint64_t>(out.sample_stride)](const auto &e) {
|
|
return e.samples == 0u ? 0u : e.inclusive_sample_ns * stride;
|
|
};
|
|
std::sort(out.units.begin(), out.units.end(), [&](const auto &a, const auto &b) {
|
|
const auto ac = estimated_cost(a), bc = estimated_cost(b);
|
|
if (ac != bc) return ac > bc;
|
|
return a.calls > b.calls;
|
|
});
|
|
std::sort(out.pcs.begin(), out.pcs.end(), [&](const auto &a, const auto &b) {
|
|
const auto ac = estimated_cost(a), bc = estimated_cost(b);
|
|
if (ac != bc) return ac > bc;
|
|
return a.samples > b.samples;
|
|
});
|
|
if (out.units.size() > unit_limit) out.units.resize(unit_limit);
|
|
if (out.pcs.size() > pc_limit) out.pcs.resize(pc_limit);
|
|
return out;
|
|
}
|
|
|
|
Runtime::Runtime(std::uint32_t ram_size) : memory_(ram_size) {
|
|
// Most commercial PSP titles use a few hundred import stubs. Seed a small
|
|
// binding table so first use of a late-numbered import does not reallocate
|
|
// in the middle of guest execution; larger profiles can still grow it.
|
|
import_bindings_.resize(256u, nullptr);
|
|
hle_histogram_enabled_ = std::getenv("PSPRECOMP_HLE_HISTOGRAM") != nullptr;
|
|
g_unit_profile_enabled = std::getenv("PSPRECOMP_UNIT_PROFILE") != nullptr;
|
|
#if defined(PSPRECOMP_AOT_PRODUCTION_FASTPATHS)
|
|
track_dispatch_counters_ = false;
|
|
#else
|
|
track_dispatch_counters_ =
|
|
std::getenv("PSPRECOMP_REPORT_DISPATCH_COUNT") != nullptr ||
|
|
std::getenv("PSPRECOMP_PROFILE_DISPATCH") != nullptr ||
|
|
std::getenv("PSPRECOMP_COUNT_PC") != nullptr;
|
|
#endif
|
|
// PSPRECOMP_NO_CHAIN disables cross-unit chaining outright;
|
|
// PSPRECOMP_CHAIN_DEPTH tunes how deep it may nest without a rebuild.
|
|
//
|
|
// Do not raise this without measuring native stack use per chained frame.
|
|
// 1024 was tried on the theory that exceeding the limit is a cliff -- the
|
|
// chain refuses, the native stack unwinds to Runtime::run, and the target is
|
|
// re-entered through the guest-PC table -- and that bench.bat already passed
|
|
// 1024 by hand. It crashed the game during boot. A benchmark does not reach
|
|
// the call depths gameplay does, and generated frames are not small: each
|
|
// one carries the unit's locals, and inlining the memory fast path at every
|
|
// call site (/Ob1) made them larger still. 48 is the value that runs.
|
|
chain_depth_limit_ = 48u;
|
|
if (const char *depth = std::getenv("PSPRECOMP_CHAIN_DEPTH")) {
|
|
char *end = nullptr;
|
|
const unsigned long value = std::strtoul(depth, &end, 0);
|
|
if (end != depth && *end == '\0' && value <= 4096u)
|
|
chain_depth_limit_ = static_cast<std::uint32_t>(value);
|
|
}
|
|
if (std::getenv("PSPRECOMP_NO_CHAIN") != nullptr) chain_depth_limit_ = 0u;
|
|
}
|
|
|
|
namespace {
|
|
// The six registers the dispatch error messages print are rarely enough to tell
|
|
// which operand produced a bad guest address. Note that the hot-register cache
|
|
// (r2/r4-r7/r29/r31) is only materialized back into AllegrexContext by the
|
|
// generated outer wrapper on a normal return -- an exception unwinds past that
|
|
// flush, so exactly those six read stale here. Every other register, including
|
|
// the callee-saved ones, is current.
|
|
void append_gpr_dump(std::ostringstream &message, const AllegrexContext &ctx) {
|
|
static const char *const kGprNames[32] = {
|
|
"zero", "at", "v0", "v1", "a0", "a1", "a2", "a3",
|
|
"t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7",
|
|
"s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7",
|
|
"t8", "t9", "k0", "k1", "gp", "sp", "fp", "ra"};
|
|
message << "\n gpr:";
|
|
for (std::size_t index = 0u; index < 32u; ++index) {
|
|
if (index % 8u == 0u) message << "\n ";
|
|
message << ' ' << kGprNames[index] << '=' << hex32(ctx.gpr[index]);
|
|
}
|
|
message << "\n (r2/r4-r7/r29/r31 may be stale: hot-register cache not flushed on unwind)";
|
|
}
|
|
}
|
|
|
|
bool Runtime::run_starvation_boundary(AllegrexContext &ctx) {
|
|
const std::uint64_t interval = g_runtime_starvation_interval_fast;
|
|
if (g_starvation_hook == nullptr || interval == 0u) return true;
|
|
// Preserve excess work if a context switch deferred several boundaries.
|
|
dispatches_since_import_ -= std::min(dispatches_since_import_, interval);
|
|
const RuntimeExecutionContextToken before = capture_runtime_execution_context();
|
|
g_starvation_hook(*this, ctx);
|
|
const bool same_context = runtime_execution_context_matches(before);
|
|
if (!same_context) chain_context_invalidated_ = true;
|
|
return same_context;
|
|
}
|
|
|
|
bool Runtime::account_dispatch_work(AllegrexContext &ctx, bool allow_preemption) {
|
|
#if !defined(PSPRECOMP_AOT_PRODUCTION_FASTPATHS)
|
|
if (track_dispatch_counters_) ++dispatch_work_count_;
|
|
#endif
|
|
const std::uint64_t interval = g_runtime_starvation_interval_fast;
|
|
if (interval == 0u) return true;
|
|
++dispatches_since_import_;
|
|
if (!allow_preemption || dispatches_since_import_ < interval) return true;
|
|
return run_starvation_boundary(ctx);
|
|
}
|
|
|
|
bool Runtime::invoke_chained_call(AllegrexContext &ctx, GuestMemory::AotFastView *shared_aot_mem) {
|
|
#if !defined(PSPRECOMP_AOT_PRODUCTION_FASTPATHS)
|
|
count_pc(ctx.pc);
|
|
#endif
|
|
if (chain_depth_ >= chain_depth_limit_) return false;
|
|
|
|
// Indirect jalr/vcall targets usually land in ordinary generated code too.
|
|
// Prefer the tiny dense unit table. Units overlapped by a host/import
|
|
// replacement are poisoned and fall back to exact PC lookup. Ownership is
|
|
// still guarded on every native chain frame because a nested descendant
|
|
// can cross the scheduler boundary even when this immediate unit is clean.
|
|
// dynamic JR/JALR chains used to canonicalize the same target
|
|
// twice: once in lookup_generated_unit() and again to locate the compact
|
|
// direct-entry function. Do the dense generated-unit probe once and reuse
|
|
// its unit index for both function and entry lookup.
|
|
RecompiledFunction function = nullptr;
|
|
RecompiledEntryFunction entry_function = nullptr;
|
|
if (generated_unit_layout_valid_ && generated_unit_span_ != 0u) {
|
|
const std::uint32_t canonical_pc = memory_.canonical(ctx.pc);
|
|
if (canonical_pc >= generated_unit_base_) {
|
|
const std::uint32_t unit_delta = canonical_pc - generated_unit_base_;
|
|
const std::uint32_t unit_index = generated_unit_span_ == 16384u
|
|
? (unit_delta >> 14u) : (unit_delta / generated_unit_span_);
|
|
if (unit_index < kGeneratedUnitFastCapacity) {
|
|
function = generated_units_[unit_index];
|
|
if (function != nullptr && shared_aot_mem != nullptr &&
|
|
generated_unit_disabled_[unit_index] == 0u)
|
|
entry_function = generated_unit_entries_[unit_index];
|
|
}
|
|
}
|
|
}
|
|
if (function == nullptr) {
|
|
const std::uint32_t delta = memory_.canonical(ctx.pc) - direct_base_;
|
|
if ((delta & 3u) != 0u) return false;
|
|
const std::size_t index = static_cast<std::size_t>(delta) / 4u;
|
|
if (index >= direct_chainable_.size()) return false;
|
|
function = direct_chainable_[index];
|
|
if (function == nullptr) return false;
|
|
}
|
|
|
|
const std::uint32_t target_pc = ctx.pc;
|
|
std::uint32_t guest_hotspot_unit = static_cast<std::uint32_t>(kUnitProfileCapacity);
|
|
bool guest_hotspot_sample = false;
|
|
std::uint64_t guest_hotspot_start_ns = 0u;
|
|
if (g_guest_hotspot_profile_enabled && generated_unit_layout_valid_ && generated_unit_span_ != 0u) {
|
|
const std::uint32_t canonical_pc = memory_.canonical(target_pc);
|
|
if (canonical_pc >= generated_unit_base_) {
|
|
const std::uint32_t delta = canonical_pc - generated_unit_base_;
|
|
const std::uint32_t unit_index = generated_unit_span_ == 16384u
|
|
? (delta >> 14u) : (delta / generated_unit_span_);
|
|
if (unit_index < kUnitProfileCapacity) {
|
|
guest_hotspot_unit = unit_index;
|
|
++g_guest_hotspot_unit_calls[unit_index];
|
|
const std::uint64_t ticket = ++guest_hotspot_ticket_;
|
|
guest_hotspot_sample =
|
|
(ticket & static_cast<std::uint64_t>(g_guest_hotspot_sample_mask)) == 0u;
|
|
if (guest_hotspot_sample) guest_hotspot_start_ns = guest_hotspot_clock_ns();
|
|
}
|
|
}
|
|
}
|
|
const std::uint32_t native_depth = chain_depth_;
|
|
// Always guard execution-context ownership. Even a clean generated unit can
|
|
// reach a nested direct chain whose scheduler boundary switches PSP thread.
|
|
// guarded every native chain frame; accidentally weakened
|
|
// that invariant for the dense generated-unit path.
|
|
// Generation alone is sufficient: it increments on every PSP thread
|
|
// ownership change. Avoid constructing/checking a two-field token on every
|
|
// dynamic native chain boundary in the city hot path.
|
|
const std::uint64_t caller_generation = g_runtime_thread_switch_generation_fast;
|
|
#if !defined(PSPRECOMP_AOT_PRODUCTION_FASTPATHS)
|
|
if (g_pre_chained_call_hook != nullptr)
|
|
g_pre_chained_call_hook(*this, ctx, target_pc, native_depth);
|
|
#endif
|
|
struct DepthGuard {
|
|
std::uint32_t &depth;
|
|
explicit DepthGuard(std::uint32_t &value) : depth(value) { ++depth; }
|
|
~DepthGuard() { --depth; }
|
|
} guard(chain_depth_);
|
|
if (entry_function != nullptr && shared_aot_mem != nullptr) {
|
|
entry_function(*this, ctx, 0u, *shared_aot_mem);
|
|
} else {
|
|
function(*this, ctx);
|
|
}
|
|
if (guest_hotspot_sample) {
|
|
const std::uint64_t end_ns = guest_hotspot_clock_ns();
|
|
guest_hotspot_record_sample(guest_hotspot_unit, target_pc,
|
|
end_ns >= guest_hotspot_start_ns
|
|
? end_ns - guest_hotspot_start_ns : 0u);
|
|
}
|
|
#if !defined(PSPRECOMP_AOT_PRODUCTION_FASTPATHS)
|
|
if (g_post_chained_call_hook != nullptr)
|
|
g_post_chained_call_hook(*this, ctx, target_pc, native_depth);
|
|
if (track_dispatch_counters_) ++chained_dispatches_;
|
|
#endif
|
|
|
|
// Dynamic targets may enter a profile/native function which switches PSP
|
|
// ownership without passing through run_starvation_boundary(). Keep the
|
|
// generation guard here even in production; compile-time direct chains use
|
|
// chain_context_invalidated_ and retain their cheaper hot path.
|
|
if (caller_generation != g_runtime_thread_switch_generation_fast) {
|
|
(void)account_dispatch_work(ctx, false);
|
|
return false;
|
|
}
|
|
return account_dispatch_work(ctx, true);
|
|
}
|
|
|
|
bool Runtime::invoke_chained_unit(AllegrexContext &ctx, std::uint32_t unit_index,
|
|
GuestMemory::AotFastView *shared_aot_mem) {
|
|
#if !defined(PSPRECOMP_AOT_PRODUCTION_FASTPATHS)
|
|
count_pc(ctx.pc);
|
|
#endif
|
|
if (chain_depth_ >= chain_depth_limit_) return false;
|
|
if (unit_index >= kGeneratedUnitFastCapacity || !generated_unit_layout_valid_) return false;
|
|
RecompiledFunction function = generated_units_[unit_index];
|
|
if (function == nullptr) return false;
|
|
if (g_unit_profile_enabled) ++g_unit_profile_counts[unit_index];
|
|
|
|
const std::uint32_t target_pc = ctx.pc;
|
|
bool guest_hotspot_sample = false;
|
|
std::uint64_t guest_hotspot_start_ns = 0u;
|
|
if (g_guest_hotspot_profile_enabled) {
|
|
++g_guest_hotspot_unit_calls[unit_index];
|
|
const std::uint64_t ticket = ++guest_hotspot_ticket_;
|
|
guest_hotspot_sample =
|
|
(ticket & static_cast<std::uint64_t>(g_guest_hotspot_sample_mask)) == 0u;
|
|
if (guest_hotspot_sample) guest_hotspot_start_ns = guest_hotspot_clock_ns();
|
|
}
|
|
const std::uint32_t native_depth = chain_depth_;
|
|
const std::uint64_t caller_generation = g_runtime_thread_switch_generation_fast;
|
|
#if !defined(PSPRECOMP_AOT_PRODUCTION_FASTPATHS)
|
|
if (g_pre_chained_call_hook != nullptr)
|
|
g_pre_chained_call_hook(*this, ctx, target_pc, native_depth);
|
|
#endif
|
|
struct DepthGuard {
|
|
std::uint32_t &depth;
|
|
explicit DepthGuard(std::uint32_t &value) : depth(value) { ++depth; }
|
|
~DepthGuard() { --depth; }
|
|
} guard(chain_depth_);
|
|
if (shared_aot_mem != nullptr && generated_unit_entries_[unit_index] != nullptr) {
|
|
generated_unit_entries_[unit_index](*this, ctx, 0u, *shared_aot_mem);
|
|
} else {
|
|
function(*this, ctx);
|
|
}
|
|
if (guest_hotspot_sample) {
|
|
const std::uint64_t end_ns = guest_hotspot_clock_ns();
|
|
guest_hotspot_record_sample(unit_index, target_pc,
|
|
end_ns >= guest_hotspot_start_ns
|
|
? end_ns - guest_hotspot_start_ns : 0u);
|
|
}
|
|
#if !defined(PSPRECOMP_AOT_PRODUCTION_FASTPATHS)
|
|
if (g_post_chained_call_hook != nullptr)
|
|
g_post_chained_call_hook(*this, ctx, target_pc, native_depth);
|
|
if (track_dispatch_counters_) ++chained_dispatches_;
|
|
#endif
|
|
|
|
// A descendant scheduler boundary may have switched the PSP context while
|
|
// this frame was active. Never let a stale native caller resume it.
|
|
if (caller_generation != g_runtime_thread_switch_generation_fast) {
|
|
(void)account_dispatch_work(ctx, false);
|
|
return false;
|
|
}
|
|
return account_dispatch_work(ctx, true);
|
|
}
|
|
|
|
void Runtime::register_generated_unit(std::uint32_t unit_index,
|
|
std::uint32_t unit_address,
|
|
std::uint32_t unit_span,
|
|
RecompiledFunction function,
|
|
RecompiledEntryFunction entry_function) {
|
|
if (function == nullptr || unit_span == 0u || !generated_unit_layout_valid_) return;
|
|
if (unit_index >= kGeneratedUnitFastCapacity) return;
|
|
const std::uint32_t canonical_address = memory_.canonical(unit_address);
|
|
const std::uint64_t offset = static_cast<std::uint64_t>(unit_index) * unit_span;
|
|
if (canonical_address < offset) return;
|
|
const std::uint32_t base = canonical_address - static_cast<std::uint32_t>(offset);
|
|
if (generated_unit_span_ == 0u) {
|
|
generated_unit_base_ = base;
|
|
generated_unit_span_ = unit_span;
|
|
} else if (generated_unit_base_ != base || generated_unit_span_ != unit_span) {
|
|
// Mixed automatic-codegen layouts are unsupported. Permanently disable
|
|
// the optional dense path for this Runtime; exact PC dispatch remains.
|
|
generated_units_.fill(nullptr);
|
|
generated_unit_disabled_.fill(0u);
|
|
generated_unit_base_ = 0u;
|
|
generated_unit_span_ = 0u;
|
|
generated_unit_layout_valid_ = false;
|
|
return;
|
|
}
|
|
if (generated_unit_disabled_[unit_index] == 0u)
|
|
generated_units_[unit_index] = function;
|
|
generated_unit_entries_[unit_index] = entry_function;
|
|
}
|
|
|
|
bool Runtime::invoke_isolated_aot(std::uint32_t address, AllegrexContext &ctx) {
|
|
const RecompiledFunction function = lookup_function(address);
|
|
if (function == nullptr) return false;
|
|
ctx.pc = address;
|
|
function(*this, ctx);
|
|
ctx.gpr[0] = 0u;
|
|
return true;
|
|
}
|
|
|
|
std::vector<std::pair<std::string, std::uint64_t>> Runtime::hle_histogram() const {
|
|
std::vector<std::pair<std::string, std::uint64_t>> entries(hle_histogram_.begin(), hle_histogram_.end());
|
|
std::sort(entries.begin(), entries.end(), [](const auto &left, const auto &right) {
|
|
if (left.second != right.second) return left.second > right.second;
|
|
return left.first < right.first;
|
|
});
|
|
return entries;
|
|
}
|
|
|
|
bool g_unit_profile_enabled = false;
|
|
std::uint64_t g_unit_profile_counts[kUnitProfileCapacity]{};
|
|
|
|
void report_unit_profile(std::size_t limit) {
|
|
if (!g_unit_profile_enabled) return;
|
|
std::vector<std::pair<std::uint64_t, std::size_t>> entries;
|
|
std::uint64_t total = 0u;
|
|
for (std::size_t index = 0; index < kUnitProfileCapacity; ++index) {
|
|
if (g_unit_profile_counts[index] == 0u) continue;
|
|
entries.emplace_back(g_unit_profile_counts[index], index);
|
|
total += g_unit_profile_counts[index];
|
|
}
|
|
std::sort(entries.begin(), entries.end(), std::greater<>());
|
|
std::cerr << "[unit-profile] units=" << entries.size() << " calls=" << total << "\n";
|
|
std::uint64_t running = 0u;
|
|
for (std::size_t i = 0; i < std::min(limit, entries.size()); ++i) {
|
|
running += entries[i].first;
|
|
std::cerr << "[unit-profile] unit=" << std::setw(4) << std::setfill('0')
|
|
<< entries[i].second << std::setfill(' ')
|
|
<< " calls=" << entries[i].first
|
|
<< " share=" << (100.0 * static_cast<double>(entries[i].first) /
|
|
static_cast<double>(total ? total : 1u))
|
|
<< "% cumulative=" << (100.0 * static_cast<double>(running) /
|
|
static_cast<double>(total ? total : 1u)) << "%\n";
|
|
}
|
|
}
|
|
|
|
void Runtime::report_hle_histogram(std::size_t limit) const {
|
|
if (!hle_histogram_enabled_) return;
|
|
const auto entries = hle_histogram();
|
|
std::uint64_t total = 0u;
|
|
for (const auto &entry : entries) total += entry.second;
|
|
std::cerr << "[hle-histogram] unique=" << entries.size() << " calls=" << total << "\n";
|
|
for (std::size_t index = 0; index < std::min(limit, entries.size()); ++index) {
|
|
const auto &entry = entries[index];
|
|
std::cerr << "[hle-histogram] " << entry.first << " count=" << entry.second
|
|
<< " name=" << nids_.resolve(entry.first.substr(0, entry.first.find(':')),
|
|
static_cast<std::uint32_t>(
|
|
std::strtoul(entry.first.substr(entry.first.find(':') + 1u).c_str(),
|
|
nullptr, 16)))
|
|
.value_or("unknown")
|
|
<< "\n";
|
|
}
|
|
}
|
|
NidRegistry &Runtime::nids() noexcept { return nids_; }
|
|
const NidRegistry &Runtime::nids() const noexcept { return nids_; }
|
|
|
|
void Runtime::register_function(std::uint32_t address, RecompiledFunction function, std::string name) {
|
|
if (!function) throw Error("Attempted to register a null recompiled function");
|
|
functions_[address] = FunctionEntry{function, std::move(name)};
|
|
const std::uint32_t c = memory_.canonical(address);
|
|
if ((c & 3u) != 0u || c < GuestMemory::kPhysicalBase) return;
|
|
|
|
// The direct table covers only the window that actually holds registered
|
|
// code. Spanning the whole 32 MiB of guest RAM made it a 64 MiB array that
|
|
// every outer dispatch probed at an effectively random offset, so the hot
|
|
// path took a cache/TLB miss per dispatch. Growing on demand keeps it at
|
|
// the size of the executable image instead.
|
|
constexpr std::size_t chunk = 1024u * 1024u / 4u;
|
|
const auto shift_forward = [](std::vector<RecompiledFunction> &table, std::size_t slots) {
|
|
std::vector<RecompiledFunction> grown(table.size() + slots, nullptr);
|
|
std::copy(table.begin(), table.end(), grown.begin() + static_cast<std::ptrdiff_t>(slots));
|
|
table = std::move(grown);
|
|
};
|
|
if (direct_functions_.empty()) {
|
|
direct_base_ = c & ~0xFFFFFu; // 1 MiB-aligned window start
|
|
direct_functions_.assign(chunk, nullptr);
|
|
direct_chainable_.assign(chunk, nullptr);
|
|
} else if (c < direct_base_) {
|
|
const std::size_t slots = static_cast<std::size_t>(direct_base_ - (c & ~0xFFFFFu)) / 4u;
|
|
shift_forward(direct_functions_, slots);
|
|
shift_forward(direct_chainable_, slots);
|
|
direct_base_ = c & ~0xFFFFFu;
|
|
}
|
|
const std::size_t index = static_cast<std::size_t>(c - direct_base_) / 4u;
|
|
if (index >= direct_functions_.size()) {
|
|
const std::size_t grown = ((index / chunk) + 1u) * chunk;
|
|
direct_functions_.resize(grown, nullptr);
|
|
direct_chainable_.resize(grown, nullptr);
|
|
}
|
|
direct_functions_[index] = function;
|
|
// Import wrappers and HLE entries are deliberately excluded: chaining must
|
|
// never run one inline, because they are where thread switches happen.
|
|
// Assign unconditionally -- an import stub can also be covered by a unit,
|
|
// and a later import registration has to clear the earlier unit pointer or
|
|
// chaining would keep running the raw code instead of the wrapper.
|
|
const auto ®istered = functions_[address].name;
|
|
const bool chainable = registered.starts_with("recomp_unit_");
|
|
direct_chainable_[index] = chainable ? function : nullptr;
|
|
|
|
if (!chainable && generated_unit_span_ != 0u && c >= generated_unit_base_) {
|
|
const std::uint32_t unit_delta = c - generated_unit_base_;
|
|
const std::size_t unit_index = static_cast<std::size_t>(unit_delta / generated_unit_span_);
|
|
if (unit_index < kGeneratedUnitFastCapacity) {
|
|
generated_unit_disabled_[unit_index] = 1u;
|
|
generated_units_[unit_index] = nullptr;
|
|
}
|
|
}
|
|
}
|
|
|
|
Runtime::RecompiledFunction Runtime::lookup_function(std::uint32_t address) const noexcept {
|
|
const std::uint32_t c = memory_.canonical(address);
|
|
const std::uint32_t delta = c - direct_base_;
|
|
if ((delta & 3u) == 0u) {
|
|
const std::size_t index = static_cast<std::size_t>(delta) / 4u;
|
|
if (index < direct_functions_.size()) return direct_functions_[index];
|
|
}
|
|
const auto found = functions_.find(address);
|
|
return found != functions_.end() ? found->second.function : nullptr;
|
|
}
|
|
|
|
Runtime::RecompiledFunction Runtime::lookup_generated_unit(std::uint32_t address) const noexcept {
|
|
if (!generated_unit_layout_valid_ || generated_unit_span_ == 0u) return nullptr;
|
|
const std::uint32_t c = memory_.canonical(address);
|
|
if (c < generated_unit_base_) return nullptr;
|
|
const std::uint32_t delta = c - generated_unit_base_;
|
|
const std::size_t unit_index = generated_unit_span_ == 16384u
|
|
? static_cast<std::size_t>(delta >> 14u)
|
|
: static_cast<std::size_t>(delta / generated_unit_span_);
|
|
if (unit_index >= kGeneratedUnitFastCapacity) return nullptr;
|
|
return generated_units_[unit_index];
|
|
}
|
|
|
|
const Runtime::FunctionEntry *Runtime::lookup_entry(std::uint32_t address) const noexcept {
|
|
const auto found = functions_.find(address);
|
|
return found != functions_.end() ? &found->second : nullptr;
|
|
}
|
|
|
|
std::string Runtime::hle_key(std::string_view library, std::uint32_t nid) {
|
|
std::string key;
|
|
key.reserve(library.size() + 11u);
|
|
key.append(library);
|
|
key.push_back(':');
|
|
key.append(hex32(nid));
|
|
return key;
|
|
}
|
|
|
|
void Runtime::register_hle(std::string library, std::uint32_t nid, HleFunction function) {
|
|
hle_[std::move(library)][nid] = std::move(function);
|
|
}
|
|
|
|
bool Runtime::has_function(std::uint32_t address) const { return lookup_function(address) != nullptr; }
|
|
std::size_t Runtime::function_count() const noexcept { return functions_.size(); }
|
|
|
|
void Runtime::set_game_root(std::filesystem::path root) { game_root_ = std::filesystem::weakly_canonical(std::move(root)); }
|
|
const std::filesystem::path &Runtime::game_root() const noexcept { return game_root_; }
|
|
|
|
std::filesystem::path Runtime::translate_path(const std::string &psp_path) const {
|
|
std::string relative = psp_path;
|
|
const auto colon = relative.find(':');
|
|
if (colon != std::string::npos) relative.erase(0, colon + 1u);
|
|
while (!relative.empty() && (relative.front() == '/' || relative.front() == '\\')) relative.erase(relative.begin());
|
|
std::replace(relative.begin(), relative.end(), '\\', '/');
|
|
std::filesystem::path clean;
|
|
for (const auto &part : std::filesystem::path(relative)) {
|
|
if (part == "..") throw Error("Rejected PSP path traversal: " + psp_path);
|
|
if (part != ".") clean /= part;
|
|
}
|
|
return game_root_ / clean;
|
|
}
|
|
|
|
void Runtime::run(std::uint32_t entry, std::uint64_t max_dispatches) {
|
|
stopped_ = false;
|
|
stop_reason_.clear();
|
|
cpu_.pc = entry;
|
|
load_counted_pcs();
|
|
const bool profile_dispatch = std::getenv("PSPRECOMP_PROFILE_DISPATCH") != nullptr;
|
|
const auto parse_environment_u64 = [](const char *name, std::uint64_t fallback = 0u) {
|
|
const char *text = std::getenv(name);
|
|
if (text == nullptr || *text == '\0') return fallback;
|
|
char *end = nullptr;
|
|
const unsigned long long value = std::strtoull(text, &end, 0);
|
|
return end != text && *end == '\0' ? static_cast<std::uint64_t>(value) : fallback;
|
|
};
|
|
const std::uint64_t profile_dispatch_start =
|
|
parse_environment_u64("PSPRECOMP_PROFILE_DISPATCH_START");
|
|
const std::uint64_t profile_dispatch_count =
|
|
parse_environment_u64("PSPRECOMP_PROFILE_DISPATCH_COUNT");
|
|
std::unordered_map<std::uint32_t, std::uint64_t> dispatch_counts;
|
|
if (profile_dispatch) dispatch_counts.reserve(4096u);
|
|
struct DispatchSnapshot {
|
|
std::uint32_t pc{};
|
|
std::uint32_t a0{};
|
|
std::uint32_t a1{};
|
|
std::uint32_t a2{};
|
|
std::uint32_t a3{};
|
|
std::uint32_t sp{};
|
|
std::uint32_t ra{};
|
|
std::int32_t thread_uid{-1};
|
|
std::array<char, 64> thread_name{};
|
|
};
|
|
std::array<DispatchSnapshot, 64> recent_dispatches{};
|
|
std::size_t recent_count = 0u;
|
|
std::size_t recent_next = 0u;
|
|
const bool trace_on_error = std::getenv("PSPRECOMP_TRACE_ON_ERROR") != nullptr;
|
|
const bool world_stream_diag = std::getenv("PSPRECOMP_WORLD_STREAM_DIAG") != nullptr;
|
|
const bool request_alloc_diag = std::getenv("PSPRECOMP_REQUEST_ALLOC_DIAG") != nullptr;
|
|
const bool world_stream_stop_at_callback = std::getenv("PSPRECOMP_WORLD_STREAM_STOP_AT_CALLBACK") != nullptr;
|
|
const std::uint32_t world_stream_manager = static_cast<std::uint32_t>(
|
|
parse_environment_u64("PSPRECOMP_WORLD_STREAM_MANAGER", 0x08E91200u));
|
|
bool world_stream_active_known = false;
|
|
std::uint32_t world_stream_previous_active = 0u;
|
|
const bool heap_diag = std::getenv("PSPRECOMP_HEAP_DIAG") != nullptr;
|
|
const bool file_object_diag = std::getenv("PSPRECOMP_FILE_OBJECT_DIAG") != nullptr;
|
|
const bool file_object_stop_on_null = std::getenv("PSPRECOMP_FILE_OBJECT_STOP_ON_NULL") != nullptr;
|
|
std::uint32_t trace_pc = 0u;
|
|
bool trace_pc_enabled = false;
|
|
if (const char *trace_pc_text = std::getenv("PSPRECOMP_TRACE_PC")) {
|
|
char *end = nullptr;
|
|
trace_pc = static_cast<std::uint32_t>(std::strtoul(trace_pc_text, &end, 0));
|
|
trace_pc_enabled = end != trace_pc_text && *end == '\0';
|
|
}
|
|
const bool trace_dispatch = std::getenv("PSPRECOMP_TRACE") != nullptr;
|
|
const bool strict_pc_progress = std::getenv("PSPRECOMP_STRICT_PC_PROGRESS") != nullptr;
|
|
const bool diagnostic_dispatch = profile_dispatch || trace_on_error || world_stream_diag ||
|
|
request_alloc_diag || world_stream_stop_at_callback || heap_diag || file_object_diag || file_object_stop_on_null ||
|
|
trace_pc_enabled || trace_dispatch;
|
|
const bool deferred_profile_only = profile_dispatch && profile_dispatch_start != 0u &&
|
|
!trace_on_error && !world_stream_diag && !request_alloc_diag && !world_stream_stop_at_callback && !heap_diag &&
|
|
!file_object_diag && !file_object_stop_on_null && !trace_pc_enabled && !trace_dispatch;
|
|
if (deferred_profile_only) {
|
|
const std::uint64_t profile_end = profile_dispatch_count == 0u ||
|
|
profile_dispatch_start > std::numeric_limits<std::uint64_t>::max() - profile_dispatch_count
|
|
? max_dispatches
|
|
: std::min(max_dispatches, profile_dispatch_start + profile_dispatch_count);
|
|
std::uint64_t dispatch = 0u;
|
|
auto execute_once = [&]() {
|
|
const std::uint32_t before = cpu_.pc;
|
|
chain_context_invalidated_ = false;
|
|
const std::int32_t dispatch_thread_uid = g_runtime_thread_uid;
|
|
RecompiledFunction function = lookup_function(before);
|
|
if (function == nullptr) {
|
|
stop("No recompiled function registered at " + hex32(before));
|
|
return;
|
|
}
|
|
g_runtime_dispatch_pc = before;
|
|
count_pc(before);
|
|
if (g_pre_dispatch_hook != nullptr)
|
|
g_pre_dispatch_hook(*this, cpu_, before, dispatch_thread_uid);
|
|
try {
|
|
function(*this, cpu_);
|
|
} catch (const Error &e) {
|
|
const FunctionEntry *function_entry = lookup_entry(before);
|
|
const std::string name = function_entry != nullptr ? function_entry->name : "unknown";
|
|
std::ostringstream message;
|
|
message << e.what() << " while executing " << name
|
|
<< " dispatch=" << hex32(before) << " guest_pc=" << hex32(cpu_.pc)
|
|
<< " (a0=" << hex32(cpu_.gpr[4]) << ", a1=" << hex32(cpu_.gpr[5])
|
|
<< ", a2=" << hex32(cpu_.gpr[6]) << ", a3=" << hex32(cpu_.gpr[7])
|
|
<< ", sp=" << hex32(cpu_.gpr[29]) << ", ra=" << hex32(cpu_.gpr[31]) << ")";
|
|
append_gpr_dump(message, cpu_);
|
|
throw Error(message.str());
|
|
}
|
|
cpu_.gpr[0] = 0u;
|
|
if (!stopped_ && g_post_dispatch_hook != nullptr)
|
|
g_post_dispatch_hook(*this, cpu_, before, dispatch_thread_uid);
|
|
if (!stopped_) (void)account_dispatch_work(cpu_, true);
|
|
if (!stopped_ && strict_pc_progress && cpu_.pc == before) {
|
|
const FunctionEntry *function_entry = lookup_entry(before);
|
|
const std::string name = function_entry != nullptr ? function_entry->name : "unknown";
|
|
stop("Recompiled function returned without changing PC: " + name + " at " + hex32(before));
|
|
}
|
|
};
|
|
for (; dispatch < std::min(profile_dispatch_start, max_dispatches) && !stopped_; ++dispatch)
|
|
execute_once();
|
|
for (; dispatch < profile_end && !stopped_; ++dispatch) {
|
|
++dispatch_counts[cpu_.pc];
|
|
execute_once();
|
|
}
|
|
std::vector<std::pair<std::uint32_t, std::uint64_t>> hot(dispatch_counts.begin(), dispatch_counts.end());
|
|
std::sort(hot.begin(), hot.end(), [](const auto &left, const auto &right) {
|
|
return left.second > right.second;
|
|
});
|
|
std::cerr << "[profile-window] start=" << profile_dispatch_start
|
|
<< " executed=" << dispatch
|
|
<< " sampled=" << (dispatch >= profile_dispatch_start ? dispatch - profile_dispatch_start : 0u)
|
|
<< " pc=" << hex32(cpu_.pc) << " unique=" << hot.size() << "\n";
|
|
const std::size_t count = std::min<std::size_t>(30u, hot.size());
|
|
for (std::size_t index = 0; index < count; ++index) {
|
|
const FunctionEntry *profile_entry = lookup_entry(hot[index].first);
|
|
std::cerr << "[profile-window] " << hex32(hot[index].first)
|
|
<< " count=" << hot[index].second
|
|
<< " name=" << (profile_entry != nullptr ? profile_entry->name : "unknown") << "\n";
|
|
}
|
|
if (!stopped_) stop("Dispatch profile window complete at " + hex32(cpu_.pc));
|
|
return;
|
|
}
|
|
if (!diagnostic_dispatch) {
|
|
const std::uint64_t progress_every = parse_environment_u64("PSPRECOMP_PROGRESS_EVERY");
|
|
std::uint64_t next_progress = progress_every;
|
|
const RuntimeHeartbeatHook heartbeat = g_heartbeat_hook;
|
|
const std::uint64_t heartbeat_every = heartbeat != nullptr ? g_heartbeat_interval : 0u;
|
|
std::uint64_t next_heartbeat = heartbeat_every;
|
|
const RuntimeStarvationHook starvation = g_starvation_hook;
|
|
const std::uint64_t starvation_every = starvation != nullptr ? g_starvation_interval : 0u;
|
|
std::uint64_t executed_dispatches = 0u;
|
|
for (; executed_dispatches < max_dispatches && !stopped_; ++executed_dispatches) {
|
|
const std::uint32_t before = cpu_.pc;
|
|
chain_context_invalidated_ = false;
|
|
const std::int32_t dispatch_thread_uid = g_runtime_thread_uid;
|
|
// Most outer dispatches are ordinary AOT PCs. Resolve those
|
|
// through the tiny generated-unit table first; the old per-PC
|
|
// table spans several MiB and an effectively random probe here can
|
|
// cost a cache/TLB miss at every outer return. Any unit containing
|
|
// a host/import override is poisoned at registration and falls
|
|
// back to exact per-PC dispatch automatically.
|
|
RecompiledFunction function = lookup_generated_unit(before);
|
|
if (function == nullptr) function = lookup_function(before);
|
|
if (function == nullptr) {
|
|
stop("No recompiled function registered at " + hex32(before));
|
|
break;
|
|
}
|
|
g_runtime_dispatch_pc = before;
|
|
count_pc(before);
|
|
if (g_pre_dispatch_hook != nullptr)
|
|
g_pre_dispatch_hook(*this, cpu_, before, dispatch_thread_uid);
|
|
try {
|
|
function(*this, cpu_);
|
|
} catch (const Error &e) {
|
|
const FunctionEntry *function_entry = lookup_entry(before);
|
|
const std::string name = function_entry != nullptr ? function_entry->name : "unknown";
|
|
std::ostringstream message;
|
|
message << e.what() << " while executing " << name
|
|
<< " dispatch=" << hex32(before) << " guest_pc=" << hex32(cpu_.pc)
|
|
<< " (a0=" << hex32(cpu_.gpr[4]) << ", a1=" << hex32(cpu_.gpr[5])
|
|
<< ", a2=" << hex32(cpu_.gpr[6]) << ", a3=" << hex32(cpu_.gpr[7])
|
|
<< ", sp=" << hex32(cpu_.gpr[29]) << ", ra=" << hex32(cpu_.gpr[31]) << ")";
|
|
append_gpr_dump(message, cpu_);
|
|
throw Error(message.str());
|
|
}
|
|
// `$zero` is now enforced at every generated/HLE write site. Do not
|
|
// dirty the context register cache line after every outer dispatch.
|
|
if (!stopped_ && g_post_dispatch_hook != nullptr)
|
|
g_post_dispatch_hook(*this, cpu_, before, dispatch_thread_uid);
|
|
if (progress_every != 0u && executed_dispatches + 1u >= next_progress) {
|
|
std::cerr << "[progress] dispatch=" << (executed_dispatches + 1u)
|
|
<< " pc=" << hex32(cpu_.pc) << "\n";
|
|
next_progress += progress_every;
|
|
}
|
|
if (heartbeat_every != 0u && executed_dispatches + 1u >= next_heartbeat) {
|
|
heartbeat(executed_dispatches + 1u, cpu_.pc);
|
|
next_heartbeat += heartbeat_every;
|
|
}
|
|
if (starvation_every != 0u && starvation != nullptr)
|
|
(void)account_dispatch_work(cpu_, true);
|
|
if (!stopped_ && strict_pc_progress && cpu_.pc == before) {
|
|
const FunctionEntry *function_entry = lookup_entry(before);
|
|
const std::string name = function_entry != nullptr ? function_entry->name : "unknown";
|
|
stop("Recompiled function returned without changing PC: " + name + " at " + hex32(before));
|
|
}
|
|
}
|
|
if (std::getenv("PSPRECOMP_REPORT_DISPATCH_COUNT") != nullptr) {
|
|
std::cerr << "[dispatch-count] executed=" << executed_dispatches
|
|
<< " chained=" << chained_dispatches_
|
|
<< " total=" << (executed_dispatches + chained_dispatches_)
|
|
<< " pc=" << hex32(cpu_.pc) << "\n";
|
|
}
|
|
if (!stopped_ && max_dispatches != 0u)
|
|
stop("Dispatch limit reached at " + hex32(cpu_.pc));
|
|
return;
|
|
}
|
|
for (std::uint64_t dispatch = 0; dispatch < max_dispatches && !stopped_; ++dispatch) {
|
|
if (profile_dispatch) ++dispatch_counts[cpu_.pc];
|
|
const std::uint32_t before = cpu_.pc;
|
|
chain_context_invalidated_ = false;
|
|
RecompiledFunction function = lookup_function(before);
|
|
const FunctionEntry *function_entry = lookup_entry(before);
|
|
if (function == nullptr) {
|
|
stop("No recompiled function registered at " + hex32(before));
|
|
break;
|
|
}
|
|
g_runtime_dispatch_pc = before;
|
|
count_pc(before);
|
|
DispatchSnapshot current_snapshot{};
|
|
current_snapshot.pc = before;
|
|
current_snapshot.a0 = cpu_.gpr[4];
|
|
current_snapshot.a1 = cpu_.gpr[5];
|
|
current_snapshot.a2 = cpu_.gpr[6];
|
|
current_snapshot.a3 = cpu_.gpr[7];
|
|
current_snapshot.sp = cpu_.gpr[29];
|
|
current_snapshot.ra = cpu_.gpr[31];
|
|
current_snapshot.thread_uid = g_runtime_thread_uid;
|
|
current_snapshot.thread_name = g_runtime_thread_name;
|
|
recent_dispatches[recent_next] = current_snapshot;
|
|
recent_next = (recent_next + 1u) % recent_dispatches.size();
|
|
recent_count = std::min(recent_count + 1u, recent_dispatches.size());
|
|
if (before == 0x089345B0u && heap_diag) {
|
|
constexpr std::uint32_t manager = 0x08BC6500u;
|
|
std::cerr << "[heapdiag] callback entered manager=" << hex32(manager)
|
|
<< " global=" << hex32(memory_.load32(0x08BADEF8u)) << "\n";
|
|
for (std::uint32_t offset = 0; offset < 0x140u; offset += 16u) {
|
|
std::cerr << "[heapdiag] " << hex32(manager + offset);
|
|
for (std::uint32_t word = 0; word < 16u; word += 4u)
|
|
std::cerr << " " << hex32(memory_.load32(manager + offset + word));
|
|
std::cerr << "\n";
|
|
}
|
|
std::uint32_t block = memory_.load32(manager + 8u);
|
|
for (std::uint32_t index = 0u; block != 0u && index < 64u; ++index) {
|
|
if (!memory_.contains(block, 16u)) {
|
|
std::cerr << "[heapdiag-free] invalid=" << hex32(block) << "\n";
|
|
break;
|
|
}
|
|
const std::uint32_t size = memory_.load32(block + 0u);
|
|
const std::uint32_t prev = memory_.load32(block + 8u);
|
|
const std::uint32_t next = memory_.load32(block + 12u);
|
|
std::cerr << "[heapdiag-free] index=" << index << " block=" << hex32(block)
|
|
<< " size=" << hex32(size) << " prev=" << hex32(prev)
|
|
<< " next=" << hex32(next) << " end=" << hex32(block + size) << "\n";
|
|
block = next;
|
|
}
|
|
}
|
|
if (request_alloc_diag &&
|
|
(before == 0x089390ACu || before == 0x08939114u ||
|
|
before == 0x08939590u || before == 0x089395D4u ||
|
|
before == 0x0893961Cu || before == 0x089396D8u ||
|
|
before == 0x089397CCu || before == 0x08956258u)) {
|
|
const RuntimeExecutionContextToken token = capture_runtime_execution_context();
|
|
std::uint32_t manager = 0u;
|
|
if (before == 0x08939590u) manager = cpu_.gpr[4];
|
|
else if (before == 0x089395D4u || before == 0x0893961Cu ||
|
|
before == 0x089396D8u) manager = cpu_.gpr[16];
|
|
else if ((before == 0x089390ACu || before == 0x08939114u) &&
|
|
memory_.contains(cpu_.gpr[28] + 5908u, 4u))
|
|
manager = memory_.load32(cpu_.gpr[28] + 5908u);
|
|
else if (before == 0x089397CCu) manager = cpu_.gpr[4];
|
|
std::cerr << "[reqalloc] pc=" << hex32(before)
|
|
<< " uid=" << token.thread_uid
|
|
<< " gen=" << token.switch_generation
|
|
<< " v0=" << hex32(cpu_.gpr[2])
|
|
<< " a0=" << hex32(cpu_.gpr[4])
|
|
<< " a1=" << hex32(cpu_.gpr[5])
|
|
<< " a2=" << hex32(cpu_.gpr[6])
|
|
<< " a3=" << hex32(cpu_.gpr[7])
|
|
<< " t0=" << hex32(cpu_.gpr[8])
|
|
<< " t1=" << hex32(cpu_.gpr[9])
|
|
<< " s0=" << hex32(cpu_.gpr[16])
|
|
<< " s1=" << hex32(cpu_.gpr[17])
|
|
<< " s2=" << hex32(cpu_.gpr[18])
|
|
<< " sp=" << hex32(cpu_.gpr[29])
|
|
<< " ra=" << hex32(cpu_.gpr[31]);
|
|
if (manager != 0u && memory_.contains(manager + 6912u, 4u)) {
|
|
const std::uint32_t free_head = memory_.load32(manager + 6900u);
|
|
const std::uint32_t active_head = memory_.load32(manager + 6908u);
|
|
std::cerr << " manager=" << hex32(manager)
|
|
<< " free_head=" << hex32(free_head)
|
|
<< " active_head=" << hex32(active_head);
|
|
}
|
|
std::uint32_t request = 0u;
|
|
if (before == 0x089397CCu) request = cpu_.gpr[5];
|
|
else if (before == 0x089396D8u) request = cpu_.gpr[18];
|
|
else if (before == 0x08956258u || before == 0x08939114u) request = cpu_.gpr[2];
|
|
if (request != 0u && memory_.contains(request, 52u)) {
|
|
std::cerr << " req=" << hex32(request)
|
|
<< " prev=" << hex32(memory_.load32(request + 0u))
|
|
<< " next=" << hex32(memory_.load32(request + 4u))
|
|
<< " size=" << memory_.load32(request + 8u)
|
|
<< " source=" << hex32(memory_.load32(request + 16u))
|
|
<< " offset=" << memory_.load32(request + 20u)
|
|
<< " remaining=" << memory_.load32(request + 24u)
|
|
<< " progressed=" << memory_.load32(request + 28u)
|
|
<< " callback=" << hex32(memory_.load32(request + 48u));
|
|
}
|
|
std::cerr << "\n";
|
|
}
|
|
if (world_stream_diag &&
|
|
(before == 0x08953990u || before == 0x08955134u ||
|
|
before == 0x08955E7Cu || before == 0x08956258u ||
|
|
before == 0x089563C0u || before == 0x08956408u ||
|
|
before == 0x089569C0u || before == 0x089569E0u)) {
|
|
std::cerr << "[worlddiag] pc=" << hex32(before)
|
|
<< " uid=" << g_runtime_thread_uid
|
|
<< " a0=" << hex32(cpu_.gpr[4])
|
|
<< " a1=" << hex32(cpu_.gpr[5])
|
|
<< " ra=" << hex32(cpu_.gpr[31]);
|
|
if (before == 0x089563C0u && memory_.contains(cpu_.gpr[4], 1040u)) {
|
|
const std::uint32_t manager = cpu_.gpr[4];
|
|
std::cerr << " req=" << hex32(cpu_.gpr[5])
|
|
<< " f596=" << hex32(memory_.load32(manager + 596u))
|
|
<< " f600=" << hex32(memory_.load32(manager + 600u))
|
|
<< " event=" << memory_.load32(manager + 616u)
|
|
<< " active=" << hex32(memory_.load32(manager + 628u))
|
|
<< " mode=" << memory_.load32(manager + 636u)
|
|
<< " work=" << hex32(memory_.load32(manager + 640u))
|
|
<< " limit=" << memory_.load32(manager + 648u)
|
|
<< " stack=" << memory_.load32(manager + 1036u);
|
|
}
|
|
std::uint32_t manager = world_stream_manager;
|
|
if (before == 0x08955E7Cu || before == 0x08956258u) manager = cpu_.gpr[17];
|
|
else if (before == 0x08956408u) manager = cpu_.gpr[16];
|
|
else if (before == 0x089563C0u) manager = cpu_.gpr[4];
|
|
if (manager != 0u && memory_.contains(manager + 628u, 4u)) {
|
|
std::cerr << " manager=" << hex32(manager)
|
|
<< " active_before=" << hex32(memory_.load32(manager + 628u))
|
|
<< " pending_v0=" << hex32(cpu_.gpr[2]);
|
|
}
|
|
std::cerr << "\n";
|
|
if (world_stream_stop_at_callback && before == 0x089563C0u) {
|
|
stop("World-stream diagnostic stop at callback " + hex32(before));
|
|
break;
|
|
}
|
|
}
|
|
if (file_object_diag &&
|
|
(before == 0x08938F04u || before == 0x08938F7Cu || before == 0x089394A4u ||
|
|
before == 0x08955DCCu || before == 0x08955DFCu || before == 0x08955E58u ||
|
|
before == 0x08955E7Cu)) {
|
|
const std::uint32_t manager =
|
|
before == 0x08955DCCu ? cpu_.gpr[4] :
|
|
((before == 0x08955DFCu || before == 0x08955E58u || before == 0x08955E7Cu) ? cpu_.gpr[17] : 0u);
|
|
const std::uint32_t object = before == 0x08938F7Cu ? cpu_.gpr[4] :
|
|
(before == 0x089394A4u ? cpu_.gpr[5] : 0u);
|
|
const bool seek_pc = before == 0x08938F7Cu || before == 0x089394A4u;
|
|
bool manager_changed = false;
|
|
static std::uint32_t previous_manager = 0u;
|
|
static std::uint32_t previous_file_object = 0xFFFFFFFFu;
|
|
static std::uint32_t previous_active = 0xFFFFFFFFu;
|
|
static std::uint32_t previous_mode = 0xFFFFFFFFu;
|
|
std::uint32_t manager_file_object = 0u;
|
|
std::uint32_t manager_active = 0u;
|
|
std::uint32_t manager_mode = 0u;
|
|
if (manager != 0u && memory_.contains(manager, 652u)) {
|
|
manager_file_object = memory_.load32(manager + 624u);
|
|
manager_active = memory_.load32(manager + 628u);
|
|
manager_mode = memory_.load32(manager + 636u);
|
|
manager_changed = manager != previous_manager ||
|
|
manager_file_object != previous_file_object ||
|
|
manager_active != previous_active || manager_mode != previous_mode;
|
|
previous_manager = manager;
|
|
previous_file_object = manager_file_object;
|
|
previous_active = manager_active;
|
|
previous_mode = manager_mode;
|
|
}
|
|
const bool should_log = before == 0x08938F04u ||
|
|
(seek_pc && object == 0u) || manager_changed;
|
|
if (should_log) {
|
|
std::cerr << "[fileobj] pc=" << hex32(before)
|
|
<< " uid=" << g_runtime_thread_uid
|
|
<< " name=" << g_runtime_thread_name.data()
|
|
<< " v0=" << hex32(cpu_.gpr[2])
|
|
<< " a0=" << hex32(cpu_.gpr[4])
|
|
<< " a1=" << hex32(cpu_.gpr[5])
|
|
<< " a2=" << hex32(cpu_.gpr[6])
|
|
<< " a3=" << hex32(cpu_.gpr[7])
|
|
<< " s0=" << hex32(cpu_.gpr[16])
|
|
<< " s1=" << hex32(cpu_.gpr[17])
|
|
<< " sp=" << hex32(cpu_.gpr[29])
|
|
<< " ra=" << hex32(cpu_.gpr[31]);
|
|
if (before == 0x08938F04u && memory_.contains(cpu_.gpr[4])) {
|
|
try { std::cerr << " path=\"" << memory_.read_c_string(cpu_.gpr[4], 512u) << "\""; } catch (...) {}
|
|
}
|
|
if (manager != 0u && memory_.contains(manager, 652u)) {
|
|
std::cerr << " manager=" << hex32(manager)
|
|
<< " f596=" << hex32(memory_.load32(manager + 596u))
|
|
<< " f600=" << hex32(memory_.load32(manager + 600u))
|
|
<< " event=" << memory_.load32(manager + 616u)
|
|
<< " fileobj=" << hex32(manager_file_object)
|
|
<< " active=" << hex32(manager_active)
|
|
<< " mode=" << manager_mode
|
|
<< " work=" << hex32(memory_.load32(manager + 640u))
|
|
<< " offset=" << memory_.load32(manager + 644u)
|
|
<< " length=" << memory_.load32(manager + 648u);
|
|
}
|
|
if (object != 0u && memory_.contains(object, 16u)) {
|
|
std::cerr << " object=" << hex32(object)
|
|
<< " words=" << hex32(memory_.load32(object + 0u))
|
|
<< "," << hex32(memory_.load32(object + 4u))
|
|
<< "," << hex32(memory_.load32(object + 8u))
|
|
<< "," << hex32(memory_.load32(object + 12u));
|
|
}
|
|
std::cerr << "\n";
|
|
}
|
|
if (file_object_stop_on_null && seek_pc && object == 0u) {
|
|
stop("File-object diagnostic stop before null seek at " + hex32(before));
|
|
break;
|
|
}
|
|
}
|
|
if (trace_pc_enabled && before == trace_pc) {
|
|
std::cerr << "[trace-pc] " << hex32(before) << " " << (function_entry != nullptr ? function_entry->name : std::string("unknown"))
|
|
<< " v0=" << hex32(cpu_.gpr[2])
|
|
<< " a0=" << hex32(cpu_.gpr[4])
|
|
<< " a1=" << hex32(cpu_.gpr[5])
|
|
<< " a2=" << hex32(cpu_.gpr[6])
|
|
<< " a3=" << hex32(cpu_.gpr[7])
|
|
<< " s0=" << hex32(cpu_.gpr[16])
|
|
<< " s1=" << hex32(cpu_.gpr[17])
|
|
<< " s2=" << hex32(cpu_.gpr[18])
|
|
<< " s3=" << hex32(cpu_.gpr[19])
|
|
<< " sp=" << hex32(cpu_.gpr[29])
|
|
<< " ra=" << hex32(cpu_.gpr[31]);
|
|
if (memory_.contains(cpu_.gpr[4])) {
|
|
try { std::cerr << " a0str=\"" << memory_.read_c_string(cpu_.gpr[4], 256u) << "\""; } catch (...) {}
|
|
}
|
|
if (memory_.contains(cpu_.gpr[5])) {
|
|
try { std::cerr << " a1str=\"" << memory_.read_c_string(cpu_.gpr[5], 256u) << "\""; } catch (...) {}
|
|
}
|
|
if (std::getenv("PSPRECOMP_TRACE_MEM") != nullptr) {
|
|
for (const auto &[label, address] : std::array<std::pair<const char *, std::uint32_t>, 2>{{{"a0mem", cpu_.gpr[4]}, {"a1mem", cpu_.gpr[5]}}}) {
|
|
if (!memory_.contains(address, 32u)) continue;
|
|
std::cerr << " " << label << "=";
|
|
for (std::uint32_t offset = 0; offset < 32u; offset += 4u)
|
|
std::cerr << (offset == 0u ? "" : ",") << hex32(memory_.load32(address + offset));
|
|
}
|
|
}
|
|
std::cerr << "\n";
|
|
}
|
|
if (std::getenv("PSPRECOMP_TRACE") != nullptr) {
|
|
std::cerr << "[dispatch] " << hex32(before) << " " << (function_entry != nullptr ? function_entry->name : std::string("unknown"))
|
|
<< " a0=" << hex32(cpu_.gpr[4])
|
|
<< " a1=" << hex32(cpu_.gpr[5])
|
|
<< " a2=" << hex32(cpu_.gpr[6])
|
|
<< " a3=" << hex32(cpu_.gpr[7])
|
|
<< " sp=" << hex32(cpu_.gpr[29])
|
|
<< " ra=" << hex32(cpu_.gpr[31]) << "\n";
|
|
}
|
|
try {
|
|
function(*this, cpu_);
|
|
} catch (const Error &e) {
|
|
if (trace_on_error) {
|
|
std::cerr << "[recent-dispatches] oldest-to-newest count=" << recent_count << "\n";
|
|
const std::size_t first = (recent_next + recent_dispatches.size() - recent_count) % recent_dispatches.size();
|
|
for (std::size_t index = 0; index < recent_count; ++index) {
|
|
const DispatchSnapshot &snapshot = recent_dispatches[(first + index) % recent_dispatches.size()];
|
|
std::cerr << "[recent] uid=" << snapshot.thread_uid
|
|
<< " name=" << snapshot.thread_name.data()
|
|
<< " pc=" << hex32(snapshot.pc)
|
|
<< " a0=" << hex32(snapshot.a0)
|
|
<< " a1=" << hex32(snapshot.a1)
|
|
<< " a2=" << hex32(snapshot.a2)
|
|
<< " a3=" << hex32(snapshot.a3)
|
|
<< " sp=" << hex32(snapshot.sp)
|
|
<< " ra=" << hex32(snapshot.ra) << "\n";
|
|
}
|
|
}
|
|
std::ostringstream message;
|
|
message << e.what() << " while executing " << (function_entry != nullptr ? function_entry->name : std::string("unknown")) << " dispatch=" << hex32(before) << " guest_pc=" << hex32(cpu_.pc)
|
|
<< " (a0=" << hex32(cpu_.gpr[4]) << ", a1=" << hex32(cpu_.gpr[5])
|
|
<< ", a2=" << hex32(cpu_.gpr[6]) << ", a3=" << hex32(cpu_.gpr[7])
|
|
<< ", sp=" << hex32(cpu_.gpr[29]) << ", ra=" << hex32(cpu_.gpr[31]) << ")";
|
|
append_gpr_dump(message, cpu_);
|
|
throw Error(message.str());
|
|
}
|
|
cpu_.gpr[0] = 0u;
|
|
if (world_stream_diag && world_stream_manager != 0u &&
|
|
memory_.contains(world_stream_manager + 628u, 4u)) {
|
|
const std::uint32_t active = memory_.load32(world_stream_manager + 628u);
|
|
if (!world_stream_active_known) {
|
|
world_stream_previous_active = active;
|
|
world_stream_active_known = true;
|
|
} else if (active != world_stream_previous_active) {
|
|
std::cerr << "[worlddiag-transition] dispatch_pc=" << hex32(before)
|
|
<< " uid_before=" << current_snapshot.thread_uid
|
|
<< " uid_after=" << g_runtime_thread_uid
|
|
<< " manager=" << hex32(world_stream_manager)
|
|
<< " active=" << hex32(world_stream_previous_active)
|
|
<< "->" << hex32(active)
|
|
<< " next_pc=" << hex32(cpu_.pc);
|
|
if (active != 0u && memory_.contains(active, 52u)) {
|
|
std::cerr << " source=" << hex32(memory_.load32(active + 16u))
|
|
<< " offset=" << memory_.load32(active + 20u)
|
|
<< " remaining=" << memory_.load32(active + 24u)
|
|
<< " progressed=" << memory_.load32(active + 28u)
|
|
<< " callback=" << hex32(memory_.load32(active + 48u));
|
|
}
|
|
std::cerr << "\n";
|
|
world_stream_previous_active = active;
|
|
}
|
|
if (before == 0x08955E7Cu || before == 0x08956258u ||
|
|
before == 0x08956408u || before == 0x089563C0u) {
|
|
std::cerr << "[worlddiag-after] dispatch_pc=" << hex32(before)
|
|
<< " uid_before=" << current_snapshot.thread_uid
|
|
<< " uid_after=" << g_runtime_thread_uid
|
|
<< " manager=" << hex32(world_stream_manager)
|
|
<< " active_after=" << hex32(active)
|
|
<< " next_pc=" << hex32(cpu_.pc) << "\n";
|
|
}
|
|
}
|
|
if (!stopped_ && g_post_dispatch_hook != nullptr)
|
|
g_post_dispatch_hook(*this, cpu_, before, current_snapshot.thread_uid);
|
|
// Starvation preemption must behave identically under diagnostics, or a
|
|
// traced run would stall where a production run makes progress.
|
|
if (g_starvation_hook != nullptr && g_starvation_interval != 0u)
|
|
(void)account_dispatch_work(cpu_, true);
|
|
// A recompiled unit may explicitly redispatch to its own entry through
|
|
// a guest jump table or self-loop. This is legal PSP control flow and
|
|
// must not be confused with a missing PC write. The dispatch cap still
|
|
// bounds genuine infinite loops. Keep the old assertion available as
|
|
// an opt-in diagnostic when investigating generated-code fallthrough.
|
|
if (!stopped_ && cpu_.pc == before && strict_pc_progress) {
|
|
if (trace_on_error) {
|
|
std::cerr << "[unchanged-pc] pc=" << hex32(before)
|
|
<< " function=" << (function_entry != nullptr ? function_entry->name : std::string("unknown"))
|
|
<< " v0=" << hex32(cpu_.gpr[2])
|
|
<< " a0=" << hex32(cpu_.gpr[4])
|
|
<< " a1=" << hex32(cpu_.gpr[5])
|
|
<< " a2=" << hex32(cpu_.gpr[6])
|
|
<< " a3=" << hex32(cpu_.gpr[7])
|
|
<< " t0=" << hex32(cpu_.gpr[8])
|
|
<< " t1=" << hex32(cpu_.gpr[9])
|
|
<< " t2=" << hex32(cpu_.gpr[10])
|
|
<< " t3=" << hex32(cpu_.gpr[11])
|
|
<< " sp=" << hex32(cpu_.gpr[29])
|
|
<< " ra=" << hex32(cpu_.gpr[31]) << "\n";
|
|
}
|
|
stop("Recompiled function returned without changing PC: " + (function_entry != nullptr ? function_entry->name : std::string("unknown")) + " at " + hex32(before));
|
|
}
|
|
}
|
|
if (!stopped_ && max_dispatches != 0u) {
|
|
if (trace_on_error) {
|
|
std::cerr << "[recent-dispatches] dispatch-limit oldest-to-newest count=" << recent_count << "\n";
|
|
const std::size_t first = (recent_next + recent_dispatches.size() - recent_count) % recent_dispatches.size();
|
|
for (std::size_t index = 0; index < recent_count; ++index) {
|
|
const DispatchSnapshot &snapshot = recent_dispatches[(first + index) % recent_dispatches.size()];
|
|
std::cerr << "[recent] uid=" << snapshot.thread_uid
|
|
<< " name=" << snapshot.thread_name.data()
|
|
<< " pc=" << hex32(snapshot.pc)
|
|
<< " a0=" << hex32(snapshot.a0)
|
|
<< " a1=" << hex32(snapshot.a1)
|
|
<< " a2=" << hex32(snapshot.a2)
|
|
<< " a3=" << hex32(snapshot.a3)
|
|
<< " sp=" << hex32(snapshot.sp)
|
|
<< " ra=" << hex32(snapshot.ra) << "\n";
|
|
}
|
|
}
|
|
if (profile_dispatch) {
|
|
std::vector<std::pair<std::uint32_t, std::uint64_t>> hot(dispatch_counts.begin(), dispatch_counts.end());
|
|
std::sort(hot.begin(), hot.end(), [](const auto &left, const auto &right) {
|
|
return left.second > right.second;
|
|
});
|
|
std::cerr << "[profile] dispatch limit pc=" << hex32(cpu_.pc)
|
|
<< " unique=" << hot.size() << "\n";
|
|
const std::size_t count = std::min<std::size_t>(20u, hot.size());
|
|
for (std::size_t index = 0; index < count; ++index) {
|
|
const auto function = functions_.find(hot[index].first);
|
|
std::cerr << "[profile] " << hex32(hot[index].first)
|
|
<< " count=" << hot[index].second
|
|
<< " name=" << (function != functions_.end() ? function->second.name : "unknown")
|
|
<< "\n";
|
|
}
|
|
}
|
|
stop("Dispatch limit reached at " + hex32(cpu_.pc));
|
|
}
|
|
}
|
|
|
|
void Runtime::stop(std::string reason) { stopped_ = true; stop_reason_ = std::move(reason); }
|
|
bool Runtime::stopped() const noexcept { return stopped_; }
|
|
const std::string &Runtime::stop_reason() const noexcept { return stop_reason_; }
|
|
|
|
void Runtime::unsupported(std::uint32_t pc, std::uint32_t instruction, const std::string &reason) {
|
|
stop("Unsupported Allegrex instruction " + hex32(instruction) + " at " + hex32(pc) + ": " + reason);
|
|
}
|
|
|
|
void Runtime::arithmetic_overflow(std::uint32_t pc, std::uint32_t instruction) {
|
|
stop("Allegrex arithmetic overflow for " + hex32(instruction) + " at " + hex32(pc));
|
|
}
|
|
|
|
void Runtime::register_native_fast_path(std::uint32_t address, NativeFastPath function) {
|
|
const std::uint32_t canonical_address = memory_.canonical(address);
|
|
if (!function) {
|
|
native_fast_paths_.erase(canonical_address);
|
|
return;
|
|
}
|
|
native_fast_paths_[canonical_address] = std::move(function);
|
|
}
|
|
|
|
void Runtime::invoke_native_fast_path(std::uint32_t address, AllegrexContext &ctx) {
|
|
const std::uint32_t canonical_address = memory_.canonical(address);
|
|
const std::uint32_t current_pc = memory_.canonical(ctx.pc);
|
|
|
|
// A profile may use this entry at an indirect call site whose target varies.
|
|
// If the current target is not the registered native leaf, preserve the old
|
|
// low-overhead direct-chainable fallback without exposing profile details.
|
|
if (current_pc != canonical_address) {
|
|
if (current_pc < direct_base_) return;
|
|
const std::uint32_t delta = current_pc - direct_base_;
|
|
if ((delta & 3u) != 0u) return;
|
|
const std::size_t index = static_cast<std::size_t>(delta) / 4u;
|
|
if (index >= direct_chainable_.size()) return;
|
|
const RecompiledFunction target_function = direct_chainable_[index];
|
|
if (target_function == nullptr) return;
|
|
target_function(*this, ctx);
|
|
ctx.gpr[0] = 0u;
|
|
return;
|
|
}
|
|
|
|
const auto fast = native_fast_paths_.find(canonical_address);
|
|
if (fast != native_fast_paths_.end()) {
|
|
fast->second(*this, ctx);
|
|
ctx.gpr[0] = 0u;
|
|
return;
|
|
}
|
|
|
|
// Missing profile registration is safe: execute the generated AOT body.
|
|
const RecompiledFunction original = lookup_function(canonical_address);
|
|
if (original == nullptr)
|
|
throw Error("Missing AOT function for native fast path " + hex32(canonical_address));
|
|
original(*this, ctx);
|
|
ctx.gpr[0] = 0u;
|
|
}
|
|
|
|
void Runtime::invoke_import_cached(std::uint32_t slot, std::string_view library,
|
|
std::uint32_t nid, AllegrexContext &ctx) {
|
|
if (hle_histogram_enabled_) ++hle_histogram_[hle_key(library, nid)];
|
|
|
|
const HleFunction *bound = slot < import_bindings_.size() ? import_bindings_[slot] : nullptr;
|
|
if (bound == nullptr) {
|
|
const auto library_it = hle_.find(library);
|
|
const auto function_it = library_it != hle_.end()
|
|
? library_it->second.find(nid) : HleLibrary::const_iterator{};
|
|
if (library_it == hle_.end() || function_it == library_it->second.end()) {
|
|
const std::string library_name(library);
|
|
const auto name = nids_.resolve(library_name, nid).value_or(hex32(nid));
|
|
stop("Missing HLE import " + library_name + "::" + name);
|
|
return;
|
|
}
|
|
if (slot >= import_bindings_.size()) import_bindings_.resize(static_cast<std::size_t>(slot) + 1u, nullptr);
|
|
bound = &function_it->second;
|
|
import_bindings_[slot] = bound;
|
|
}
|
|
|
|
(*bound)(*this, ctx);
|
|
if (!stopped_ && g_post_import_hook != nullptr) g_post_import_hook(*this, ctx);
|
|
}
|
|
|
|
void Runtime::invoke_import(std::string_view library, std::uint32_t nid, AllegrexContext &ctx) {
|
|
// PSPRECOMP_HLE_HISTOGRAM distinguishes a genuine synchronous workload from
|
|
// a kernel-wait livelock: real translated work barely calls into the HLE,
|
|
// while a thread spinning on an operation the host never completes shows up
|
|
// as a huge count on one or two wait imports.
|
|
if (hle_histogram_enabled_) ++hle_histogram_[hle_key(library, nid)];
|
|
const auto library_it = hle_.find(library);
|
|
const auto function_it = library_it != hle_.end()
|
|
? library_it->second.find(nid) : HleLibrary::const_iterator{};
|
|
if (library_it == hle_.end() || function_it == library_it->second.end()) {
|
|
const std::string library_name(library);
|
|
const auto name = nids_.resolve(library_name, nid).value_or(hex32(nid));
|
|
stop("Missing HLE import " + library_name + "::" + name);
|
|
return;
|
|
}
|
|
function_it->second(*this, ctx);
|
|
if (!stopped_ && g_post_import_hook != nullptr) g_post_import_hook(*this, ctx);
|
|
}
|
|
|
|
AllegrexContext &Runtime::cpu() noexcept { return cpu_; }
|
|
const AllegrexContext &Runtime::cpu() const noexcept { return cpu_; }
|
|
|
|
} // namespace psprecomp
|