mirror of
https://github.com/patchzyy/wiicompiled
synced 2026-09-10 17:16:47 -04:00
patchzyy
795 lines
32 KiB
C++
795 lines
32 KiB
C++
#pragma once
|
|
#include "memory.h"
|
|
#include "ppc_runtime.h"
|
|
#include "system_bridge.h"
|
|
#include "game_graphics_options.h"
|
|
#include "runtime_log.h"
|
|
|
|
#include <array>
|
|
#include <atomic>
|
|
#include <cstddef>
|
|
#include <cstdint>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <iostream>
|
|
#include <optional>
|
|
#include <stdexcept>
|
|
#include <sstream>
|
|
#include <tuple>
|
|
#include <type_traits>
|
|
|
|
inline void InvokeIndirectCpu(uint32_t target, CpuContext* ctx);
|
|
|
|
// Some game-facing runtime options alter arguments at well-defined ABI
|
|
// boundaries. Keep this independent of the dispatch mechanism: generated
|
|
// static calls deliberately bypass InvokeDirectCpu for performance.
|
|
// (used for the path mask filtering in ScnRenderer::createPath: depth of
|
|
// field is always removed, bloom when the user disabled it)
|
|
inline void ApplyRuntimeCallOptions(uint32_t target, CpuContext* ctx) {
|
|
if (target == 0x8023BD38u) {
|
|
// ScnRenderer::createPath receives the post-processing path mask in r4.
|
|
ctx->gpr[4] = RuntimeGameGraphicsOptions::FilterScnRendererPathMask(ctx->gpr[4]);
|
|
}
|
|
}
|
|
|
|
// Persistent per-thread CPU context used across translated function calls.
|
|
CpuContext& GetPersistentCpuContext();
|
|
void InitializePersistentCpuContext();
|
|
|
|
enum class FunctionKind : uint8_t {
|
|
BaseTranslated = 0,
|
|
ModTranslated = 1,
|
|
Native = 2,
|
|
};
|
|
|
|
inline constexpr uint32_t kPpcAllNonvolatileFprMask = 0xffffc000u;
|
|
inline constexpr uint32_t kBaseTranslatedFunctionPriority = 0;
|
|
inline constexpr uint32_t kModTranslatedFunctionPriorityBase = 100;
|
|
inline constexpr uint32_t kNativeFunctionPriority = 10000;
|
|
|
|
#include "isa/ppc_isa_cr.h"
|
|
|
|
struct TranslatedFunctionInfo {
|
|
uint32_t address = 0;
|
|
const char* name = "";
|
|
uint64_t moduleId = 0;
|
|
uint32_t priority = 0;
|
|
uint32_t nonvolatileFprWriteMask = kPpcAllNonvolatileFprMask;
|
|
void* entryPoint = nullptr; // Raw typed function pointer for indirect-call resolution
|
|
void (*rawCpuInvoker)(CpuContext*) = nullptr;
|
|
bool mustRemainDynamicallyDispatchable = true;
|
|
FunctionKind kind = FunctionKind::BaseTranslated;
|
|
};
|
|
|
|
struct RawDispatchRecord {
|
|
uint32_t address = 0;
|
|
void (*entry)(CpuContext*) = nullptr;
|
|
uint32_t nonvolatileFprWriteMask = kPpcAllNonvolatileFprMask;
|
|
bool preserveNonvolatileGprs = false;
|
|
};
|
|
|
|
// Translator-owned indirect dispatch is split by the high address byte and
|
|
// then by 4 KiB guest pages. Each populated segment stores only the contiguous
|
|
// page range that contains verified function entry points. The final search is
|
|
// bounded to at most the 1024 aligned instruction addresses in one guest page.
|
|
struct StaticIndirectDispatchPage {
|
|
uint32_t firstEntry = 0;
|
|
uint16_t entryCount = 0;
|
|
// Explicit tail padding to an 8-byte stride. The emitter writes it as a
|
|
// literal third initializer, so it is part of the generated-data contract.
|
|
uint16_t reserved = 0;
|
|
};
|
|
|
|
struct StaticIndirectDispatchSegment {
|
|
const StaticIndirectDispatchPage* pages = nullptr;
|
|
uint16_t firstPage = 0;
|
|
uint16_t pageCount = 0;
|
|
};
|
|
|
|
struct StaticIndirectDispatchTable {
|
|
const char* profileName = nullptr;
|
|
const StaticIndirectDispatchSegment* segments = nullptr; // Exactly 256 high-byte segments.
|
|
const RawDispatchRecord* entries = nullptr;
|
|
size_t entryCount = 0;
|
|
};
|
|
|
|
inline const RawDispatchRecord* FindStaticIndirectDispatchEntry(
|
|
const StaticIndirectDispatchTable* table,
|
|
uint32_t address) noexcept {
|
|
if (!table || !table->segments || !table->entries || table->entryCount == 0) {
|
|
return nullptr;
|
|
}
|
|
|
|
const auto& segment = table->segments[address >> 24];
|
|
const uint32_t pageNumber = (address >> 12) & 0x0FFFu;
|
|
if (!segment.pages || pageNumber < segment.firstPage) {
|
|
return nullptr;
|
|
}
|
|
const uint32_t relativePage = pageNumber - segment.firstPage;
|
|
if (relativePage >= segment.pageCount) {
|
|
return nullptr;
|
|
}
|
|
|
|
const auto& page = segment.pages[relativePage];
|
|
const size_t first = page.firstEntry;
|
|
const size_t count = page.entryCount;
|
|
if (first > table->entryCount || count > table->entryCount - first) {
|
|
return nullptr;
|
|
}
|
|
|
|
size_t lower = first;
|
|
size_t upper = first + count;
|
|
while (lower < upper) {
|
|
const size_t middle = lower + (upper - lower) / 2;
|
|
const uint32_t candidate = table->entries[middle].address;
|
|
if (candidate < address) {
|
|
lower = middle + 1;
|
|
} else {
|
|
upper = middle;
|
|
}
|
|
}
|
|
return lower < first + count && table->entries[lower].address == address
|
|
? &table->entries[lower]
|
|
: nullptr;
|
|
}
|
|
|
|
void RegisterStaticIndirectDispatchTable(const StaticIndirectDispatchTable* table);
|
|
inline std::atomic<const StaticIndirectDispatchTable*> g_publishedStaticIndirectDispatchTable{nullptr};
|
|
|
|
class StaticIndirectDispatchTableRegistrar {
|
|
public:
|
|
explicit StaticIndirectDispatchTableRegistrar(const StaticIndirectDispatchTable* table) {
|
|
RegisterStaticIndirectDispatchTable(table);
|
|
}
|
|
};
|
|
|
|
inline unsigned PpcLowestSetBitIndex(uint32_t value) noexcept {
|
|
return static_cast<unsigned>(__builtin_ctz(value));
|
|
}
|
|
|
|
class PpcNonvolatileFprGuard {
|
|
public:
|
|
explicit PpcNonvolatileFprGuard(CpuContext* cpu, uint32_t mask = kPpcAllNonvolatileFprMask) noexcept
|
|
: cpu_(cpu), mask_(mask & kPpcAllNonvolatileFprMask) {
|
|
if (!cpu_ || mask_ == 0) {
|
|
return;
|
|
}
|
|
// Real masks are a single contiguous register run - f14..f31 for the
|
|
// conservative default and f23..f31 or similar for a measured writer -
|
|
// so resolve the run once and copy it as one straight-line block instead
|
|
// of testing 18 mask bits. Any other mask keeps the per-bit loop, which
|
|
// saves and restores exactly the same registers.
|
|
const uint32_t lowest = mask_ & (0u - mask_);
|
|
const uint32_t aboveRun = mask_ + lowest;
|
|
if ((aboveRun & (aboveRun - 1u)) == 0u) {
|
|
const unsigned first = PpcLowestSetBitIndex(mask_);
|
|
const unsigned end = aboveRun == 0u ? 32u : PpcLowestSetBitIndex(aboveRun);
|
|
firstSlot_ = static_cast<uint8_t>(first - 14u);
|
|
slotCount_ = static_cast<uint8_t>(end - first);
|
|
const size_t count = slotCount_;
|
|
for (size_t i = 0; i < count; ++i) {
|
|
saved_[firstSlot_ + i] = cpu_->fpr[first + i];
|
|
}
|
|
return;
|
|
}
|
|
for (size_t i = 0; i < saved_.size(); ++i) {
|
|
if ((mask_ & (1u << (14 + i))) == 0) {
|
|
continue;
|
|
}
|
|
saved_[i] = cpu_->fpr[14 + i];
|
|
}
|
|
}
|
|
|
|
~PpcNonvolatileFprGuard() noexcept {
|
|
if (!cpu_ || mask_ == 0) {
|
|
return;
|
|
}
|
|
if (slotCount_ != 0) {
|
|
const unsigned first = 14u + firstSlot_;
|
|
const size_t count = slotCount_;
|
|
for (size_t i = 0; i < count; ++i) {
|
|
cpu_->fpr[first + i] = saved_[firstSlot_ + i];
|
|
}
|
|
return;
|
|
}
|
|
for (size_t i = 0; i < saved_.size(); ++i) {
|
|
if ((mask_ & (1u << (14 + i))) == 0) {
|
|
continue;
|
|
}
|
|
cpu_->fpr[14 + i] = saved_[i];
|
|
}
|
|
}
|
|
|
|
PpcNonvolatileFprGuard(const PpcNonvolatileFprGuard&) = delete;
|
|
PpcNonvolatileFprGuard& operator=(const PpcNonvolatileFprGuard&) = delete;
|
|
|
|
private:
|
|
CpuContext* cpu_ = nullptr;
|
|
uint32_t mask_ = 0;
|
|
// Contiguous-run description of mask_. slotCount_ == 0 means the mask is not
|
|
// a single run, so the destructor mirrors the constructor's per-bit loop.
|
|
uint8_t firstSlot_ = 0;
|
|
uint8_t slotCount_ = 0;
|
|
// Every element selected by mask_ is written before it is read. Leaving the
|
|
// remaining slots uninitialized avoids zero-filling all 18 values around a
|
|
// call that may preserve only one or two architectural FPRs.
|
|
std::array<PPC_FPR, 18> saved_;
|
|
};
|
|
|
|
class PpcNonvolatileGprGuard {
|
|
public:
|
|
explicit PpcNonvolatileGprGuard(CpuContext* cpu, bool enabled = true) noexcept
|
|
: cpu_(enabled ? cpu : nullptr) {
|
|
if (!cpu_) {
|
|
return;
|
|
}
|
|
for (size_t i = 0; i < saved_.size(); ++i) {
|
|
saved_[i] = cpu_->gpr[14 + i];
|
|
}
|
|
}
|
|
|
|
~PpcNonvolatileGprGuard() noexcept {
|
|
if (!cpu_) {
|
|
return;
|
|
}
|
|
for (size_t i = 0; i < saved_.size(); ++i) {
|
|
cpu_->gpr[14 + i] = saved_[i];
|
|
}
|
|
}
|
|
|
|
PpcNonvolatileGprGuard(const PpcNonvolatileGprGuard&) = delete;
|
|
PpcNonvolatileGprGuard& operator=(const PpcNonvolatileGprGuard&) = delete;
|
|
|
|
private:
|
|
CpuContext* cpu_ = nullptr;
|
|
// Left uninitialized: enabled writes all 18 slots before reading them, disabled reads none,
|
|
// so skipping the zero-fill saves 72 bytes of stack work on the common disabled hot path.
|
|
std::array<uint32_t, 18> saved_;
|
|
};
|
|
|
|
// Guest address of OSLoadContext, which models rfi-style context restoration
|
|
// and intentionally replaces the full guest register file instead of returning
|
|
// like a normal ABI call. It can never be guarded.
|
|
inline constexpr uint32_t kOSLoadContextAddress = 0x801A1F58u;
|
|
|
|
inline bool ShouldPreserveNonvolatileGprsForRawCpuCall(const TranslatedFunctionInfo* info) noexcept {
|
|
return info->kind == FunctionKind::Native && info->address != kOSLoadContextAddress;
|
|
}
|
|
|
|
inline uint32_t NonvolatileFprGuardMaskFor(const TranslatedFunctionInfo* info) noexcept {
|
|
if (!info || info->nonvolatileFprWriteMask == 0) {
|
|
return 0;
|
|
}
|
|
return info->nonvolatileFprWriteMask & kPpcAllNonvolatileFprMask;
|
|
}
|
|
|
|
template <uint32_t Target>
|
|
struct KnownTranslatedCpuCall {
|
|
static constexpr bool kAvailable = false;
|
|
static constexpr uint32_t kNonvolatileFprWriteMask = kPpcAllNonvolatileFprMask;
|
|
static constexpr bool kMustRemainDynamicallyDispatchable = true;
|
|
static constexpr void (*Entry)(CpuContext*) = nullptr;
|
|
};
|
|
|
|
// Translator-emitted trait specializations, one macro shared by every generated shard.
|
|
// `addr` is the 8-digit hex entry point, `winner` the resolved symbol (mods/Retro Rewind can
|
|
// publish their own name). A specialization only exists for a single statically bound winner,
|
|
// so overridable/dynamically-dispatchable are hardcoded here rather than passed in.
|
|
#define MKW_TRANSLATED_TRAIT(addr, winner, nonvolatile_fpr_write_mask) \
|
|
extern "C" void winner(CpuContext* ctx); \
|
|
template <> \
|
|
struct KnownTranslatedCpuCall<0x##addr##u> { \
|
|
static constexpr bool kAvailable = true; \
|
|
static constexpr uint32_t kNonvolatileFprWriteMask = nonvolatile_fpr_write_mask; \
|
|
static constexpr bool kMustRemainDynamicallyDispatchable = false; \
|
|
static constexpr void (*Entry)(CpuContext*) = &winner; \
|
|
}
|
|
|
|
template <uint32_t Target>
|
|
struct KnownNativeCpuCall {
|
|
static constexpr bool kAvailable = false;
|
|
static constexpr uint32_t kNonvolatileFprWriteMask = kPpcAllNonvolatileFprMask;
|
|
static constexpr void (*Entry)(CpuContext*) = nullptr;
|
|
};
|
|
|
|
template <uint32_t Target>
|
|
struct KnownTypedNativeCpuCall {
|
|
static constexpr bool kAvailable = false;
|
|
};
|
|
|
|
#include "native_cpu_calls.inc"
|
|
|
|
// Direct-mapped thread-local memo in front of the sorted registry lookup every bctrl performs;
|
|
// a miss just re-runs the sorted lookup. 512 entries (12 KiB) covers the per-frame indirect
|
|
// working set while staying L1/L2 resident, unlike 4096 which would thrash L2. Must stay a
|
|
// power of two, the index masks with (size - 1).
|
|
inline constexpr size_t kIndirectDispatchCacheEntries = 512;
|
|
|
|
// Namespace-scope `inline thread_local`, not function-local `static thread_local`, to avoid a
|
|
// thread-static init epoch check on every bctrl (same as g_currentCpuContext in ppc_runtime.h).
|
|
struct IndirectResolvedDispatchMemoEntry {
|
|
bool valid;
|
|
uint32_t address;
|
|
const TranslatedFunctionInfo* info;
|
|
};
|
|
inline thread_local IndirectResolvedDispatchMemoEntry
|
|
g_indirectResolvedDispatchMemo[kIndirectDispatchCacheEntries]{};
|
|
|
|
struct IndirectRawDispatchMemoEntry {
|
|
bool valid;
|
|
uint32_t address;
|
|
const RawDispatchRecord* record;
|
|
};
|
|
inline thread_local IndirectRawDispatchMemoEntry
|
|
g_indirectRawDispatchMemo[kIndirectDispatchCacheEntries]{};
|
|
|
|
class TranslatedFunctionRegistry {
|
|
public:
|
|
static void Register(TranslatedFunctionInfo info);
|
|
static void Finalize();
|
|
static inline bool IsLookupPublished() noexcept {
|
|
return lookupPublished_.load(std::memory_order_acquire);
|
|
}
|
|
static std::optional<TranslatedFunctionInfo> FindNearestByAddress(uint32_t address);
|
|
static inline const TranslatedFunctionInfo* FindByAddressPtr(uint32_t address) {
|
|
if (!lookupPublished_.load(std::memory_order_acquire)) {
|
|
return FindByAddressPtrSlow(address);
|
|
}
|
|
|
|
auto& cached = g_indirectResolvedDispatchMemo[
|
|
((address >> 2) * 2654435761u) & (kIndirectDispatchCacheEntries - 1)];
|
|
if (cached.valid && cached.address == address) {
|
|
return cached.info;
|
|
}
|
|
|
|
const auto* info = FindByAddressPtrSlow(address);
|
|
cached = {true, address, info};
|
|
return info;
|
|
}
|
|
|
|
// Hot wrapper: only the direct-mapped memo probe stays inline. Inlining the
|
|
// generated-table binary search alongside it pushed the whole function past
|
|
// the inliner's budget and out-of-lined the probe itself, which is the part
|
|
// every bctrl executes.
|
|
static MKW_PPC_FORCE_INLINE const RawDispatchRecord* FindRawByAddressPtr(uint32_t address) {
|
|
if (!lookupPublished_.load(std::memory_order_acquire)) {
|
|
return nullptr;
|
|
}
|
|
|
|
auto& cached = g_indirectRawDispatchMemo[
|
|
((address >> 2) * 2654435761u) & (kIndirectDispatchCacheEntries - 1)];
|
|
if (cached.valid && cached.address == address) {
|
|
return cached.record;
|
|
}
|
|
|
|
return FindRawByAddressPtrMiss(address, cached);
|
|
}
|
|
// Look up a registered function by host (native) address.
|
|
// This enables stack trace symbolication for translated functions.
|
|
static std::optional<TranslatedFunctionInfo> FindByHostAddress(uintptr_t hostAddr);
|
|
|
|
private:
|
|
// Memo miss: resolve against the header-inline generated table first; only dynamically
|
|
// registered records or a not-yet-published table fall through to the slow path.
|
|
static MKW_PPC_NO_INLINE MKW_PPC_COLD const RawDispatchRecord* FindRawByAddressPtrMiss(
|
|
uint32_t address, IndirectRawDispatchMemoEntry& cached) {
|
|
const RawDispatchRecord* record = nullptr;
|
|
if (const auto* table =
|
|
g_publishedStaticIndirectDispatchTable.load(std::memory_order_acquire)) {
|
|
record = FindStaticIndirectDispatchEntry(table, address);
|
|
}
|
|
if (record == nullptr) {
|
|
record = FindRawByAddressPtrSlow(address);
|
|
}
|
|
cached = {true, address, record};
|
|
return record;
|
|
}
|
|
|
|
static const TranslatedFunctionInfo* FindByAddressPtrSlow(uint32_t address);
|
|
static const RawDispatchRecord* FindRawByAddressPtrSlow(uint32_t address);
|
|
|
|
// Finalize publishes immutable registry, generated-table, and dynamic raw
|
|
// dispatch storage. Header-inlined cache hits are enabled only after that
|
|
// release publication, so a pre-finalization miss can never poison them.
|
|
inline static std::atomic_bool lookupPublished_{false};
|
|
};
|
|
|
|
// Cold, profile-owned registration data is emitted in compact table shards.
|
|
// Keeping it out of generated function bodies avoids one static constructor
|
|
// per translated function.
|
|
struct BulkTranslatedFunctionRecord {
|
|
uint32_t address;
|
|
const char* name;
|
|
void (*entry)(CpuContext*);
|
|
FunctionKind kind;
|
|
bool preservesNonvolatileFprs;
|
|
uint32_t nonvolatileFprWriteMask;
|
|
uint32_t priority;
|
|
uint64_t moduleId;
|
|
bool mustRemainDynamicallyDispatchable;
|
|
};
|
|
|
|
void RegisterBulkTranslatedFunctions(const BulkTranslatedFunctionRecord* records, size_t count);
|
|
|
|
class BulkTranslatedFunctionRegistrar {
|
|
public:
|
|
BulkTranslatedFunctionRegistrar(const BulkTranslatedFunctionRecord* records, size_t count) {
|
|
RegisterBulkTranslatedFunctions(records, count);
|
|
}
|
|
};
|
|
|
|
MKW_PPC_FORCE_INLINE bool TryDispatchResolvedCpuTarget(const TranslatedFunctionInfo* info, CpuContext* cpu) {
|
|
if (!info || !info->rawCpuInvoker) {
|
|
return false;
|
|
}
|
|
|
|
RecompMod::ScopedTranslatedExecutionAddress translatedExecution(info->address);
|
|
PpcNonvolatileFprGuard fprGuard(cpu, NonvolatileFprGuardMaskFor(info));
|
|
PpcNonvolatileGprGuard gprGuard(cpu, ShouldPreserveNonvolatileGprsForRawCpuCall(info));
|
|
if (TryGetCpuContext() != cpu) {
|
|
CpuContextScope scope(cpu);
|
|
info->rawCpuInvoker(cpu);
|
|
return true;
|
|
}
|
|
info->rawCpuInvoker(cpu);
|
|
return true;
|
|
}
|
|
|
|
inline bool TryDispatchRawCpuTarget(const RawDispatchRecord* record, CpuContext* cpu) {
|
|
if (!record || !record->entry) {
|
|
return false;
|
|
}
|
|
RecompMod::ScopedTranslatedExecutionAddress translatedExecution(record->address);
|
|
PpcNonvolatileGprGuard gprGuard(cpu, record->preserveNonvolatileGprs);
|
|
PpcNonvolatileFprGuard fprGuard(cpu, record->nonvolatileFprWriteMask);
|
|
if (TryGetCpuContext() != cpu) {
|
|
CpuContextScope scope(cpu);
|
|
record->entry(cpu);
|
|
} else {
|
|
record->entry(cpu);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
template <uint32_t Target>
|
|
inline void DispatchKnownTranslatedCpuTargetStatic(CpuContext* cpu) {
|
|
const auto invokeKnownTranslated = [&]() {
|
|
KnownTranslatedCpuCall<Target>::Entry(cpu);
|
|
};
|
|
|
|
// A statically resolved, non-overridable translated call already running
|
|
// in this CpuContext needs no registry, context, or diagnostic target
|
|
// transition. This is the normal generated-to-generated hot path.
|
|
if (TryGetCpuContext() == cpu) {
|
|
if constexpr (KnownTranslatedCpuCall<Target>::kNonvolatileFprWriteMask == 0) {
|
|
invokeKnownTranslated();
|
|
return;
|
|
} else {
|
|
PpcNonvolatileFprGuard fprGuard(
|
|
cpu, KnownTranslatedCpuCall<Target>::kNonvolatileFprWriteMask);
|
|
invokeKnownTranslated();
|
|
return;
|
|
}
|
|
}
|
|
|
|
CpuContextScope contextScope(cpu);
|
|
RecompMod::ScopedTranslatedExecutionAddress translatedExecution(Target);
|
|
if constexpr (KnownTranslatedCpuCall<Target>::kNonvolatileFprWriteMask == 0) {
|
|
invokeKnownTranslated();
|
|
return;
|
|
} else {
|
|
PpcNonvolatileFprGuard fprGuard(cpu, KnownTranslatedCpuCall<Target>::kNonvolatileFprWriteMask);
|
|
invokeKnownTranslated();
|
|
}
|
|
}
|
|
|
|
[[noreturn]] inline void ReportMissingCpuTarget(uint32_t target, CpuContext* cpu) {
|
|
// Centralized crash path: stderr diagnostics, heuristics, crash artifacts
|
|
// in the run's log folder, popup, exit.
|
|
RuntimeCrash::FatalMissingGuestTarget(target, cpu);
|
|
}
|
|
|
|
template <uint32_t Target>
|
|
inline const TranslatedFunctionInfo* ResolveDirectCpuTargetInfo() {
|
|
return TranslatedFunctionRegistry::FindByAddressPtr(Target);
|
|
}
|
|
|
|
template <uint32_t Target>
|
|
inline const TranslatedFunctionInfo* ResolveDirectCpuTargetInfoCached() {
|
|
// Only latch a result once the registry is published (immutable for the process lifetime);
|
|
// a pre-finalization miss or provisional winner must never be cached. 0 = not yet resolved,
|
|
// 1 = negative-cache sentinel, avoiding a second ready flag on the hot path.
|
|
static std::atomic<uintptr_t> cachedValue{0};
|
|
constexpr uintptr_t kNegativeCache = 1;
|
|
const uintptr_t cached = cachedValue.load(std::memory_order_acquire);
|
|
if (cached != 0) {
|
|
return cached == kNegativeCache
|
|
? nullptr
|
|
: reinterpret_cast<const TranslatedFunctionInfo*>(cached);
|
|
}
|
|
if (!TranslatedFunctionRegistry::IsLookupPublished()) {
|
|
return ResolveDirectCpuTargetInfo<Target>();
|
|
}
|
|
const auto* info = ResolveDirectCpuTargetInfo<Target>();
|
|
cachedValue.store(info != nullptr
|
|
? reinterpret_cast<uintptr_t>(info)
|
|
: kNegativeCache,
|
|
std::memory_order_release);
|
|
return info;
|
|
}
|
|
|
|
// Caches the "is the published winner still base translation?" verdict for state-free fast
|
|
// paths guarding tens of thousands of call sites. Same latching rule as above: 0 = unresolved,
|
|
// 1 = negative, 2 = positive, and a verdict computed pre-publication is never stored.
|
|
template <uint32_t Target>
|
|
inline bool IsBaseTranslatedCpuTargetActive() {
|
|
static std::atomic<uint8_t> cachedVerdict{0};
|
|
constexpr uint8_t kNegativeVerdict = 1;
|
|
constexpr uint8_t kPositiveVerdict = 2;
|
|
const uint8_t cached = cachedVerdict.load(std::memory_order_acquire);
|
|
if (cached != 0) {
|
|
return cached == kPositiveVerdict;
|
|
}
|
|
const bool publishable = TranslatedFunctionRegistry::IsLookupPublished();
|
|
const auto* info = ResolveDirectCpuTargetInfo<Target>();
|
|
const bool active = info != nullptr && info->kind == FunctionKind::BaseTranslated;
|
|
if (publishable) {
|
|
cachedVerdict.store(active ? kPositiveVerdict : kNegativeVerdict,
|
|
std::memory_order_release);
|
|
}
|
|
return active;
|
|
}
|
|
|
|
template <uint32_t Target>
|
|
inline void InvokeDirectCpu(CpuContext* ctx) {
|
|
static_assert(Target != 0, "InvokeDirectCpu cannot target address 0");
|
|
CpuContext* cpu = ctx ? ctx : &GetPersistentCpuContext();
|
|
ApplyRuntimeCallOptions(Target, cpu);
|
|
if constexpr (KnownNativeCpuCall<Target>::kAvailable) {
|
|
const auto invokeKnownNative = [&]() {
|
|
PpcNonvolatileGprGuard gprGuard(cpu);
|
|
if (TryGetCpuContext() != cpu) {
|
|
CpuContextScope scope(cpu);
|
|
KnownNativeCpuCall<Target>::Entry(cpu);
|
|
return;
|
|
}
|
|
KnownNativeCpuCall<Target>::Entry(cpu);
|
|
};
|
|
if constexpr (KnownNativeCpuCall<Target>::kNonvolatileFprWriteMask == 0) {
|
|
invokeKnownNative();
|
|
} else {
|
|
PpcNonvolatileFprGuard fprGuard(cpu, KnownNativeCpuCall<Target>::kNonvolatileFprWriteMask);
|
|
invokeKnownNative();
|
|
}
|
|
return;
|
|
}
|
|
|
|
if constexpr (KnownTypedNativeCpuCall<Target>::kAvailable) {
|
|
PpcNonvolatileGprGuard gprGuard(cpu);
|
|
if (TryGetCpuContext() != cpu) {
|
|
CpuContextScope scope(cpu);
|
|
KnownTypedNativeCpuCall<Target>::Invoke(cpu);
|
|
} else {
|
|
KnownTypedNativeCpuCall<Target>::Invoke(cpu);
|
|
}
|
|
return;
|
|
}
|
|
|
|
if constexpr (KnownTranslatedCpuCall<Target>::kAvailable) {
|
|
DispatchKnownTranslatedCpuTargetStatic<Target>(cpu);
|
|
return;
|
|
}
|
|
|
|
const auto* registeredInfo = ResolveDirectCpuTargetInfoCached<Target>();
|
|
if (TryDispatchResolvedCpuTarget(registeredInfo, cpu)) {
|
|
return;
|
|
}
|
|
|
|
ReportMissingCpuTarget(Target, cpu);
|
|
}
|
|
|
|
// Indirect jump (bctr without link) - used for switch tables.
|
|
inline void InvokeIndirectJump(uint32_t target, CpuContext* ctx) {
|
|
// Use the caller's context, not the persistent global context!
|
|
// This is essential for tail calls via bctr where the 'this' pointer (r3)
|
|
// must be passed correctly to the target function.
|
|
CpuContext* cpu = ctx ? ctx : &GetPersistentCpuContext();
|
|
if (TryDispatchRawCpuTarget(TranslatedFunctionRegistry::FindRawByAddressPtr(target), cpu)) {
|
|
return;
|
|
}
|
|
|
|
// Cold fallback for dynamically registered or signature-only targets.
|
|
const auto* info = TranslatedFunctionRegistry::FindByAddressPtr(target);
|
|
if (info) {
|
|
if (TryDispatchResolvedCpuTarget(info, cpu)) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
// If not registered, this is likely a switch table jump to an intra-function label.
|
|
// The translator should have recognized this pattern and generated proper switch code.
|
|
RT_LOG(RT_TAG_RUNTIME) << "InvokeIndirectJump: target 0x"
|
|
<< std::hex << target << std::dec
|
|
<< " is not a registered function.\n"
|
|
<< "This is likely a switch statement jump table that the translator\n"
|
|
<< "failed to recognize. The function containing this bctr needs\n"
|
|
<< "proper switch pattern recognition." << std::endl;
|
|
RT_LOG(RT_TAG_RUNTIME) << "Indirect jump context: pc=0x" << std::hex << cpu->pc
|
|
<< " lr=0x" << cpu->lr << " ctr=0x" << cpu->ctr
|
|
<< " r1=0x" << cpu->gpr[1] << " r3=0x" << cpu->gpr[3]
|
|
<< " r29=0x" << cpu->gpr[29] << " r30=0x" << cpu->gpr[30]
|
|
<< " r31=0x" << cpu->gpr[31] << std::dec << std::endl;
|
|
DumpHostStackTraceForRuntimeHelper();
|
|
std::fflush(stderr);
|
|
std::ostringstream message;
|
|
message << "The game stopped because an indirect jump targeted guest address 0x"
|
|
<< std::hex << target
|
|
<< ", but that address is not a registered translated function.\n\n"
|
|
<< "The translator may have missed a switch/jump-table target.";
|
|
// Same artifact set as RuntimeCrash::FatalMissingGuestTarget, which is this
|
|
// path's sibling for indirect *calls*: MarkFatalErrorReported below stops the
|
|
// atexit reporter, so the crash log has to be written here.
|
|
RuntimeCrash::WriteCrashArtifacts("missing_jump_target", message.str(), &target);
|
|
SetRuntimeExitCode(EXIT_FAILURE);
|
|
ShowRuntimeFatalPopup("Missing indirect jump target", message.str());
|
|
MarkFatalErrorReported();
|
|
std::exit(EXIT_FAILURE);
|
|
}
|
|
|
|
inline void InvokeIndirectCpu(uint32_t target, CpuContext* ctx) {
|
|
CpuContext* cpu = ctx ? ctx : &GetPersistentCpuContext();
|
|
if (target == 0) {
|
|
ReportMissingCpuTarget(target, cpu);
|
|
}
|
|
ApplyRuntimeCallOptions(target, cpu);
|
|
if (TryDispatchRawCpuTarget(TranslatedFunctionRegistry::FindRawByAddressPtr(target), cpu)) {
|
|
return;
|
|
}
|
|
if (TryDispatchResolvedCpuTarget(TranslatedFunctionRegistry::FindByAddressPtr(target), cpu)) {
|
|
return;
|
|
}
|
|
ReportMissingCpuTarget(target, cpu);
|
|
}
|
|
|
|
template <typename Signature>
|
|
class AbiTrampoline;
|
|
|
|
template <typename Ret, typename... Args>
|
|
class AbiTrampoline<Ret(Args...)> {
|
|
public:
|
|
using TargetFn = Ret (*)(Args...);
|
|
|
|
AbiTrampoline(uint32_t address,
|
|
const char* name,
|
|
TargetFn target,
|
|
FunctionKind kind = FunctionKind::BaseTranslated,
|
|
bool preservesNonvolatileFprs = false,
|
|
uint32_t nonvolatileFprWriteMask = kPpcAllNonvolatileFprMask,
|
|
uint32_t priority = 0,
|
|
uint64_t moduleId = 0,
|
|
void (*rawCpuInvoker)(CpuContext*) = nullptr,
|
|
bool mustRemainDynamicallyDispatchable = true)
|
|
{
|
|
TranslatedFunctionInfo info;
|
|
info.address = address;
|
|
info.name = name ? name : "";
|
|
info.moduleId = moduleId;
|
|
info.priority = priority;
|
|
const bool effectivePreservesNonvolatileFprs =
|
|
preservesNonvolatileFprs || kind == FunctionKind::Native;
|
|
info.nonvolatileFprWriteMask = effectivePreservesNonvolatileFprs ? 0 : (nonvolatileFprWriteMask & kPpcAllNonvolatileFprMask);
|
|
info.entryPoint = reinterpret_cast<void*>(target);
|
|
info.rawCpuInvoker = rawCpuInvoker;
|
|
info.mustRemainDynamicallyDispatchable = mustRemainDynamicallyDispatchable;
|
|
info.kind = kind;
|
|
TranslatedFunctionRegistry::Register(std::move(info));
|
|
}
|
|
|
|
static std::tuple<Args...> BuildArgs(CpuContext* cpu) {
|
|
size_t gprIndex = 0;
|
|
size_t fprIndex = 0;
|
|
return std::tuple<Args...>{LoadArgument<Args>(cpu, gprIndex, fprIndex)...};
|
|
}
|
|
|
|
template <typename T>
|
|
static std::remove_reference_t<T> LoadArgument(CpuContext* cpu, size_t& gprIndex, size_t& fprIndex) {
|
|
using CleanT = std::remove_reference_t<T>;
|
|
if constexpr (std::is_floating_point_v<CleanT>) {
|
|
if (fprIndex >= 13) {
|
|
throw std::out_of_range("FPR argument overflow");
|
|
}
|
|
return static_cast<CleanT>(cpu->fpr[1 + fprIndex++].d);
|
|
} else {
|
|
if (gprIndex >= 8) {
|
|
size_t stackArgIndex = gprIndex - 8;
|
|
uint32_t sp = cpu->gpr[1];
|
|
uint32_t argAddr = sp + 8 + static_cast<uint32_t>(stackArgIndex * 4);
|
|
gprIndex++;
|
|
try {
|
|
return static_cast<CleanT>(Memory::Read32(argAddr));
|
|
} catch (const Memory::AccessViolation&) {
|
|
RT_LOG(RT_TAG_RUNTIME) << "Stack parameter read failed at 0x" << std::hex << argAddr
|
|
<< " (SP=0x" << sp << ", argIndex=" << std::dec << (gprIndex - 1) << ")" << std::endl;
|
|
throw;
|
|
}
|
|
}
|
|
return static_cast<CleanT>(cpu->gpr[3 + gprIndex++]);
|
|
}
|
|
}
|
|
|
|
static void InvokeCpu(TargetFn target, CpuContext* cpu) {
|
|
auto args = BuildArgs(cpu);
|
|
if constexpr (std::is_void_v<Ret>) {
|
|
std::apply(target, args);
|
|
} else {
|
|
auto result = std::apply(target, args);
|
|
if constexpr (std::is_floating_point_v<Ret>) {
|
|
cpu->fpr[1].d = static_cast<double>(result);
|
|
} else {
|
|
cpu->gpr[3] = static_cast<uint32_t>(result);
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
template <auto Target>
|
|
struct AbiRawCpuThunk {
|
|
static void Invoke(CpuContext* cpu) {
|
|
if constexpr (std::is_invocable_v<decltype(Target), CpuContext*>) {
|
|
(void)Target(cpu);
|
|
} else {
|
|
AbiTrampoline<std::remove_pointer_t<decltype(Target)>>::InvokeCpu(Target, cpu);
|
|
}
|
|
}
|
|
};
|
|
|
|
template <typename Ret>
|
|
class AbiTrampoline<Ret(CpuContext*)> {
|
|
public:
|
|
using TargetFn = Ret (*)(CpuContext*);
|
|
|
|
AbiTrampoline(uint32_t address,
|
|
const char* name,
|
|
TargetFn target,
|
|
FunctionKind kind = FunctionKind::BaseTranslated,
|
|
bool preservesNonvolatileFprs = false,
|
|
uint32_t nonvolatileFprWriteMask = kPpcAllNonvolatileFprMask,
|
|
uint32_t priority = 0,
|
|
uint64_t moduleId = 0,
|
|
void (*rawCpuInvoker)(CpuContext*) = nullptr,
|
|
bool mustRemainDynamicallyDispatchable = true)
|
|
{
|
|
TranslatedFunctionInfo info;
|
|
info.address = address;
|
|
info.name = name ? name : "";
|
|
info.moduleId = moduleId;
|
|
info.priority = priority;
|
|
// Deliberately *not* mirroring the typed specialization's
|
|
// `|| kind == FunctionKind::Native`. A CpuContext* native receives the
|
|
// whole guest register file and can legitimately write f14-f31 through
|
|
// it, so it stays conservative until the individual address has been
|
|
// measured clean and its registration opts out below.
|
|
info.nonvolatileFprWriteMask = preservesNonvolatileFprs ? 0 : (nonvolatileFprWriteMask & kPpcAllNonvolatileFprMask);
|
|
info.entryPoint = reinterpret_cast<void*>(target);
|
|
info.rawCpuInvoker = rawCpuInvoker ? rawCpuInvoker : reinterpret_cast<void (*)(CpuContext*)>(target);
|
|
info.mustRemainDynamicallyDispatchable = mustRemainDynamicallyDispatchable;
|
|
info.kind = kind;
|
|
TranslatedFunctionRegistry::Register(std::move(info));
|
|
}
|
|
};
|
|
|
|
#define MKW_DETAIL_CAT(a, b) a##b
|
|
#define MKW_DETAIL_MAKE_UNIQUE(a, b) MKW_DETAIL_CAT(a, b)
|
|
|
|
// Three hand-written registration macros. Generated code never registers per function; every
|
|
// translated function reaches the registry via the bulk BulkTranslatedFunctionRecord tables
|
|
// in the translator's *_registration TUs.
|
|
#define REGISTER_TRANSLATED_FUNCTION(address, fn) \
|
|
static AbiTrampoline<decltype(fn)> MKW_DETAIL_MAKE_UNIQUE(_abi_trampoline_, __COUNTER__)(address, #fn, fn, FunctionKind::BaseTranslated, false, kPpcAllNonvolatileFprMask, 0, 0, nullptr, KnownTranslatedCpuCall<address>::kMustRemainDynamicallyDispatchable)
|
|
|
|
#define REGISTER_NATIVE_FUNCTION(address, fn) \
|
|
static AbiTrampoline<decltype(fn)> MKW_DETAIL_MAKE_UNIQUE(_abi_native_trampoline_, __COUNTER__)(address, #fn, fn, FunctionKind::Native, false, kPpcAllNonvolatileFprMask, 0, 0, &AbiRawCpuThunk<&fn>::Invoke)
|
|
|
|
#define REGISTER_NATIVE_FUNCTION_AS(address, fn, pretty_name) \
|
|
static AbiTrampoline<decltype(fn)> MKW_DETAIL_MAKE_UNIQUE(_abi_native_trampoline_named_, __COUNTER__)(address, pretty_name, fn, FunctionKind::Native, false, kPpcAllNonvolatileFprMask, 0, 0, &AbiRawCpuThunk<&fn>::Invoke)
|