mirror of
https://github.com/patchzyy/wiicompiled
synced 2026-09-10 17:16:47 -04:00
theofficialgman
704 lines
28 KiB
C++
704 lines
28 KiB
C++
#pragma once
|
|
|
|
#include "guest_flat_memory.h"
|
|
#include "memory.h"
|
|
#include "recomp_mod_loader.h"
|
|
|
|
#include <cstddef>
|
|
#include <cstdint>
|
|
#include <cstring>
|
|
|
|
// Hooks for GX HLE FIFO handling
|
|
extern "C" {
|
|
void GX_HLE_FIFO_WriteFloat(float val);
|
|
void GX_HLE_FIFO_Write32(uint32_t val);
|
|
void GX_HLE_FIFO_Write16(uint16_t val);
|
|
void GX_HLE_FIFO_Write8(uint8_t val);
|
|
void GX_HLE_FIFO_WriteBurst(const uint8_t* data, uint32_t sizeBytes);
|
|
}
|
|
|
|
namespace MemoryInline {
|
|
#if defined(_WIN32)
|
|
#define MKW_MEMORY_FORCE_INLINE __forceinline
|
|
#define MKW_MEMORY_NO_INLINE __declspec(noinline)
|
|
#else
|
|
// See runtime/include/isa/ppc_isa_config.h for why non-Windows Clang needs the GNU-attribute
|
|
// spellings instead of the MS-extension keywords.
|
|
#define MKW_MEMORY_FORCE_INLINE __attribute__((always_inline)) inline
|
|
#define MKW_MEMORY_NO_INLINE __attribute__((noinline))
|
|
#endif
|
|
#define MKW_MEMORY_COLD __attribute__((cold))
|
|
inline constexpr uint32_t kPageShift = 20;
|
|
inline constexpr uint32_t kPageSize = 1u << kPageShift;
|
|
inline constexpr uint32_t kPageMask = kPageSize - 1u;
|
|
inline constexpr uint32_t kPageCount = 1u << (32 - kPageShift);
|
|
inline constexpr uint32_t kMaxFastScalarSize = 8;
|
|
inline constexpr uint32_t kWritableSubPageShift = RecompMod::kExecutableWriteGuardPageShift;
|
|
inline constexpr uint32_t kWritableSubPageSize = 1u << kWritableSubPageShift;
|
|
inline constexpr uint32_t kWritableSubPagesPerPage = kPageSize / kWritableSubPageSize;
|
|
|
|
struct PageEntry {
|
|
uint8_t* base = nullptr;
|
|
uint32_t limit = 0;
|
|
};
|
|
|
|
extern PageEntry g_pageTable[kPageCount];
|
|
// Encoded (host page base - guest page base) + 1 for full pages whose next
|
|
// page is contiguous. This permits any native access up to 8 bytes without a
|
|
// per-access mask/limit check. Zero retains the general PageEntry fallback.
|
|
extern uintptr_t g_fullPageBias[kPageCount];
|
|
// Runtime-active readable biases. Deferred-read pages clear these entries once
|
|
// instead of paying a mode branch on every translated read.
|
|
extern uintptr_t g_fullReadablePageBias[kPageCount];
|
|
// Same encoding, but only for pages proven not to contain executable bytes.
|
|
// Executable-range registration invalidates entries before guest execution.
|
|
extern uintptr_t g_fullWritablePageBias[kPageCount];
|
|
|
|
// Allocated only for mapped 1 MiB pages that contain both executable and data
|
|
// 4 KiB pages. Entries use the same encoded host bias as the coarse table.
|
|
// Exact executable bits remain authoritative and are checked at lookup time,
|
|
// including both sides of a cross-4-KiB access.
|
|
struct SparseWritablePageTable {
|
|
uintptr_t encodedBias[kWritableSubPagesPerPage]{};
|
|
};
|
|
extern const SparseWritablePageTable* g_sparseWritablePageTables[kPageCount];
|
|
|
|
// Nonzero while any registered deferred read overlaps the page. Small
|
|
// mappings (e.g. the 16 KiB locked cache) have no coarse bias entry, so a
|
|
// zero readable bias alone cannot distinguish "deferred content pending"
|
|
// from "small but plain memory"; range resolution needs the explicit flag.
|
|
extern uint8_t g_deferredReadCoveredPages[kPageCount];
|
|
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD uint8_t Read8Slow(uint32_t addr);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD uint16_t Read16Slow(uint32_t addr);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD uint32_t Read32Slow(uint32_t addr);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD uint64_t Read64Slow(uint32_t addr);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD float ReadFloat32Slow(uint32_t addr);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD double ReadFloat64Slow(uint32_t addr);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD void Write8Slow(uint32_t addr, uint8_t val);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD void Write16Slow(uint32_t addr, uint16_t val);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD void Write32Slow(uint32_t addr, uint32_t val);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD void Write64Slow(uint32_t addr, uint64_t val);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD void WriteFloat32Slow(uint32_t addr, double val);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD void WriteFloat64Slow(uint32_t addr, double val);
|
|
template <typename T>
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD T ReadResolvedFallback(uint32_t addr);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD float ReadResolvedFallbackFloat32(uint32_t addr);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD double ReadResolvedFallbackFloat64(uint32_t addr);
|
|
template <typename T>
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD void WriteResolvedFallback(uint32_t addr, T value);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD void WriteResolvedFallbackFloat32(uint32_t addr, double val);
|
|
MKW_MEMORY_NO_INLINE MKW_MEMORY_COLD void WriteResolvedFallbackFloat64(uint32_t addr, double val);
|
|
bool ResolveDeferredReads(uint32_t addr, size_t length);
|
|
|
|
constexpr bool IsMmioAddress(uint32_t addr) {
|
|
return addr >= 0xCC000000u && addr < 0xCE000000u;
|
|
}
|
|
|
|
constexpr bool IsGpuFifoAddress(uint32_t addr) {
|
|
return addr >= 0xCC008000u && addr < 0xCC008100u;
|
|
}
|
|
|
|
// Page protections can't cover this: an MMIO write must reach GX HLE with its value or be
|
|
// reported, and a fault record can't carry the value, so this mask/compare sits in front of
|
|
// every flat store instead.
|
|
MKW_MEMORY_FORCE_INLINE bool FlatWriteNeedsPolicy(uint32_t address) {
|
|
return (address & 0xFE000000u) == 0xCC000000u; // 0xCC000000..0xCDFFFFFF
|
|
}
|
|
|
|
// Gekko stfs conversion is a bit-level narrowing operation. In particular it
|
|
// does not behave like a host double-to-float cast for values which were left
|
|
// in double precision, and it has hardware-tested handling for tiny values.
|
|
MKW_MEMORY_FORCE_INLINE uint32_t ConvertPpcDoubleToSingleBits(double value) {
|
|
uint64_t bits = 0;
|
|
std::memcpy(&bits, &value, sizeof(bits));
|
|
const uint32_t exponent = static_cast<uint32_t>((bits >> 52) & 0x7FFu);
|
|
// The subnormal-single arm applies to exactly the exponents 874..896; every
|
|
// other exponent takes the plain sign/exponent/fraction narrowing below.
|
|
// Inside that window the exponent field is nonzero, so the magnitude cannot
|
|
// be zero and needs no separate test - the zero case (exponent 0) reaches
|
|
// the narrowing exactly as it did when the two arms shared that test.
|
|
if (exponent - 874u <= 22u) [[unlikely]]
|
|
{
|
|
uint32_t narrowed = static_cast<uint32_t>(
|
|
0x80000000ULL | ((bits & 0x000FFFFFFFFFFFFFULL) >> 21));
|
|
narrowed >>= (905u - exponent);
|
|
narrowed |= static_cast<uint32_t>((bits >> 32) & 0x80000000ULL);
|
|
return narrowed;
|
|
}
|
|
|
|
// Results below the documented conversion range are architecturally
|
|
// undefined; this is the behavior measured on Gekko/Broadway hardware.
|
|
return static_cast<uint32_t>(
|
|
((bits >> 32) & 0xC0000000ULL) | ((bits >> 29) & 0x3FFFFFFFULL));
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE float PpcSingleBitsToFloat(uint32_t bits) {
|
|
float value = 0.0f;
|
|
std::memcpy(&value, &bits, sizeof(value));
|
|
return value;
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE bool TryGetPointerFast(uint32_t address, size_t length, uint8_t*& pointer) {
|
|
const uint32_t page = address >> kPageShift;
|
|
if (length <= 8) {
|
|
const uintptr_t encodedBias = g_fullPageBias[page];
|
|
if (encodedBias != 0) {
|
|
pointer = reinterpret_cast<uint8_t*>((encodedBias - 1u) + address);
|
|
return true;
|
|
}
|
|
}
|
|
const uint32_t offset = address & kPageMask;
|
|
const auto& entry = g_pageTable[page];
|
|
if (!entry.base || offset + length > entry.limit) {
|
|
pointer = nullptr;
|
|
return false;
|
|
}
|
|
pointer = entry.base + offset;
|
|
return true;
|
|
}
|
|
|
|
inline uint8_t* GetPointerFast(uint32_t address, size_t length) {
|
|
uint8_t* pointer = nullptr;
|
|
return TryGetPointerFast(address, length, pointer) ? pointer : nullptr;
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE bool TryGetWritablePointerFast(
|
|
uint32_t address, size_t length, uint8_t*& pointer) {
|
|
// The sparse-table hit below distinguishes itself from the small-mapping
|
|
// hit through `pointer`, so the out-parameter must start null regardless
|
|
// of what the caller passed in. (A caller handing in an uninitialized
|
|
// pointer used to turn every sparse hit into a write through stack
|
|
// garbage - random host memory corruption.)
|
|
pointer = nullptr;
|
|
if (length == 0 || length > 8 || address > UINT32_MAX - (length - 1))
|
|
return false;
|
|
const uint32_t coarsePage = address >> kPageShift;
|
|
uintptr_t encodedBias = g_fullWritablePageBias[coarsePage];
|
|
const uint32_t endAddress = address + static_cast<uint32_t>(length - 1);
|
|
const uint32_t firstExactPage = address >> kWritableSubPageShift;
|
|
const uint32_t lastExactPage = endAddress >> kWritableSubPageShift;
|
|
if (encodedBias != 0 && (endAddress >> kPageShift) != coarsePage &&
|
|
RecompMod::g_executableWriteGuardPages[lastExactPage].load(
|
|
std::memory_order_relaxed) != 0) {
|
|
return false;
|
|
}
|
|
if (encodedBias == 0) {
|
|
const auto* subTable = g_sparseWritablePageTables[coarsePage];
|
|
if (subTable != nullptr) {
|
|
encodedBias = subTable->encodedBias[
|
|
(address & kPageMask) >> kWritableSubPageShift];
|
|
if (encodedBias == 0)
|
|
return false;
|
|
} else {
|
|
// Small mappings such as Broadway's 16 KiB locked cache cannot
|
|
// populate the full-1-MiB bias table. Keep them native by proving
|
|
// the exact access against the ordinary page entry, then applying
|
|
// the same executable-write policy as a sparse-table hit.
|
|
const uint32_t offset = address & kPageMask;
|
|
const auto& entry = g_pageTable[coarsePage];
|
|
if (!entry.base || offset + length > entry.limit)
|
|
return false;
|
|
pointer = entry.base + offset;
|
|
}
|
|
|
|
// The exact 4 KiB guard is the final authority. Checking it on every
|
|
// checked/sparse hit also makes later executable-range registration
|
|
// safe when a prebuilt table still contains the old mapped bias.
|
|
if (RecompMod::g_executableWriteGuardPages[firstExactPage].load(
|
|
std::memory_order_relaxed) != 0 ||
|
|
(lastExactPage != firstExactPage &&
|
|
RecompMod::g_executableWriteGuardPages[lastExactPage].load(
|
|
std::memory_order_relaxed) != 0)) {
|
|
return false;
|
|
}
|
|
|
|
if (pointer != nullptr)
|
|
return true;
|
|
}
|
|
if (encodedBias == 0)
|
|
return false;
|
|
pointer = reinterpret_cast<uint8_t*>((encodedBias - 1u) + address);
|
|
return true;
|
|
}
|
|
|
|
// Flat form: guest_flat_memory.h's page protections already answer mapped/non-deferred/non-executable, so resolving is pure address arithmetic.
|
|
// Two checks stay inline: a wrapped guest address can't survive 64-bit `host + rangeOffset`, and an MMIO write's value isn't recoverable from a
|
|
// fault record, so a write touching that window must resolve null and fall back to Memory::Write*. Checking both range endpoints is a complete
|
|
// proof since length <= kPageSize (1 MiB) can't straddle the 32 MiB MMIO window.
|
|
MKW_MEMORY_FORCE_INLINE uint8_t* ResolveRangeHost(uint32_t base, int32_t minOffset, uint32_t length,
|
|
bool needsRead, bool needsWrite) {
|
|
(void)needsRead;
|
|
const uint32_t guestStart = base + static_cast<uint32_t>(minOffset);
|
|
if (length == 0 || length > kPageSize || guestStart > UINT32_MAX - (length - 1)) return nullptr;
|
|
if (needsWrite &&
|
|
(FlatWriteNeedsPolicy(guestStart) || FlatWriteNeedsPolicy(guestStart + (length - 1))))
|
|
[[unlikely]] return nullptr;
|
|
return MKW_FLAT_GUEST_BASE + guestStart;
|
|
}
|
|
|
|
// Guest-address byte order. Distinct from isa/big_endian.h, which is the
|
|
// host-pointer codec; do not "unify" them.
|
|
inline uint16_t ByteSwap16(uint16_t value) {
|
|
return __builtin_bswap16(value);
|
|
}
|
|
|
|
inline uint32_t ByteSwap32(uint32_t value) {
|
|
return __builtin_bswap32(value);
|
|
}
|
|
|
|
inline uint64_t ByteSwap64(uint64_t value) {
|
|
return __builtin_bswap64(value);
|
|
}
|
|
|
|
template <typename T>
|
|
inline T MaybeByteSwap(T value) {
|
|
if constexpr (sizeof(T) == 1) {
|
|
return value;
|
|
} else if constexpr (sizeof(T) == 2) {
|
|
return static_cast<T>(ByteSwap16(static_cast<uint16_t>(value)));
|
|
} else if constexpr (sizeof(T) == 4) {
|
|
return static_cast<T>(ByteSwap32(static_cast<uint32_t>(value)));
|
|
} else if constexpr (sizeof(T) == 8) {
|
|
return static_cast<T>(ByteSwap64(static_cast<uint64_t>(value)));
|
|
} else {
|
|
return value;
|
|
}
|
|
}
|
|
|
|
template <typename T>
|
|
MKW_MEMORY_FORCE_INLINE bool ReadResolvedScalar(uint8_t* host, uint32_t rangeOffset, T& outValue) {
|
|
if (!host) return false;
|
|
if constexpr (sizeof(T) == 1) {
|
|
outValue = host[rangeOffset];
|
|
} else {
|
|
T value = 0;
|
|
std::memcpy(&value, host + rangeOffset, sizeof(T));
|
|
outValue = MaybeByteSwap(value);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
struct ResolvedLoadPair {
|
|
uint32_t first = 0;
|
|
uint32_t second = 0;
|
|
bool valid = false;
|
|
};
|
|
|
|
MKW_MEMORY_FORCE_INLINE ResolvedLoadPair ReadResolvedPair16(
|
|
uint8_t* host, uint32_t rangeOffset) {
|
|
uint32_t packed = 0;
|
|
if (!ReadResolvedScalar(host, rangeOffset, packed)) return {};
|
|
return {packed >> 16, packed & 0xFFFFu, true};
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE ResolvedLoadPair ReadResolvedPair32(uint8_t* host, uint32_t rangeOffset) {
|
|
uint64_t packed = 0;
|
|
if (!ReadResolvedScalar(host, rangeOffset, packed)) return {};
|
|
return {static_cast<uint32_t>(packed >> 32), static_cast<uint32_t>(packed), true};
|
|
}
|
|
|
|
template <typename Packed>
|
|
MKW_MEMORY_FORCE_INLINE bool WriteResolvedPairFast(
|
|
uint8_t* host, uint32_t rangeOffset, Packed packed) {
|
|
if (!host) return false;
|
|
const Packed swapped = MaybeByteSwap(packed);
|
|
std::memcpy(host + rangeOffset, &swapped, sizeof(swapped));
|
|
return true;
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE bool WriteResolvedPair16(
|
|
uint8_t* host, uint32_t rangeOffset, uint32_t packed) {
|
|
return WriteResolvedPairFast(host, rangeOffset, packed);
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE bool WriteResolvedPair32(
|
|
uint8_t* host, uint32_t rangeOffset, uint64_t packed) {
|
|
return WriteResolvedPairFast(host, rangeOffset, packed);
|
|
}
|
|
|
|
template <typename T>
|
|
MKW_MEMORY_FORCE_INLINE bool WriteResolvedScalar(uint8_t* host, uint32_t rangeOffset, T value) {
|
|
if (!host) return false;
|
|
if constexpr (sizeof(T) == 1) {
|
|
host[rangeOffset] = static_cast<uint8_t>(value);
|
|
} else {
|
|
const T swapped = MaybeByteSwap(value);
|
|
std::memcpy(host + rangeOffset, &swapped, sizeof(T));
|
|
}
|
|
return true;
|
|
}
|
|
|
|
template <typename T>
|
|
inline bool TryReadMappedScalar(uint32_t address, T& outValue) {
|
|
uint8_t* ptr = nullptr;
|
|
if (TryGetPointerFast(address, sizeof(T), ptr)) {
|
|
if constexpr (sizeof(T) == 1) {
|
|
outValue = *ptr;
|
|
} else {
|
|
T value = 0;
|
|
std::memcpy(&value, ptr, sizeof(T));
|
|
outValue = MaybeByteSwap(value);
|
|
}
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
template <typename T>
|
|
MKW_MEMORY_FORCE_INLINE bool TryReadGuestScalar(uint32_t address, T& outValue) {
|
|
const uintptr_t encodedBias = g_fullReadablePageBias[address >> kPageShift];
|
|
if (encodedBias == 0) [[unlikely]]
|
|
return false;
|
|
auto* ptr = reinterpret_cast<uint8_t*>((encodedBias - 1u) + address);
|
|
if constexpr (sizeof(T) == 1) {
|
|
outValue = *ptr;
|
|
} else {
|
|
T value = 0;
|
|
std::memcpy(&value, ptr, sizeof(T));
|
|
outValue = MaybeByteSwap(value);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
template <typename T>
|
|
MKW_MEMORY_FORCE_INLINE bool TryWriteGuestScalar(uint32_t address, T value) {
|
|
static_assert(sizeof(T) >= 1 && sizeof(T) <= kMaxFastScalarSize);
|
|
// A mixed executable/data 1 MiB page zeroes the coarse writable bias even though most of
|
|
// its 4 KiB sub-pages are plain data; MKW's THP buffers share such a page with .text, which
|
|
// used to force ~15% of total CPU through the cold path. The sparse sub-page tier below
|
|
// keeps those stores native while the exact 4 KiB executable guards stay authoritative.
|
|
if (address > UINT32_MAX - static_cast<uint32_t>(sizeof(T) - 1u)) [[unlikely]]
|
|
return false;
|
|
const uint32_t coarsePage = address >> kPageShift;
|
|
const uint32_t endAddress = address + static_cast<uint32_t>(sizeof(T) - 1u);
|
|
if ((endAddress >> kPageShift) != coarsePage) [[unlikely]]
|
|
return false;
|
|
const uintptr_t encodedBias = g_fullWritablePageBias[coarsePage];
|
|
uint8_t* ptr = nullptr;
|
|
if (encodedBias != 0) {
|
|
ptr = reinterpret_cast<uint8_t*>((encodedBias - 1u) + address);
|
|
} else if (!TryGetWritablePointerFast(address, sizeof(T), ptr)) [[unlikely]] {
|
|
return false;
|
|
}
|
|
if constexpr (sizeof(T) == 1) {
|
|
*ptr = static_cast<uint8_t>(value);
|
|
} else {
|
|
const T swapped = MaybeByteSwap(value);
|
|
std::memcpy(ptr, &swapped, sizeof(T));
|
|
}
|
|
return true;
|
|
}
|
|
|
|
template <typename T>
|
|
inline bool WriteStackScalarFast(uint32_t address, T value) {
|
|
uint8_t* ptr = nullptr;
|
|
if (TryGetPointerFast(address, sizeof(T), ptr)) {
|
|
if constexpr (sizeof(T) == 1) {
|
|
*ptr = static_cast<uint8_t>(value);
|
|
} else {
|
|
const T swapped = MaybeByteSwap(value);
|
|
std::memcpy(ptr, &swapped, sizeof(T));
|
|
}
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
inline uint8_t ReadStack8(uint32_t address) {
|
|
uint8_t value = 0;
|
|
return TryReadMappedScalar(address, value) ? value : Memory::Read8(address);
|
|
}
|
|
|
|
inline uint16_t ReadStack16(uint32_t address) {
|
|
uint16_t value = 0;
|
|
return TryReadMappedScalar(address, value) ? value : Memory::Read16(address);
|
|
}
|
|
|
|
inline uint32_t ReadStack32(uint32_t address) {
|
|
uint32_t value = 0;
|
|
return TryReadMappedScalar(address, value) ? value : Memory::Read32(address);
|
|
}
|
|
|
|
inline uint64_t ReadStack64(uint32_t address) {
|
|
uint64_t value = 0;
|
|
return TryReadMappedScalar(address, value) ? value : Memory::Read64(address);
|
|
}
|
|
|
|
|
|
|
|
inline void WriteStack8(uint32_t address, uint8_t value) {
|
|
if (!WriteStackScalarFast(address, value)) {
|
|
Memory::Write8(address, value);
|
|
}
|
|
}
|
|
|
|
inline void WriteStack16(uint32_t address, uint16_t value) {
|
|
if (!WriteStackScalarFast(address, value)) {
|
|
Memory::Write16(address, value);
|
|
}
|
|
}
|
|
|
|
inline void WriteStack32(uint32_t address, uint32_t value) {
|
|
if (!WriteStackScalarFast(address, value)) {
|
|
Memory::Write32(address, value);
|
|
}
|
|
}
|
|
|
|
inline void WriteStack64(uint32_t address, uint64_t value) {
|
|
if (!WriteStackScalarFast(address, value)) {
|
|
Memory::Write64(address, value);
|
|
}
|
|
}
|
|
|
|
inline void WriteStackFloat32(uint32_t address, double value) {
|
|
const uint32_t bits = ConvertPpcDoubleToSingleBits(value);
|
|
if (!WriteStackScalarFast(address, bits)) {
|
|
Memory::WriteFloat32(address, value);
|
|
}
|
|
}
|
|
|
|
inline void WriteStackFloat64(uint32_t address, double value) {
|
|
uint64_t bits = 0;
|
|
std::memcpy(&bits, &value, sizeof(bits));
|
|
if (!WriteStackScalarFast(address, bits)) {
|
|
Memory::WriteFloat64(address, value);
|
|
}
|
|
}
|
|
|
|
template <typename T, typename SlowRead>
|
|
MKW_MEMORY_FORCE_INLINE T ReadResolved(uint8_t* host, uint32_t rangeOffset, uint32_t address,
|
|
SlowRead slow) {
|
|
T value = 0;
|
|
if (ReadResolvedScalar(host, rangeOffset, value)) {
|
|
return value;
|
|
}
|
|
[[unlikely]] return slow(address);
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE uint8_t ReadResolved8(uint8_t* r, uint32_t o, uint32_t a) { return ReadResolved<uint8_t>(r, o, a, ReadResolvedFallback<uint8_t>); }
|
|
MKW_MEMORY_FORCE_INLINE uint16_t ReadResolved16(uint8_t* r, uint32_t o, uint32_t a) { return ReadResolved<uint16_t>(r, o, a, ReadResolvedFallback<uint16_t>); }
|
|
MKW_MEMORY_FORCE_INLINE uint32_t ReadResolved32(uint8_t* r, uint32_t o, uint32_t a) { return ReadResolved<uint32_t>(r, o, a, ReadResolvedFallback<uint32_t>); }
|
|
// Live via isa/ppc_isa_quantized.h (the psq resolved tier packs two lanes into
|
|
// one 64-bit access); generated code never names it directly.
|
|
MKW_MEMORY_FORCE_INLINE uint64_t ReadResolved64(uint8_t* r, uint32_t o, uint32_t a) { return ReadResolved<uint64_t>(r, o, a, ReadResolvedFallback<uint64_t>); }
|
|
MKW_MEMORY_FORCE_INLINE float ReadResolvedFloat32(uint8_t* r, uint32_t o, uint32_t a) {
|
|
uint32_t bits = 0;
|
|
if (!ReadResolvedScalar(r, o, bits)) [[unlikely]] return ReadResolvedFallbackFloat32(a);
|
|
float value; std::memcpy(&value, &bits, sizeof(value)); return value;
|
|
}
|
|
MKW_MEMORY_FORCE_INLINE double ReadResolvedFloat64(uint8_t* r, uint32_t o, uint32_t a) {
|
|
uint64_t bits = 0;
|
|
if (!ReadResolvedScalar(r, o, bits)) [[unlikely]] return ReadResolvedFallbackFloat64(a);
|
|
double value; std::memcpy(&value, &bits, sizeof(value)); return value;
|
|
}
|
|
|
|
template <typename T, typename SlowWrite>
|
|
MKW_MEMORY_FORCE_INLINE void WriteResolved(uint8_t* host, uint32_t rangeOffset, uint32_t address, T value,
|
|
SlowWrite slow) {
|
|
if (WriteResolvedScalar(host, rangeOffset, value)) {
|
|
return;
|
|
}
|
|
[[unlikely]] slow(address, value);
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE void WriteResolved8(uint8_t* r, uint32_t o, uint32_t a, uint8_t v) { WriteResolved(r, o, a, v, WriteResolvedFallback<uint8_t>); }
|
|
MKW_MEMORY_FORCE_INLINE void WriteResolved16(uint8_t* r, uint32_t o, uint32_t a, uint16_t v) { WriteResolved(r, o, a, v, WriteResolvedFallback<uint16_t>); }
|
|
MKW_MEMORY_FORCE_INLINE void WriteResolved32(uint8_t* r, uint32_t o, uint32_t a, uint32_t v) { WriteResolved(r, o, a, v, WriteResolvedFallback<uint32_t>); }
|
|
// Live via isa/ppc_isa_quantized.h, as ReadResolved64 above.
|
|
MKW_MEMORY_FORCE_INLINE void WriteResolved64(uint8_t* r, uint32_t o, uint32_t a, uint64_t v) { WriteResolved(r, o, a, v, WriteResolvedFallback<uint64_t>); }
|
|
MKW_MEMORY_FORCE_INLINE void WriteResolvedFloat32(uint8_t* r, uint32_t o, uint32_t a, double v) {
|
|
const uint32_t bits = ConvertPpcDoubleToSingleBits(v);
|
|
if (WriteResolvedScalar(r, o, bits)) return;
|
|
[[unlikely]] WriteResolvedFallbackFloat32(a, v);
|
|
}
|
|
MKW_MEMORY_FORCE_INLINE void WriteResolvedFloat64(uint8_t* r, uint32_t o, uint32_t a, double v) {
|
|
uint64_t bits; std::memcpy(&bits, &v, sizeof(bits));
|
|
if (WriteResolvedScalar(r, o, bits)) return;
|
|
[[unlikely]] WriteResolvedFallbackFloat64(a, v);
|
|
}
|
|
|
|
// Flat guest memory (audit item T-MEM): the 4 GiB reservation makes a guest access a byte swap
|
|
// around `*(T*)(base + addr)`, no page-table load or limit check (interception model documented
|
|
// in guest_flat_memory.h). The one exception kept inline is the MMIO write policy, since the
|
|
// written value can't be recovered from a fault record.
|
|
|
|
template <typename T>
|
|
MKW_MEMORY_FORCE_INLINE T FlatLoad(uint32_t address) {
|
|
T value{};
|
|
std::memcpy(&value, MKW_FLAT_GUEST_BASE + address, sizeof(T));
|
|
return MaybeByteSwap(value);
|
|
}
|
|
|
|
template <typename T>
|
|
MKW_MEMORY_FORCE_INLINE void FlatStore(uint32_t address, T value) {
|
|
const T swapped = MaybeByteSwap(value);
|
|
std::memcpy(MKW_FLAT_GUEST_BASE + address, &swapped, sizeof(T));
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE uint8_t FlatRead8(uint32_t address) { return FlatLoad<uint8_t>(address); }
|
|
MKW_MEMORY_FORCE_INLINE uint16_t FlatRead16(uint32_t address) { return FlatLoad<uint16_t>(address); }
|
|
MKW_MEMORY_FORCE_INLINE uint32_t FlatRead32(uint32_t address) { return FlatLoad<uint32_t>(address); }
|
|
|
|
MKW_MEMORY_FORCE_INLINE float FlatReadFloat32(uint32_t address) {
|
|
const uint32_t bits = FlatLoad<uint32_t>(address);
|
|
float value = 0.0f;
|
|
std::memcpy(&value, &bits, sizeof(value));
|
|
return value;
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE double FlatReadFloat64(uint32_t address) {
|
|
const uint64_t bits = FlatLoad<uint64_t>(address);
|
|
double value = 0.0;
|
|
std::memcpy(&value, &bits, sizeof(value));
|
|
return value;
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE void FlatWrite8(uint32_t address, uint8_t value) {
|
|
if (FlatWriteNeedsPolicy(address)) [[unlikely]] { Write8Slow(address, value); return; }
|
|
FlatStore<uint8_t>(address, value);
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE void FlatWrite16(uint32_t address, uint16_t value) {
|
|
if (FlatWriteNeedsPolicy(address)) [[unlikely]] { Write16Slow(address, value); return; }
|
|
FlatStore<uint16_t>(address, value);
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE void FlatWrite32(uint32_t address, uint32_t value) {
|
|
if (FlatWriteNeedsPolicy(address)) [[unlikely]] { Write32Slow(address, value); return; }
|
|
FlatStore<uint32_t>(address, value);
|
|
}
|
|
|
|
|
|
MKW_MEMORY_FORCE_INLINE void FlatWriteFloat32(uint32_t address, double value) {
|
|
const uint32_t bits = ConvertPpcDoubleToSingleBits(value);
|
|
if (FlatWriteNeedsPolicy(address)) [[unlikely]] { WriteFloat32Slow(address, value); return; }
|
|
FlatStore<uint32_t>(address, bits);
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE void FlatWriteFloat64(uint32_t address, double value) {
|
|
uint64_t bits = 0;
|
|
std::memcpy(&bits, &value, sizeof(bits));
|
|
if (FlatWriteNeedsPolicy(address)) [[unlikely]] { WriteFloat64Slow(address, value); return; }
|
|
FlatStore<uint64_t>(address, bits);
|
|
}
|
|
|
|
// Check-free stores: emitted ONLY for addresses the translator proved at translate time are ordinary guest RAM (r1-relative stack slots, ~45%
|
|
// of flat stores), skipping the MMIO mask/compare that's pure overhead there. Still safe if that proof were ever wrong: the flat view maps
|
|
// 0xCC000000..0xCDFFFFFF PAGE_NOACCESS, so a stray MMIO store faults into the same handler and diagnostic as the checked path, just reported
|
|
// instead of dispatched inline. Never use these for an address the translator hasn't proven.
|
|
|
|
MKW_MEMORY_FORCE_INLINE void FlatWriteRam8(uint32_t address, uint8_t value) {
|
|
FlatStore<uint8_t>(address, value);
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE void FlatWriteRam16(uint32_t address, uint16_t value) {
|
|
FlatStore<uint16_t>(address, value);
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE void FlatWriteRam32(uint32_t address, uint32_t value) {
|
|
FlatStore<uint32_t>(address, value);
|
|
}
|
|
|
|
|
|
MKW_MEMORY_FORCE_INLINE void FlatWriteRamFloat32(uint32_t address, double value) {
|
|
FlatStore<uint32_t>(address, ConvertPpcDoubleToSingleBits(value));
|
|
}
|
|
|
|
MKW_MEMORY_FORCE_INLINE void FlatWriteRamFloat64(uint32_t address, double value) {
|
|
uint64_t bits = 0;
|
|
std::memcpy(&bits, &value, sizeof(bits));
|
|
FlatStore<uint64_t>(address, bits);
|
|
}
|
|
|
|
} // namespace MemoryInline
|
|
|
|
#undef MKW_MEMORY_FORCE_INLINE
|
|
#undef MKW_MEMORY_NO_INLINE
|
|
#undef MKW_MEMORY_COLD
|
|
|
|
inline uint8_t Memory::Read8(uint32_t addr) {
|
|
uint8_t value = 0;
|
|
if (!MemoryInline::TryReadGuestScalar(addr, value)) [[unlikely]]
|
|
return MemoryInline::Read8Slow(addr);
|
|
return value;
|
|
}
|
|
|
|
inline uint16_t Memory::Read16(uint32_t addr) {
|
|
uint16_t value = 0;
|
|
if (!MemoryInline::TryReadGuestScalar(addr, value)) [[unlikely]]
|
|
return MemoryInline::Read16Slow(addr);
|
|
return value;
|
|
}
|
|
|
|
inline uint32_t Memory::Read32(uint32_t addr) {
|
|
uint32_t value = 0;
|
|
if (!MemoryInline::TryReadGuestScalar(addr, value)) [[unlikely]]
|
|
return MemoryInline::Read32Slow(addr);
|
|
return value;
|
|
}
|
|
|
|
inline uint64_t Memory::Read64(uint32_t addr) {
|
|
uint64_t value = 0;
|
|
if (!MemoryInline::TryReadGuestScalar(addr, value)) [[unlikely]]
|
|
return MemoryInline::Read64Slow(addr);
|
|
return value;
|
|
}
|
|
|
|
inline float Memory::ReadFloat32(uint32_t addr) {
|
|
uint32_t bits = 0;
|
|
if (!MemoryInline::TryReadGuestScalar(addr, bits)) [[unlikely]]
|
|
return MemoryInline::ReadFloat32Slow(addr);
|
|
float value = 0.0f;
|
|
std::memcpy(&value, &bits, sizeof(value));
|
|
return value;
|
|
}
|
|
|
|
inline double Memory::ReadFloat64(uint32_t addr) {
|
|
uint64_t bits = 0;
|
|
if (!MemoryInline::TryReadGuestScalar(addr, bits)) [[unlikely]]
|
|
return MemoryInline::ReadFloat64Slow(addr);
|
|
double value = 0.0;
|
|
std::memcpy(&value, &bits, sizeof(value));
|
|
return value;
|
|
}
|
|
|
|
inline void Memory::Write8(uint32_t addr, uint8_t val) {
|
|
if (!MemoryInline::TryWriteGuestScalar(addr, val)) [[unlikely]]
|
|
MemoryInline::Write8Slow(addr, val);
|
|
}
|
|
|
|
inline void Memory::Write16(uint32_t addr, uint16_t val) {
|
|
if (!MemoryInline::TryWriteGuestScalar(addr, val)) [[unlikely]]
|
|
MemoryInline::Write16Slow(addr, val);
|
|
}
|
|
|
|
inline void Memory::Write32(uint32_t addr, uint32_t val) {
|
|
if (!MemoryInline::TryWriteGuestScalar(addr, val)) [[unlikely]]
|
|
MemoryInline::Write32Slow(addr, val);
|
|
}
|
|
|
|
inline void Memory::Write64(uint32_t addr, uint64_t val) {
|
|
if (!MemoryInline::TryWriteGuestScalar(addr, val)) [[unlikely]]
|
|
MemoryInline::Write64Slow(addr, val);
|
|
}
|
|
|
|
inline void Memory::WriteFloat32(uint32_t addr, double val) {
|
|
{
|
|
const uint32_t bits = MemoryInline::ConvertPpcDoubleToSingleBits(val);
|
|
if (MemoryInline::TryWriteGuestScalar(addr, bits))
|
|
return;
|
|
}
|
|
[[unlikely]] MemoryInline::WriteFloat32Slow(addr, val);
|
|
}
|
|
|
|
inline void Memory::WriteFloat64(uint32_t addr, double val) {
|
|
{
|
|
uint64_t bits = 0;
|
|
std::memcpy(&bits, &val, sizeof(bits));
|
|
if (MemoryInline::TryWriteGuestScalar(addr, bits))
|
|
return;
|
|
}
|
|
[[unlikely]] MemoryInline::WriteFloat64Slow(addr, val);
|
|
}
|