From 6488b7fe707d5dd22d4ca08e5e54793761af33ce Mon Sep 17 00:00:00 2001 From: patchzyy <64382339+patchzyy@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:26:48 +0200 Subject: [PATCH] Split PSQ fallbacks and add PSQ ISA tests --- runtime/CMakeLists.txt | 34 ++++ runtime/cmake/PublicProducts.cmake | 11 +- runtime/include/isa/ppc_isa_quantized.h | 158 +++++++----------- runtime/src/ppc_helpers.cpp | 8 +- runtime/src/ppc_quantized.cpp | 154 +++++++++++++++++ runtime/tests/psq_helpers_tests.cpp | 140 ++++++++++++++++ runtime/tests/psq_memory/ppc_isa_memory.h | 83 +++++++++ runtime/tests/psq_reserved_tests.cmake | 13 ++ .../TranslatorCppTestHarness.cs | 2 + 9 files changed, 495 insertions(+), 108 deletions(-) create mode 100644 runtime/src/ppc_quantized.cpp create mode 100644 runtime/tests/psq_helpers_tests.cpp create mode 100644 runtime/tests/psq_memory/ppc_isa_memory.h create mode 100644 runtime/tests/psq_reserved_tests.cmake diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 22537d3..f907db2 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -22,6 +22,7 @@ if(NOT CMAKE_BUILD_TYPE STREQUAL "Release") endif() option(MKW_BUILD_PRODUCTS "Build translated WiiCompiled product targets" ON) +option(MKW_BUILD_PSQ_TESTS "Build focused PSQ ISA tests" OFF) # Preprocessor definitions that belong to this project's own code (the runtime, # the translated shards and the product glue) and to nothing else. They are @@ -162,6 +163,18 @@ set(MKW_AURORA_DIR "${CMAKE_CURRENT_LIST_DIR}/../aurora-main") # Fast-math may erase them and change guest-visible integer conversions. set(MKW_TRANSLATED_PPC_FP_OPTIONS -fno-fast-math -ffp-contract=off) +set(MKW_PPC_SEMANTIC_RUNTIME_SOURCES + "${CMAKE_CURRENT_LIST_DIR}/src/ppc_helpers.cpp" + "${CMAKE_CURRENT_LIST_DIR}/src/fpu_helpers.cpp" + "${CMAKE_CURRENT_LIST_DIR}/src/ppc_quantized.cpp") +set_source_files_properties(${MKW_PPC_SEMANTIC_RUNTIME_SOURCES} PROPERTIES + SKIP_UNITY_BUILD_INCLUSION ON + SKIP_PRECOMPILE_HEADERS ON + COMPILE_OPTIONS "${MKW_TRANSLATED_PPC_FP_OPTIONS}") +# Match the optimization policy of the translated callers as well as their FP policy. +set_property(SOURCE "${CMAKE_CURRENT_LIST_DIR}/src/ppc_quantized.cpp" APPEND PROPERTY + COMPILE_OPTIONS -O2 -fno-slp-vectorize) + # ---------------------------------------------------------------------- # Third-party: aurora-main (provides SDL3 + GPU backends) # ---------------------------------------------------------------------- @@ -310,6 +323,27 @@ set_target_properties(mkw_platform PROPERTIES UNITY_BUILD OFF) # Keep these independent from Aurora's BUILD_TESTING option: they validate the # project's host-platform contracts, not Aurora's third-party test suite. enable_testing() +if(MKW_BUILD_PSQ_TESTS) + add_executable(mkw_psq_helpers_tests + "${CMAKE_CURRENT_LIST_DIR}/tests/psq_helpers_tests.cpp" + "${CMAKE_CURRENT_LIST_DIR}/src/ppc_quantized.cpp") + target_include_directories(mkw_psq_helpers_tests PRIVATE + "${CMAKE_CURRENT_LIST_DIR}/tests/psq_memory" + "${CMAKE_CURRENT_LIST_DIR}/include/isa" + "${CMAKE_CURRENT_LIST_DIR}/include") + target_compile_features(mkw_psq_helpers_tests PRIVATE cxx_std_17) + target_compile_options(mkw_psq_helpers_tests PRIVATE + -O2 ${MKW_TRANSLATED_PPC_FP_OPTIONS} -fno-slp-vectorize) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64|X86_64)$") + target_compile_options(mkw_psq_helpers_tests PRIVATE -march=x86-64-v3) + endif() + set_target_properties(mkw_psq_helpers_tests PROPERTIES UNITY_BUILD OFF) + add_test(NAME mkw_psq_helpers_tests COMMAND mkw_psq_helpers_tests) + add_test(NAME mkw_psq_reserved_tests COMMAND "${CMAKE_COMMAND}" + "-DPSQ_TEST_EXECUTABLE=$" + -P "${CMAKE_CURRENT_LIST_DIR}/tests/psq_reserved_tests.cmake") +endif() + add_executable(mkw_platform_paths_tests "${CMAKE_CURRENT_LIST_DIR}/tests/platform_paths_tests.cpp") target_link_libraries(mkw_platform_paths_tests PRIVATE mkw_platform) target_compile_features(mkw_platform_paths_tests PRIVATE cxx_std_17) diff --git a/runtime/cmake/PublicProducts.cmake b/runtime/cmake/PublicProducts.cmake index e76e75a..6c50958 100644 --- a/runtime/cmake/PublicProducts.cmake +++ b/runtime/cmake/PublicProducts.cmake @@ -118,16 +118,7 @@ foreach(source IN LISTS SOURCES) endif() set_source_files_properties("${source}" PROPERTIES UNITY_GROUP "${runtime_group}") endforeach() -# These translation units implement guest-visible floating-point bit -# semantics. Keep them out of the fast-math runtime unity groups and apply -# the same contraction/rounding policy as translated PPC shards. -set(MKW_PPC_SEMANTIC_RUNTIME_SOURCES - "${MKW_RUNTIME_SOURCE_DIR}/src/ppc_helpers.cpp" - "${MKW_RUNTIME_SOURCE_DIR}/src/fpu_helpers.cpp") -set_source_files_properties(${MKW_PPC_SEMANTIC_RUNTIME_SOURCES} PROPERTIES - SKIP_UNITY_BUILD_INCLUSION ON - SKIP_PRECOMPILE_HEADERS ON - COMPILE_OPTIONS "${MKW_TRANSLATED_PPC_FP_OPTIONS}") +# PPC semantic sources are excluded from unity/PCH and configured in CMakeLists.txt. set_target_properties(mkw_runtime_common PROPERTIES UNITY_BUILD ON UNITY_BUILD_MODE GROUP) target_precompile_headers(mkw_runtime_common PRIVATE "${MKW_RUNTIME_SOURCE_DIR}/include/mkw_pch.h") mkw_apply_common_compile_options(mkw_runtime_common) diff --git a/runtime/include/isa/ppc_isa_quantized.h b/runtime/include/isa/ppc_isa_quantized.h index 898d608..a5066f8 100644 --- a/runtime/include/isa/ppc_isa_quantized.h +++ b/runtime/include/isa/ppc_isa_quantized.h @@ -1385,37 +1385,13 @@ MKW_PPC_FORCE_INLINE void PPC_PsqStStackInline(uint32_t addr, double value) // Context-free PSQ entries for translated regions which own GQR state as an // ordinary native value. All architecturally valid quantization encodings are // handled directly; reserved encodings retain the generic helper's abort. -template -MKW_PPC_NO_INLINE MKW_PPC_COLD inline double PPC_PsqLStateFallback(uint32_t gqr, uint32_t addr) -{ - static_assert(W <= 1u && I < 8u); - const uint32_t type = (gqr >> 16) & 0x7u; - const uint32_t scale = (gqr >> 24) & 0x3Fu; - if constexpr (W == 0u) - { - switch (type) - { - case 0u: return Stack ? PpcLoadPairPsqFloatStackInline(addr) : PpcLoadPairPsqFloatFastInline(addr); - case 4u: return Stack ? PpcLoadPairPsqIntegerStackInline(addr, scale) : PpcLoadPairPsqIntegerFastInline(addr, scale); - case 5u: return Stack ? PpcLoadPairPsqIntegerStackInline(addr, scale) : PpcLoadPairPsqIntegerFastInline(addr, scale); - case 6u: return Stack ? PpcLoadPairPsqIntegerStackInline(addr, scale) : PpcLoadPairPsqIntegerFastInline(addr, scale); - case 7u: return Stack ? PpcLoadPairPsqIntegerStackInline(addr, scale) : PpcLoadPairPsqIntegerFastInline(addr, scale); - default: std::abort(); - } - } - else - { - switch (type) - { - case 0u: return Stack ? PpcLoadSinglePsqFloatStackInline(addr) : PpcLoadSinglePsqFloatFastInline(addr); - case 4u: return Stack ? PpcLoadSinglePsqQuantizedStackInline(addr, scale) : PpcLoadSinglePsqQuantizedFastInline(addr, scale); - case 5u: return Stack ? PpcLoadSinglePsqQuantizedStackInline(addr, scale) : PpcLoadSinglePsqQuantizedFastInline(addr, scale); - case 6u: return Stack ? PpcLoadSinglePsqQuantizedStackInline(addr, scale) : PpcLoadSinglePsqQuantizedFastInline(addr, scale); - case 7u: return Stack ? PpcLoadSinglePsqQuantizedStackInline(addr, scale) : PpcLoadSinglePsqQuantizedFastInline(addr, scale); - default: std::abort(); - } - } -} +template +MKW_PPC_NO_INLINE MKW_PPC_COLD double PPC_PsqLStateFallback(uint32_t gqr, uint32_t addr); + +extern template double PPC_PsqLStateFallback<0u, false>(uint32_t, uint32_t); +extern template double PPC_PsqLStateFallback<0u, true>(uint32_t, uint32_t); +extern template double PPC_PsqLStateFallback<1u, false>(uint32_t, uint32_t); +extern template double PPC_PsqLStateFallback<1u, true>(uint32_t, uint32_t); // Keep the normal explicit-state path small and directly optimizable. Exact // unscaled encodings cover the SDK's common GQR setup; scaled and reserved @@ -1442,41 +1418,17 @@ MKW_PPC_FORCE_INLINE double PPC_PsqLStateInline(uint32_t gqr, uint32_t addr) if constexpr (W == 0u) return Stack ? PpcLoadPairPsqIntegerStackInline(addr, 0u) : PpcLoadPairPsqIntegerFastInline(addr, 0u); else return Stack ? PpcLoadSinglePsqQuantizedStackInline(addr, 0u) : PpcLoadSinglePsqQuantizedFastInline(addr, 0u); default: - return PPC_PsqLStateFallback(gqr, addr); + return PPC_PsqLStateFallback(gqr, addr); } } -template -MKW_PPC_NO_INLINE MKW_PPC_COLD inline void PPC_PsqStStateFallback(uint32_t gqr, uint32_t addr, double value) -{ - static_assert(W <= 1u && I < 8u); - const uint32_t type = gqr & 0x7u; - const uint32_t scale = (gqr >> 8) & 0x3Fu; - if constexpr (W == 0u) - { - switch (type) - { - case 0u: Stack ? PpcStorePairPsqFloatStackInline(addr, value) : PpcStorePairPsqFloatFastInline(addr, value); return; - case 4u: if constexpr (Stack) PpcStorePairPsqQuantizedStackInline(addr, value, scale); else PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; - case 5u: if constexpr (Stack) PpcStorePairPsqQuantizedStackInline(addr, value, scale); else PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; - case 6u: if constexpr (Stack) PpcStorePairPsqQuantizedStackInline(addr, value, scale); else PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; - case 7u: if constexpr (Stack) PpcStorePairPsqQuantizedStackInline(addr, value, scale); else PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; - default: std::abort(); - } - } - else - { - switch (type) - { - case 0u: if constexpr (Stack) PpcStoreSinglePsqFloatStackInline(addr, value); else PpcStoreSinglePsqFloatFastInline(addr, value); return; - case 4u: if constexpr (Stack) PpcStoreSinglePsqQuantizedStackInline(addr, value, scale); else PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; - case 5u: if constexpr (Stack) PpcStoreSinglePsqQuantizedStackInline(addr, value, scale); else PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; - case 6u: if constexpr (Stack) PpcStoreSinglePsqQuantizedStackInline(addr, value, scale); else PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; - case 7u: if constexpr (Stack) PpcStoreSinglePsqQuantizedStackInline(addr, value, scale); else PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; - default: std::abort(); - } - } -} +template +MKW_PPC_NO_INLINE MKW_PPC_COLD void PPC_PsqStStateFallback(uint32_t gqr, uint32_t addr, double value); + +extern template void PPC_PsqStStateFallback<0u, false>(uint32_t, uint32_t, double); +extern template void PPC_PsqStStateFallback<0u, true>(uint32_t, uint32_t, double); +extern template void PPC_PsqStStateFallback<1u, false>(uint32_t, uint32_t, double); +extern template void PPC_PsqStStateFallback<1u, true>(uint32_t, uint32_t, double); template MKW_PPC_FORCE_INLINE void PPC_PsqStStateInline(uint32_t gqr, uint32_t addr, double value) @@ -1505,11 +1457,18 @@ MKW_PPC_FORCE_INLINE void PPC_PsqStStateInline(uint32_t gqr, uint32_t addr, doub else { if constexpr (Stack) PpcStoreSinglePsqQuantizedStackInline(addr, value, 0u); else PpcStoreSinglePsqQuantizedFastInline(addr, value, 0u); } return; default: - PPC_PsqStStateFallback(gqr, addr, value); + PPC_PsqStStateFallback(gqr, addr, value); return; } } +template +MKW_PPC_NO_INLINE MKW_PPC_COLD double PPC_PsqLResolvedStateFallback( + uint32_t gqr, uint8_t* resolvedHost, uint32_t offset, uint32_t addr); + +extern template double PPC_PsqLResolvedStateFallback<0u>(uint32_t, uint8_t*, uint32_t, uint32_t); +extern template double PPC_PsqLResolvedStateFallback<1u>(uint32_t, uint8_t*, uint32_t, uint32_t); + template MKW_PPC_FORCE_INLINE double PPC_PsqLResolvedStateInline( uint32_t gqr, uint8_t* resolvedHost, uint32_t offset, uint32_t addr) @@ -1517,33 +1476,41 @@ MKW_PPC_FORCE_INLINE double PPC_PsqLResolvedStateInline( static_assert(W <= 1u && I < 8u); if (!resolvedHost) [[unlikely]] return PPC_PsqLStateInline(gqr, addr); const uint32_t type = (gqr >> 16) & 0x7u; - const uint32_t scale = (gqr >> 24) & 0x3Fu; + if (type == 0u) { + if constexpr (W == 0u) return PpcLoadPairPsqFloatResolvedInline(resolvedHost, offset, addr); + else return PpcLoadSinglePsqFloatResolvedInline(resolvedHost, offset, addr); + } if constexpr (W == 0u) { - switch (type) + switch (gqr & 0x3F070000u) { - case 0u: return PpcLoadPairPsqFloatResolvedInline(resolvedHost, offset, addr); - case 4u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, scale); - case 5u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, scale); - case 6u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, scale); - case 7u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, scale); - default: std::abort(); + case 0x00040000u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, 0u); + case 0x00050000u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, 0u); + case 0x00060000u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, 0u); + case 0x00070000u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, 0u); + default: return PPC_PsqLResolvedStateFallback(gqr, resolvedHost, offset, addr); } } else { - switch (type) + switch (gqr & 0x3F070000u) { - case 0u: return PpcLoadSinglePsqFloatResolvedInline(resolvedHost, offset, addr); - case 4u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, scale); - case 5u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, scale); - case 6u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, scale); - case 7u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, scale); - default: std::abort(); + case 0x00040000u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, 0u); + case 0x00050000u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, 0u); + case 0x00060000u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, 0u); + case 0x00070000u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, 0u); + default: return PPC_PsqLResolvedStateFallback(gqr, resolvedHost, offset, addr); } } } +template +MKW_PPC_NO_INLINE MKW_PPC_COLD void PPC_PsqStResolvedStateFallback( + uint32_t gqr, uint8_t* resolvedHost, uint32_t offset, uint32_t addr, double value); + +extern template void PPC_PsqStResolvedStateFallback<0u>(uint32_t, uint8_t*, uint32_t, uint32_t, double); +extern template void PPC_PsqStResolvedStateFallback<1u>(uint32_t, uint8_t*, uint32_t, uint32_t, double); + template MKW_PPC_FORCE_INLINE void PPC_PsqStResolvedStateInline( uint32_t gqr, uint8_t* resolvedHost, uint32_t offset, uint32_t addr, double value) @@ -1555,29 +1522,32 @@ MKW_PPC_FORCE_INLINE void PPC_PsqStResolvedStateInline( return; } const uint32_t type = gqr & 0x7u; - const uint32_t scale = (gqr >> 8) & 0x3Fu; + if (type == 0u) { + if constexpr (W == 0u) PpcStorePairPsqFloatResolvedInline(resolvedHost, offset, addr, value); + else PpcStoreSinglePsqFloatResolvedInline(resolvedHost, offset, addr, value); + return; + } if constexpr (W == 0u) { - switch (type) + switch (gqr & 0x3F07u) { - case 0u: PpcStorePairPsqFloatResolvedInline(resolvedHost, offset, addr, value); return; - case 4u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; - case 5u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; - case 6u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; - case 7u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; - default: std::abort(); + case 0x3D04u: PpcStorePairPsqU8Scale61ResolvedInline(resolvedHost, offset, addr, value); return; + case 4u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, 0u); return; + case 5u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, 0u); return; + case 6u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, 0u); return; + case 7u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, 0u); return; + default: PPC_PsqStResolvedStateFallback(gqr, resolvedHost, offset, addr, value); return; } } else { - switch (type) + switch (gqr & 0x3F07u) { - case 0u: PpcStoreSinglePsqFloatResolvedInline(resolvedHost, offset, addr, value); return; - case 4u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; - case 5u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; - case 6u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; - case 7u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; - default: std::abort(); + case 4u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, 0u); return; + case 5u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, 0u); return; + case 6u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, 0u); return; + case 7u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, 0u); return; + default: PPC_PsqStResolvedStateFallback(gqr, resolvedHost, offset, addr, value); return; } } } diff --git a/runtime/src/ppc_helpers.cpp b/runtime/src/ppc_helpers.cpp index 06e8488..488f943 100644 --- a/runtime/src/ppc_helpers.cpp +++ b/runtime/src/ppc_helpers.cpp @@ -833,8 +833,8 @@ extern "C" double PPC_PsqL(uint32_t addr, uint32_t w, uint32_t i) } const uint32_t gqr = cpu->gqr[i & 7]; - return w == 0 ? PPC_PsqLStateFallback<0u, 0u, false>(gqr, addr) - : PPC_PsqLStateFallback<1u, 0u, false>(gqr, addr); + return w == 0 ? PPC_PsqLStateFallback<0u, false>(gqr, addr) + : PPC_PsqLStateFallback<1u, false>(gqr, addr); } extern "C" void PPC_PsqSt(uint32_t addr, double value, uint32_t w, uint32_t i) @@ -848,11 +848,11 @@ extern "C" void PPC_PsqSt(uint32_t addr, double value, uint32_t w, uint32_t i) const uint32_t gqr = cpu->gqr[i & 7]; if (w == 0) { - PPC_PsqStStateFallback<0u, 0u, false>(gqr, addr, value); + PPC_PsqStStateFallback<0u, false>(gqr, addr, value); } else { - PPC_PsqStStateFallback<1u, 0u, false>(gqr, addr, value); + PPC_PsqStStateFallback<1u, false>(gqr, addr, value); } } diff --git a/runtime/src/ppc_quantized.cpp b/runtime/src/ppc_quantized.cpp new file mode 100644 index 0000000..4b789d3 --- /dev/null +++ b/runtime/src/ppc_quantized.cpp @@ -0,0 +1,154 @@ +#include "isa/ppc_isa_quantized.h" + +#if defined(__FAST_MATH__) || __FINITE_MATH_ONLY__ +#error "PSQ helpers require strict PPC floating-point options" +#endif + +template +MKW_PPC_NO_INLINE MKW_PPC_COLD double PPC_PsqLStateFallback(uint32_t gqr, uint32_t addr) +{ + static_assert(W <= 1u); + const uint32_t type = (gqr >> 16) & 0x7u; + const uint32_t scale = (gqr >> 24) & 0x3Fu; + if constexpr (W == 0u) + { + switch (type) + { + case 0u: return Stack ? PpcLoadPairPsqFloatStackInline(addr) : PpcLoadPairPsqFloatFastInline(addr); + case 4u: return Stack ? PpcLoadPairPsqIntegerStackInline(addr, scale) : PpcLoadPairPsqIntegerFastInline(addr, scale); + case 5u: return Stack ? PpcLoadPairPsqIntegerStackInline(addr, scale) : PpcLoadPairPsqIntegerFastInline(addr, scale); + case 6u: return Stack ? PpcLoadPairPsqIntegerStackInline(addr, scale) : PpcLoadPairPsqIntegerFastInline(addr, scale); + case 7u: return Stack ? PpcLoadPairPsqIntegerStackInline(addr, scale) : PpcLoadPairPsqIntegerFastInline(addr, scale); + default: std::abort(); + } + } + else + { + switch (type) + { + case 0u: return Stack ? PpcLoadSinglePsqFloatStackInline(addr) : PpcLoadSinglePsqFloatFastInline(addr); + case 4u: return Stack ? PpcLoadSinglePsqQuantizedStackInline(addr, scale) : PpcLoadSinglePsqQuantizedFastInline(addr, scale); + case 5u: return Stack ? PpcLoadSinglePsqQuantizedStackInline(addr, scale) : PpcLoadSinglePsqQuantizedFastInline(addr, scale); + case 6u: return Stack ? PpcLoadSinglePsqQuantizedStackInline(addr, scale) : PpcLoadSinglePsqQuantizedFastInline(addr, scale); + case 7u: return Stack ? PpcLoadSinglePsqQuantizedStackInline(addr, scale) : PpcLoadSinglePsqQuantizedFastInline(addr, scale); + default: std::abort(); + } + } +} + +template +MKW_PPC_NO_INLINE MKW_PPC_COLD void PPC_PsqStStateFallback(uint32_t gqr, uint32_t addr, double value) +{ + static_assert(W <= 1u); + const uint32_t type = gqr & 0x7u; + const uint32_t scale = (gqr >> 8) & 0x3Fu; + if constexpr (W == 0u) + { + switch (type) + { + case 0u: Stack ? PpcStorePairPsqFloatStackInline(addr, value) : PpcStorePairPsqFloatFastInline(addr, value); return; + case 4u: if constexpr (Stack) PpcStorePairPsqQuantizedStackInline(addr, value, scale); else PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; + case 5u: if constexpr (Stack) PpcStorePairPsqQuantizedStackInline(addr, value, scale); else PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; + case 6u: if constexpr (Stack) PpcStorePairPsqQuantizedStackInline(addr, value, scale); else PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; + case 7u: if constexpr (Stack) PpcStorePairPsqQuantizedStackInline(addr, value, scale); else PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; + default: std::abort(); + } + } + else + { + switch (type) + { + case 0u: if constexpr (Stack) PpcStoreSinglePsqFloatStackInline(addr, value); else PpcStoreSinglePsqFloatFastInline(addr, value); return; + case 4u: if constexpr (Stack) PpcStoreSinglePsqQuantizedStackInline(addr, value, scale); else PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; + case 5u: if constexpr (Stack) PpcStoreSinglePsqQuantizedStackInline(addr, value, scale); else PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; + case 6u: if constexpr (Stack) PpcStoreSinglePsqQuantizedStackInline(addr, value, scale); else PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; + case 7u: if constexpr (Stack) PpcStoreSinglePsqQuantizedStackInline(addr, value, scale); else PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; + default: std::abort(); + } + } +} + +template +MKW_PPC_NO_INLINE MKW_PPC_COLD double PPC_PsqLResolvedStateFallback( + uint32_t gqr, uint8_t* resolvedHost, uint32_t offset, uint32_t addr) +{ + static_assert(W <= 1u); + if (!resolvedHost) [[unlikely]] return PPC_PsqLStateInline(gqr, addr); + const uint32_t type = (gqr >> 16) & 0x7u; + const uint32_t scale = (gqr >> 24) & 0x3Fu; + if constexpr (W == 0u) + { + switch (type) + { + case 0u: return PpcLoadPairPsqFloatResolvedInline(resolvedHost, offset, addr); + case 4u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, scale); + case 5u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, scale); + case 6u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, scale); + case 7u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, scale); + default: std::abort(); + } + } + else + { + switch (type) + { + case 0u: return PpcLoadSinglePsqFloatResolvedInline(resolvedHost, offset, addr); + case 4u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, scale); + case 5u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, scale); + case 6u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, scale); + case 7u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, scale); + default: std::abort(); + } + } +} + +template +MKW_PPC_NO_INLINE MKW_PPC_COLD void PPC_PsqStResolvedStateFallback( + uint32_t gqr, uint8_t* resolvedHost, uint32_t offset, uint32_t addr, double value) +{ + static_assert(W <= 1u); + if (!resolvedHost) [[unlikely]] + { + PPC_PsqStStateInline(gqr, addr, value); + return; + } + const uint32_t type = gqr & 0x7u; + const uint32_t scale = (gqr >> 8) & 0x3Fu; + if constexpr (W == 0u) + { + switch (type) + { + case 0u: PpcStorePairPsqFloatResolvedInline(resolvedHost, offset, addr, value); return; + case 4u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; + case 5u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; + case 6u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; + case 7u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; + default: std::abort(); + } + } + else + { + switch (type) + { + case 0u: PpcStoreSinglePsqFloatResolvedInline(resolvedHost, offset, addr, value); return; + case 4u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; + case 5u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; + case 6u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; + case 7u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; + default: std::abort(); + } + } +} + +template double PPC_PsqLStateFallback<0u, false>(uint32_t, uint32_t); +template void PPC_PsqStStateFallback<0u, false>(uint32_t, uint32_t, double); +template double PPC_PsqLStateFallback<0u, true>(uint32_t, uint32_t); +template void PPC_PsqStStateFallback<0u, true>(uint32_t, uint32_t, double); +template double PPC_PsqLResolvedStateFallback<0u>(uint32_t, uint8_t*, uint32_t, uint32_t); +template void PPC_PsqStResolvedStateFallback<0u>(uint32_t, uint8_t*, uint32_t, uint32_t, double); +template double PPC_PsqLStateFallback<1u, false>(uint32_t, uint32_t); +template void PPC_PsqStStateFallback<1u, false>(uint32_t, uint32_t, double); +template double PPC_PsqLStateFallback<1u, true>(uint32_t, uint32_t); +template void PPC_PsqStStateFallback<1u, true>(uint32_t, uint32_t, double); +template double PPC_PsqLResolvedStateFallback<1u>(uint32_t, uint8_t*, uint32_t, uint32_t); +template void PPC_PsqStResolvedStateFallback<1u>(uint32_t, uint8_t*, uint32_t, uint32_t, double); diff --git a/runtime/tests/psq_helpers_tests.cpp b/runtime/tests/psq_helpers_tests.cpp new file mode 100644 index 0000000..f54076d --- /dev/null +++ b/runtime/tests/psq_helpers_tests.cpp @@ -0,0 +1,140 @@ +#include "isa/ppc_isa_quantized.h" + +#include +#include +#include +#include + +static uint64_t checksum = 0; +static unsigned checks = 0; +static void Require(bool condition, const char* message) { + ++checks; + if (!condition) throw std::runtime_error(message); +} +static void Hash(uint64_t value) { checksum = (checksum ^ value) * 1099511628211ull; } + +template static double ReferenceLoad(uint32_t type, uint32_t scale, uint32_t addr) { + if constexpr (W == 0u) { + switch (type) { + case 0: return PpcLoadPairPsqFloatFastInline(addr); + case 4: return PpcLoadPairPsqIntegerFastInline(addr, scale); + case 5: return PpcLoadPairPsqIntegerFastInline(addr, scale); + case 6: return PpcLoadPairPsqIntegerFastInline(addr, scale); + case 7: return PpcLoadPairPsqIntegerFastInline(addr, scale); + } + } else { + switch (type) { + case 0: return PpcLoadSinglePsqFloatFastInline(addr); + case 4: return PpcLoadSinglePsqQuantizedFastInline(addr, scale); + case 5: return PpcLoadSinglePsqQuantizedFastInline(addr, scale); + case 6: return PpcLoadSinglePsqQuantizedFastInline(addr, scale); + case 7: return PpcLoadSinglePsqQuantizedFastInline(addr, scale); + } + } + std::abort(); +} +template static void ReferenceStore(uint32_t type, uint32_t scale, uint32_t addr, double value) { + if constexpr (W == 0u) { + switch (type) { + case 0: PpcStorePairPsqFloatFastInline(addr, value); return; + case 4: PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; + case 5: PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; + case 6: PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; + case 7: PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; + } + } else { + switch (type) { + case 0: PpcStoreSinglePsqFloatFastInline(addr, value); return; + case 4: PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; + case 5: PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; + case 6: PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; + case 7: PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; + } + } + std::abort(); +} + +template static void Check() { + constexpr uint32_t edges[] = { + 0, 0x80000000u, 1, 0x80000001u, 0x007FFFFFu, 0x00800000u, + 0x3F000000u, 0xBF000000u, 0x3F800000u, 0xBF800000u, + 0x437F0000u, 0x477FFF00u, 0xC7000000u, 0x7F7FFFFFu, + 0x7F800000u, 0xFF800000u, 0x7F800001u, 0x7FC01234u, 0xFFC01234u + }; + for (uint32_t type : {0u, 4u, 5u, 6u, 7u}) for (uint32_t scale = 0; scale < 64; ++scale) + for (unsigned n = 0; n < std::size(edges); ++n) { + // Include ignored GQR bits and a different type/scale in the unused half. + const uint32_t half = (scale << 8) | type | 0xC0F8u; + const uint32_t loadGqr = (half << 16) | 0x2105u; + const uint32_t storeGqr = half | 0x21050000u; + const uint64_t raw = (uint64_t(edges[n]) << 32) | edges[(n + 7) % std::size(edges)]; + const double value = PpcBitCastToDoubleInline(raw); + for (unsigned mode = 0; mode < 5; ++mode) { + const uint32_t addr = mode == 0 ? 32u : mode == 1 ? 0xCC008000u : mode == 2 ? 0u : 0xFFFFFF00u; + constexpr uint32_t offset = 17; + PsqTestMemory::directStack = mode == 4; + PsqTestMemory::bytes.fill(0xCD); + BigEndian::Write64(PsqTestMemory::Pointer(addr), raw); + const auto expected = PpcBitCastToU64Inline(ReferenceLoad(type, scale, addr)); + PsqTestMemory::accesses = 0; + double loaded; + if (mode == 0) loaded = PPC_PsqLResolvedStateInline(loadGqr, PsqTestMemory::Pointer(addr) - offset, offset, addr); + else if (mode == 1) loaded = PPC_PsqLResolvedStateInline(loadGqr, nullptr, offset, addr); + else if (mode == 2) loaded = PPC_PsqLStateInline(loadGqr, addr); + else loaded = PPC_PsqLStateInline(loadGqr, addr); + Require(PpcBitCastToU64Inline(loaded) == expected, "load bits / lane order"); + Hash(expected); + const size_t width = (type == 0 ? 4u : (type == 4 || type == 6) ? 1u : 2u) * (2u - W); + Require(PsqTestMemory::accesses == ((mode == 0 || mode == 4) ? 0u : 1u), "load route"); + if (PsqTestMemory::accesses) Require(PsqTestMemory::address == addr && PsqTestMemory::width == width, "load slow address / width"); + + PsqTestMemory::bytes.fill(0xCD); + ReferenceStore(type, scale, addr, value); + const auto expectedBytes = PsqTestMemory::bytes; + PsqTestMemory::bytes.fill(0xCD); + PsqTestMemory::accesses = 0; + if (mode == 0) PPC_PsqStResolvedStateInline(storeGqr, PsqTestMemory::Pointer(addr) - offset, offset, addr, value); + else if (mode == 1) PPC_PsqStResolvedStateInline(storeGqr, nullptr, offset, addr, value); + else if (mode == 2) PPC_PsqStStateInline(storeGqr, addr, value); + else PPC_PsqStStateInline(storeGqr, addr, value); + Require(PsqTestMemory::bytes == expectedBytes, "store bits / untouched lanes"); + Hash(PsqTestMemory::ReadHost(PsqTestMemory::Pointer(addr))); + Require(PsqTestMemory::accesses == ((mode == 0 || mode == 4) ? 0u : 1u), "store route"); + if (PsqTestMemory::accesses) Require(PsqTestMemory::address == addr && PsqTestMemory::width == width, "store slow address / width"); + } + } +} + +int main(int argc, char** argv) { + try { + if (argc > 1) { + std::signal(SIGABRT, [](int) { std::_Exit(86); }); + const uint32_t type = static_cast(std::stoul(argv[2])); + const bool single = std::string(argv[3]) == "1"; + uint8_t* host = std::string(argv[4]) == "resolved" ? PsqTestMemory::bytes.data() : nullptr; + if (std::string(argv[1]) == "load") { + if (single) PPC_PsqLResolvedStateInline<1, 7>(type << 16, host, 0, 0); + else PPC_PsqLResolvedStateInline<0, 7>(type << 16, host, 0, 0); + } else { + if (single) PPC_PsqStResolvedStateInline<1, 7>(type, host, 0, 0, 0); + else PPC_PsqStResolvedStateInline<0, 7>(type, host, 0, 0, 0); + } + return 0; + } + const auto saved = MkwGetHostFpControl(); + CpuContext ctx{}; + for (uint32_t ni : {0u, 4u}) for (uint32_t rounding = 0; rounding < 4; ++rounding) { + ctx.fpscr = ni; + CpuContextScope scope(&ctx); +#if defined(__x86_64__) + MkwSetHostFpControl((MkwGetHostFpControl() & ~(3u << 13)) | (rounding << 13)); +#endif + Check<0, 0>(); Check<1, 0>(); Check<0, 7>(); Check<1, 7>(); + } + MkwRestoreHostMxcsr(saved); + std::cout << "passed " << checks << " checks, checksum " << std::hex << checksum << '\n'; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/runtime/tests/psq_memory/ppc_isa_memory.h b/runtime/tests/psq_memory/ppc_isa_memory.h new file mode 100644 index 0000000..7e673b1 --- /dev/null +++ b/runtime/tests/psq_memory/ppc_isa_memory.h @@ -0,0 +1,83 @@ +#pragma once + +// Instrumented ISA memory seam: no guest VM reservation or GPU is needed. +#include "big_endian.h" +#include +#include +#include + +namespace PsqTestMemory { +inline std::array bytes{}; +inline uint32_t address = 0; +inline size_t width = 0; +inline unsigned accesses = 0; +inline bool directStack = false; +inline uint8_t* Pointer(uint32_t addr) { return bytes.data() + (addr & 255u); } +inline void Record(uint32_t addr, size_t size) { address = addr; width = size; ++accesses; } +template T ReadHost(const uint8_t* host) { + if constexpr (sizeof(T) == 1) return *host; + else if constexpr (sizeof(T) == 2) return BigEndian::Read16(host); + else if constexpr (sizeof(T) == 4) return BigEndian::Read32(host); + else return (uint64_t(BigEndian::Read32(host)) << 32) | BigEndian::Read32(host + 4); +} +template void WriteHost(uint8_t* host, T value) { + if constexpr (sizeof(T) == 1) *host = value; + else if constexpr (sizeof(T) == 2) BigEndian::Write16(host, value); + else if constexpr (sizeof(T) == 4) BigEndian::Write32(host, value); + else BigEndian::Write64(host, value); +} +template T Read(uint32_t addr) { + Record(addr, sizeof(T)); return ReadHost(Pointer(addr)); +} +template void Write(uint32_t addr, T value) { + Record(addr, sizeof(T)); WriteHost(Pointer(addr), value); +} +} + +namespace GuestFlat { +inline bool RequiresCheckedAccess() noexcept { return true; } +} +#define MKW_FLAT_GUEST_BASE (PsqTestMemory::bytes.data()) + +class Memory { +public: + static uint8_t Read8(uint32_t a) { return PsqTestMemory::Read(a); } + static uint16_t Read16(uint32_t a) { return PsqTestMemory::Read(a); } + static uint32_t Read32(uint32_t a) { return PsqTestMemory::Read(a); } + static uint64_t Read64(uint32_t a) { return PsqTestMemory::Read(a); } + static void Write8(uint32_t a, uint8_t v) { PsqTestMemory::Write(a, v); } + static void Write16(uint32_t a, uint16_t v) { PsqTestMemory::Write(a, v); } + static void Write32(uint32_t a, uint32_t v) { PsqTestMemory::Write(a, v); } + static void Write64(uint32_t a, uint64_t v) { PsqTestMemory::Write(a, v); } +}; + +namespace MemoryInline { +inline bool FlatWriteNeedsPolicy(uint32_t) { return true; } +inline bool TryGetPointerFast(uint32_t a, size_t, uint8_t*& host) { + host = PsqTestMemory::directStack ? PsqTestMemory::Pointer(a) : nullptr; + return host != nullptr; +} +inline bool TryGetWritablePointerFast(uint32_t a, size_t n, uint8_t*& host) { + return TryGetPointerFast(a, n, host); +} +template T ReadResolvedFallback(uint32_t a) { return PsqTestMemory::Read(a); } +template void WriteResolvedFallback(uint32_t a, T v) { PsqTestMemory::Write(a, v); } +template T ReadResolved(uint8_t* host, uint32_t o, uint32_t a) { + return host ? PsqTestMemory::ReadHost(host + o) : ReadResolvedFallback(a); +} +template void WriteResolved(uint8_t* host, uint32_t o, uint32_t a, T v) { + if (host) PsqTestMemory::WriteHost(host + o, v); else WriteResolvedFallback(a, v); +} +#define PSQ_TEST_MEMORY_WIDTH(Bits, Type) \ +inline Type ReadStack##Bits(uint32_t a) { \ + return PsqTestMemory::directStack ? PsqTestMemory::ReadHost(PsqTestMemory::Pointer(a)) : PsqTestMemory::Read(a); } \ +inline void WriteStack##Bits(uint32_t a, Type v) { \ + if (PsqTestMemory::directStack) PsqTestMemory::WriteHost(PsqTestMemory::Pointer(a), v); else PsqTestMemory::Write(a, v); } \ +inline Type ReadResolved##Bits(uint8_t* h, uint32_t o, uint32_t a) { return ReadResolved(h, o, a); } \ +inline void WriteResolved##Bits(uint8_t* h, uint32_t o, uint32_t a, Type v) { WriteResolved(h, o, a, v); } +PSQ_TEST_MEMORY_WIDTH(8, uint8_t) +PSQ_TEST_MEMORY_WIDTH(16, uint16_t) +PSQ_TEST_MEMORY_WIDTH(32, uint32_t) +PSQ_TEST_MEMORY_WIDTH(64, uint64_t) +#undef PSQ_TEST_MEMORY_WIDTH +} diff --git a/runtime/tests/psq_reserved_tests.cmake b/runtime/tests/psq_reserved_tests.cmake new file mode 100644 index 0000000..cdff941 --- /dev/null +++ b/runtime/tests/psq_reserved_tests.cmake @@ -0,0 +1,13 @@ +foreach(operation load store) + foreach(type 1 2 3) + foreach(lanes 0 1) + foreach(route resolved null) + execute_process(COMMAND "${PSQ_TEST_EXECUTABLE}" ${operation} ${type} ${lanes} ${route} + RESULT_VARIABLE result OUTPUT_VARIABLE output ERROR_VARIABLE error) + if(NOT result EQUAL 86) + message(FATAL_ERROR "Reserved PSQ ${operation}/${type}/${lanes}/${route}: ${result} ${output} ${error}") + endif() + endforeach() + endforeach() + endforeach() +endforeach() diff --git a/translator/tests/Translator.Tests/TranslatorCppTestHarness.cs b/translator/tests/Translator.Tests/TranslatorCppTestHarness.cs index e251b61..23482b2 100644 --- a/translator/tests/Translator.Tests/TranslatorCppTestHarness.cs +++ b/translator/tests/Translator.Tests/TranslatorCppTestHarness.cs @@ -15,6 +15,7 @@ internal static class TranslatorCppTestHarness Path.Combine("runtime", "src", "fpu_helpers.cpp"), Path.Combine("runtime", "src", "memory.cpp"), Path.Combine("runtime", "src", "ppc_helpers.cpp"), + Path.Combine("runtime", "src", "ppc_quantized.cpp"), }; public static string BuildCompileArguments( @@ -31,6 +32,7 @@ internal static class TranslatorCppTestHarness args.Append("-std=c++17 "); args.Append("-D_CRT_SECURE_NO_WARNINGS "); args.Append("-march=x86-64-v3 "); + args.Append("-fno-fast-math -ffp-contract=off "); if (!RuntimeHeadersDefineRestrictMacro(repoRoot)) { // Fallback for the window between an emitter change using MKW_RESTRICT and the runtime