From 6ddc153e443214519e2176b66fc0e5ba2811ab3a Mon Sep 17 00:00:00 2001 From: theofficialgman <28281419+theofficialgman@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:37:17 -0400 Subject: [PATCH] add native aarch64 (arm64) support use -mcpu=native on arm64 targets build-appimage.sh kernel architecture detection --- Launcher/build-appimage.sh | 57 +++++-- runtime/CMakeLists.txt | 4 +- runtime/cmake/PublicProducts.cmake | 19 ++- runtime/include/guest_flat_memory.h | 20 ++- runtime/include/isa/ppc_isa_config.h | 6 + runtime/include/isa/ppc_isa_context.h | 2 +- runtime/include/isa/ppc_isa_float.h | 197 ++++++++++++++++++++---- runtime/include/isa/ppc_isa_fpenv.h | 79 +++++++--- runtime/include/isa/ppc_isa_quantized.h | 52 +++++++ runtime/src/guest_flat_memory.cpp | 8 +- runtime/src/host_cpu_baseline.cpp | 15 ++ 11 files changed, 390 insertions(+), 69 deletions(-) mode change 100644 => 100755 Launcher/build-appimage.sh diff --git a/Launcher/build-appimage.sh b/Launcher/build-appimage.sh old mode 100644 new mode 100755 index fe56749..0fb3311 --- a/Launcher/build-appimage.sh +++ b/Launcher/build-appimage.sh @@ -21,6 +21,42 @@ set -euo pipefail script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) workspace=$(cd "$script_dir/.." && pwd) +# `uname -m` reports the *kernel's* architecture, which can differ from userspace - an aarch64 +# kernel can run a 32-bit armhf userland (as shipped by 32-bit Raspberry Pi OS), same as an x86_64 +# kernel can run an i686 one. What matters here is which userspace binaries (dotnet, appimagetool) +# will actually run, so this reads the ELF header of this script's own running bash interpreter - +# real userspace - rather than trusting the kernel's self-report. /proc/$$/exe (not /proc/self/exe: +# that would resolve inside the readlink subprocess below, to readlink itself, not to bash) is this +# shell's own PID. EI_CLASS (byte 4: 1=32-bit, 2=64-bit) and e_machine (bytes 18-19: 3=EM_386, +# 40=EM_ARM, 62=EM_X86_64, 183=EM_AARCH64) are read as plain little-endian bytes, which every +# real-world x86/ARM Linux userland uses; ELF's big-endian encoding is a non-issue here since no +# Linux distro ships a big-endian x86 or ARM userland. +elf_exe=$(readlink -f "/proc/$$/exe") +elf_class=$(od -An -t u1 -j 4 -N 1 "$elf_exe" | tr -d ' ') +elf_machine_lo=$(od -An -t u1 -j 18 -N 1 "$elf_exe" | tr -d ' ') +elf_machine_hi=$(od -An -t u1 -j 19 -N 1 "$elf_exe" | tr -d ' ') +elf_machine=$(( elf_machine_hi * 256 + elf_machine_lo )) + +# Mirrors the host-architecture detection NodToolProvider.cs already does (RuntimeInformation. +# OSArchitecture) so this script's own dotnet RID and appimagetool selection agree with the +# nodtool binary that same code path resolves below. local-build.sh needs no such mapping itself: +# it just drives the native CMake configure, which already accepts x86_64 or aarch64 natively +# (see runtime/CMakeLists.txt's CMAKE_SYSTEM_PROCESSOR check). +case "$elf_class:$elf_machine" in + 2:62) + dotnet_rid=linux-x64 + appimagetool_arch=x86_64 + ;; + 2:183) + dotnet_rid=linux-arm64 + appimagetool_arch=aarch64 + ;; + *) + echo "build-appimage.sh: unsupported userspace architecture (ELF class $elf_class, machine $elf_machine) - WiiCompiled requires a 64-bit x86_64 or aarch64 userland" >&2 + exit 1 + ;; +esac + output_dir="$workspace/Launcher/dist" appimagetool_override="" @@ -40,10 +76,10 @@ appdir="$workspace/Launcher/artifacts/appimage-build/AppDir" rm -rf "$appdir" mkdir -p "$appdir/usr/bin" "$appdir/workspace/Launcher" -echo "Publishing the installer (self-contained linux-x64)..." +echo "Publishing the installer (self-contained $dotnet_rid)..." publish_tmp="$workspace/Launcher/artifacts/appimage-build/publish" rm -rf "$publish_tmp" -dotnet publish "$workspace/Launcher/WiiCompiled.Setup.Linux" -c Release -r linux-x64 \ +dotnet publish "$workspace/Launcher/WiiCompiled.Setup.Linux" -c Release -r "$dotnet_rid" \ --self-contained -p:PublishSingleFile=true -p:EnableCompressionInSingleFile=true \ -o "$publish_tmp" cp "$publish_tmp/WiiCompiled.Setup.Linux" "$appdir/usr/bin/wiicompiled-setup" @@ -52,10 +88,10 @@ chmod +x "$appdir/usr/bin/wiicompiled-setup" # Published as a self-contained binary too, so an AppImage user never needs a `dotnet` SDK on # PATH at all - local-build.sh is told about it via --translator-bin and skips its own # dotnet-build-from-source step entirely (see local-build.sh's translator resolution branch). -echo "Publishing the translator (self-contained linux-x64)..." +echo "Publishing the translator (self-contained $dotnet_rid)..." translator_publish_tmp="$workspace/Launcher/artifacts/appimage-build/publish-translator" rm -rf "$translator_publish_tmp" -dotnet publish "$workspace/translator/src/Translator.Cli" -c Release -r linux-x64 \ +dotnet publish "$workspace/translator/src/Translator.Cli" -c Release -r "$dotnet_rid" \ --self-contained -p:PublishSingleFile=true -p:EnableCompressionInSingleFile=true \ -o "$translator_publish_tmp" cp "$translator_publish_tmp/Translator.Cli" "$appdir/usr/bin/translator-cli" @@ -155,11 +191,13 @@ PY echo "Resolving appimagetool..." appimagetool="$appimagetool_override" if [[ -z "$appimagetool" ]]; then - appimagetool="$workspace/Launcher/artifacts/appimagetool" + # Cache path is arch-tagged so a workspace shared or synced across an x86_64 and an aarch64 + # machine never picks up the wrong architecture's cached binary. + appimagetool="$workspace/Launcher/artifacts/appimagetool-$appimagetool_arch" if [[ ! -x "$appimagetool" ]]; then - echo "Downloading appimagetool..." + echo "Downloading appimagetool ($appimagetool_arch)..." mkdir -p "$(dirname "$appimagetool")" - curl -fsSL "https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-x86_64.AppImage" \ + curl -fsSL "https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-$appimagetool_arch.AppImage" \ -o "$appimagetool" chmod +x "$appimagetool" fi @@ -169,5 +207,6 @@ mkdir -p "$output_dir" echo "Packaging..." # appimagetool detects the target architecture from the first ELF executable it finds in the # AppDir; AppRun here is a shell script, not ELF, so ARCH must be set explicitly. -ARCH=x86_64 "$appimagetool" "$appdir" "$output_dir/WiiCompiled-Setup-x86_64.AppImage" -echo "Built: $output_dir/WiiCompiled-Setup-x86_64.AppImage" +output_name="WiiCompiled-Setup-$appimagetool_arch.AppImage" +ARCH="$appimagetool_arch" "$appimagetool" "$appdir" "$output_dir/$output_name" +echo "Built: $output_dir/$output_name" diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 9a3aad1..ba948ee 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -6,8 +6,8 @@ if((NOT (WIN32 AND MINGW)) AND (NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")) endif() if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR NOT CMAKE_SIZEOF_VOID_P EQUAL 8 OR - NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64|X86_64)$") - message(FATAL_ERROR "WiiCompiled requires 64-bit Clang targeting x86_64") + NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64|X86_64|aarch64|arm64|ARM64)$") + message(FATAL_ERROR "WiiCompiled requires 64-bit Clang targeting x86_64 or aarch64") endif() if(NOT CMAKE_BUILD_TYPE STREQUAL "Release") message(FATAL_ERROR "WiiCompiled only supports Release builds") diff --git a/runtime/cmake/PublicProducts.cmake b/runtime/cmake/PublicProducts.cmake index d1fff5e..5a8688f 100644 --- a/runtime/cmake/PublicProducts.cmake +++ b/runtime/cmake/PublicProducts.cmake @@ -298,11 +298,26 @@ else() message(STATUS "RetroRewind target disabled (run translate-mod and emit-build-shards)") endif() +# x86-64-v3 (SSE3/SSSE3/SSE4.1/FMA/AVX2/BMI2) is the baseline runtime/src/host_cpu_baseline.cpp +# guards against - a fixed, portable floor since an x86_64 build may run on a different machine +# than the one that built it. AArch64 has no such redistribution path here: every build this +# project produces runs only on the machine that built it (local-build.sh, and the AppImage which +# wraps it, always build from source on the target), so -mcpu=native is safe and strictly better - +# real per-core tuning (scheduling, whatever NEON/atomic extensions that exact CPU actually has) +# instead of the generic armv8-a baseline Clang would otherwise assume. +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64|X86_64)$") + set(MKW_BASELINE_ARCH_FLAG -march=x86-64-v3) +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64)$") + set(MKW_BASELINE_ARCH_FLAG -mcpu=native) +else() + set(MKW_BASELINE_ARCH_FLAG "") +endif() + set(MKW_ALL_BUILD_TARGETS mkw_runtime_common mkw_base_shared mkw_base_sensitive mkw_retro_sensitive mkw_retro_rewind_functions WiiCompiled RetroRewind) foreach(target IN LISTS MKW_ALL_BUILD_TARGETS) - if(TARGET ${target}) - target_compile_options(${target} PRIVATE -march=x86-64-v3) + if(TARGET ${target} AND MKW_BASELINE_ARCH_FLAG) + target_compile_options(${target} PRIVATE ${MKW_BASELINE_ARCH_FLAG}) endif() endforeach() diff --git a/runtime/include/guest_flat_memory.h b/runtime/include/guest_flat_memory.h index e33573a..0973ae7 100644 --- a/runtime/include/guest_flat_memory.h +++ b/runtime/include/guest_flat_memory.h @@ -12,10 +12,26 @@ namespace GuestFlat { // Fixed base so the emitted access is `[reg + imm64-in-register]` with no load -// of a global. 16 TiB: clear of the Windows ASan shadow (32 TiB) and of the -// usual image/heap placement. +// of a global. inline constexpr uint64_t kGuestSpaceSize = 0x1'0000'0000ull; +#if defined(__x86_64__) +// 16 TiB: clear of the Windows ASan shadow (32 TiB) and of the usual image/heap +// placement. inline constexpr uintptr_t kFixedFlatGuestBase = 0x0000'1000'0000'0000ull; +#elif defined(__aarch64__) +// 16 TiB (this arch's x86_64 sibling value) is unreachable on any AArch64 +// kernel configured for 39-bit virtual addresses (512 GiB ceiling) - common on +// older/embedded targets (e.g. this project's own tested Jetson/L4T board, kernel +// 4.9). There mmap() silently ignores a hint above the ceiling and hands back an +// address near the top of the real range instead, which this module's caller +// then rejects as "already occupied". 64 GiB is reachable on every AArch64 VA +// width in real use (39-bit minimum and up) and was confirmed via direct mmap +// probing to sit far below where the PIE image, heap, shared libraries and +// stack actually land (all clustered above ~340 GiB on a 39-bit/512 GiB system). +inline constexpr uintptr_t kFixedFlatGuestBase = 0x0000'0010'0000'0000ull; +#else +#error "guest_flat_memory.h has no fixed flat guest base chosen for this architecture" +#endif #define MKW_FLAT_GUEST_BASE (reinterpret_cast(GuestFlat::kFixedFlatGuestBase)) diff --git a/runtime/include/isa/ppc_isa_config.h b/runtime/include/isa/ppc_isa_config.h index f4c41ea..b062184 100644 --- a/runtime/include/isa/ppc_isa_config.h +++ b/runtime/include/isa/ppc_isa_config.h @@ -4,7 +4,13 @@ #include #define MKW_RESTRICT __restrict +#if defined(__x86_64__) #include +#elif defined(__aarch64__) +#include +#else +#error "ppc_isa_config.h has no SIMD intrinsics header for this architecture" +#endif inline constexpr bool MkwStateFreeAbiEnabled(uint32_t) noexcept { diff --git a/runtime/include/isa/ppc_isa_context.h b/runtime/include/isa/ppc_isa_context.h index 0c9381e..227987e 100644 --- a/runtime/include/isa/ppc_isa_context.h +++ b/runtime/include/isa/ppc_isa_context.h @@ -62,7 +62,7 @@ public: { g_currentCpuContext = ctx; - savedMxcsr_ = _mm_getcsr(); + savedMxcsr_ = MkwGetHostFpControl(); if (ctx != nullptr) MkwApplyHostNiMode(ctx->fpscr); } diff --git a/runtime/include/isa/ppc_isa_float.h b/runtime/include/isa/ppc_isa_float.h index fcd5795..f59cb84 100644 --- a/runtime/include/isa/ppc_isa_float.h +++ b/runtime/include/isa/ppc_isa_float.h @@ -78,37 +78,71 @@ inline void PpcSetPairedFprInline(PPC_FPR& fpr, double packed) fpr.d = packed; } -// Must stay inside the XMM register domain. Bitcasting through a 64-bit GPR added a movq +#if defined(__x86_64__) +using PpcPairVec = __m128; +#elif defined(__aarch64__) +// Only 2 lanes are ever meaningful (a PPC paired-single register), so a 2-lane float32x2_t +// (one 64-bit D register) is a more natural fit than mirroring x86's 128-bit register usage. +using PpcPairVec = float32x2_t; +#endif + +// Must stay inside the vector register domain. Bitcasting through a 64-bit GPR added a movq // domain crossing on every paired-single op (630 in the THP IDCT region alone); a double -// local already lives in an XMM register, so these casts compile to nothing. -inline __m128 PpcPsToM128Inline(double value) +// local already lives in a vector register, so these casts compile to nothing. +inline PpcPairVec PpcPsToM128Inline(double value) { +#if defined(__x86_64__) return _mm_castpd_ps(_mm_set_sd(value)); +#elif defined(__aarch64__) + return vreinterpret_f32_f64(vdup_n_f64(value)); +#endif } -inline double PpcM128ToPsInline(__m128 value) +inline double PpcM128ToPsInline(PpcPairVec value) { +#if defined(__x86_64__) return _mm_cvtsd_f64(_mm_castps_pd(value)); +#elif defined(__aarch64__) + return vget_lane_f64(vreinterpret_f64_f32(value), 0); +#endif } -inline __m128 PpcBroadcastPs0Inline(double value) +inline PpcPairVec PpcBroadcastPs0Inline(double value) { +#if defined(__x86_64__) const __m128 lanes = PpcPsToM128Inline(value); return _mm_shuffle_ps(lanes, lanes, _MM_SHUFFLE(1, 1, 1, 1)); +#elif defined(__aarch64__) + // ps0 lives in lane 1 (see the lane-accessor comment below). + return vdup_lane_f32(PpcPsToM128Inline(value), 1); +#endif } -inline __m128 PpcBroadcastPs1Inline(double value) +inline PpcPairVec PpcBroadcastPs1Inline(double value) { +#if defined(__x86_64__) const __m128 lanes = PpcPsToM128Inline(value); return _mm_shuffle_ps(lanes, lanes, _MM_SHUFFLE(0, 0, 0, 0)); +#elif defined(__aarch64__) + // ps1 is already lane 0. + return vdup_lane_f32(PpcPsToM128Inline(value), 0); +#endif } -inline __m128 PpcNegateNonNanLanesInline(__m128 value) +inline PpcPairVec PpcNegateNonNanLanesInline(PpcPairVec value) { +#if defined(__x86_64__) const __m128 signMask = _mm_castsi128_ps(_mm_set1_epi32(static_cast(0x80000000u))); const __m128 negated = _mm_xor_ps(value, signMask); const __m128 ordered = _mm_cmpord_ps(value, value); return _mm_or_ps(_mm_and_ps(ordered, negated), _mm_andnot_ps(ordered, value)); +#elif defined(__aarch64__) + const uint32x2_t signMask = vdup_n_u32(0x80000000u); + const float32x2_t negated = vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(value), signMask)); + // NEON has no direct "ordered compare"; a value compares equal to itself iff it's not NaN. + const uint32x2_t ordered = vceq_f32(value, value); + return vbsl_f32(ordered, negated, value); +#endif } // Paired-single lane accessors. The packed double's LOW 32 bits hold ps1 and HIGH 32 bits @@ -119,20 +153,34 @@ inline __m128 PpcNegateNonNanLanesInline(__m128 value) inline float PpcGetPs0Inline(double value) { // ps0 lives in lane 1; PpcBroadcastPs0Inline already splats it. +#if defined(__x86_64__) return _mm_cvtss_f32(PpcBroadcastPs0Inline(value)); +#elif defined(__aarch64__) + return vget_lane_f32(PpcBroadcastPs0Inline(value), 0); +#endif } inline float PpcGetPs1Inline(double value) { // ps1 is already lane 0 of the packed representation. +#if defined(__x86_64__) return _mm_cvtss_f32(PpcPsToM128Inline(value)); +#elif defined(__aarch64__) + return vget_lane_f32(PpcPsToM128Inline(value), 0); +#endif } inline double PpcPackPairedInline(float ps0, float ps1) { +#if defined(__x86_64__) // _mm_unpacklo_ps(x, y) -> { x[0], y[0], x[1], y[1] }, so lane 0 becomes // ps1 and lane 1 becomes ps0, matching the union layout bit for bit. return PpcM128ToPsInline(_mm_unpacklo_ps(_mm_set_ss(ps1), _mm_set_ss(ps0))); +#elif defined(__aarch64__) + // Lane 0 = ps1, lane 1 = ps0, matching the union layout bit for bit. + const float32x2_t lane0 = vdup_n_f32(ps1); + return PpcM128ToPsInline(vset_lane_f32(ps0, lane0, 1)); +#endif } // FPSCR[NI] is modeled by MXCSR FTZ/DAZ, so arithmetic output flushing compiles to nothing. @@ -148,14 +196,24 @@ inline float PpcForceSingleValueInline(double value) // FPSCR[NI] flushes an exact pre-round single-subnormal even when rounding would promote it // to the smallest normal. g_mkwNiFlushThreshold (2^-126 active, 0.0 inactive) turns the // flush into a branchless mask: a set compare lane keeps just the sign bit, a clear lane - // passes the value to CVTSD2SS. DAZ (set exactly when NI is) makes the compare itself treat - // a subnormal as zero, matching the mask's answer. + // passes the value to the double->float conversion. DAZ/FZ (set exactly when NI is) makes + // the compare itself treat a subnormal as zero, matching the mask's answer. +#if defined(__x86_64__) const __m128d v = _mm_set_sd(value); const __m128d signMask = _mm_set_sd(-0.0); const __m128d magnitude = _mm_andnot_pd(signMask, v); const __m128d flush = _mm_cmplt_sd(magnitude, _mm_set_sd(g_mkwNiFlushThreshold)); const __m128d kept = _mm_andnot_pd(_mm_andnot_pd(signMask, flush), v); return static_cast(_mm_cvtsd_f64(kept)); +#elif defined(__aarch64__) + const float64x1_t v = vdup_n_f64(value); + const uint64x1_t signMask = vdup_n_u64(0x8000000000000000ULL); + const float64x1_t magnitude = vreinterpret_f64_u64(vbic_u64(vreinterpret_u64_f64(v), signMask)); + const uint64x1_t flush = vclt_f64(magnitude, vdup_n_f64(g_mkwNiFlushThreshold)); + const uint64x1_t signOnly = vand_u64(vreinterpret_u64_f64(v), signMask); + const uint64x1_t kept = vbsl_u64(flush, signOnly, vreinterpret_u64_f64(v)); + return static_cast(vget_lane_f64(vreinterpret_f64_u64(kept), 0)); +#endif } inline float PpcFlushSingleForNiInline(float value) @@ -468,15 +526,24 @@ inline double PpcFnmsubsInline(double a, double c, double b) return static_cast(std::isnan(result) ? result : -result); } +inline PpcPairVec PpcMulPairInline(PpcPairVec lhs, PpcPairVec rhs) +{ +#if defined(__x86_64__) + return _mm_mul_ps(lhs, rhs); +#elif defined(__aarch64__) + return vmul_f32(lhs, rhs); +#endif +} + inline double PPC_PsMulInline(double lhs, double rhs) { return PpcFlushPairedForNiInline( - PpcM128ToPsInline(_mm_mul_ps(PpcPsToM128Inline(lhs), PpcPsToM128Inline(rhs)))); + PpcM128ToPsInline(PpcMulPairInline(PpcPsToM128Inline(lhs), PpcPsToM128Inline(rhs)))); } inline double PPC_PsMulNoNiInline(double lhs, double rhs) { - return PpcM128ToPsInline(_mm_mul_ps(PpcPsToM128Inline(lhs), PpcPsToM128Inline(rhs))); + return PpcM128ToPsInline(PpcMulPairInline(PpcPsToM128Inline(lhs), PpcPsToM128Inline(rhs))); } // The paired madd family lowers to one hardware FMA. Semantics match the scalar lanes exactly: a @@ -485,9 +552,31 @@ inline double PPC_PsMulNoNiInline(double lhs, double rhs) // PpcNegateNonNanLanesInline. NI flushing is handled by MXCSR (see // MkwApplyHostNiMode), so the NI and NoNi entry points are identical here. +// Fused multiply-add/subtract on a pair. NEON's vfma_f32(acc, a, b) = acc + a*b has an +// accumulator-first operand order, unlike x86's _mm_fmadd_ps(a, b, c) = a*b + c - msub is +// therefore expressed as an fma against a negated accumulator on both architectures, not a +// dedicated fms intrinsic, so the two branches stay structurally parallel. +inline PpcPairVec PpcFmaddPairInline(PpcPairVec multiplicand, PpcPairVec multiplier, PpcPairVec addend) +{ +#if defined(__x86_64__) + return _mm_fmadd_ps(multiplicand, multiplier, addend); +#elif defined(__aarch64__) + return vfma_f32(addend, multiplicand, multiplier); +#endif +} + +inline PpcPairVec PpcFmsubPairInline(PpcPairVec multiplicand, PpcPairVec multiplier, PpcPairVec subtractor) +{ +#if defined(__x86_64__) + return _mm_fmsub_ps(multiplicand, multiplier, subtractor); +#elif defined(__aarch64__) + return vfma_f32(vneg_f32(subtractor), multiplicand, multiplier); +#endif +} + inline double PPC_PsMsubInline(double multiplicand, double multiplier, double subtractor) { - return PpcM128ToPsInline(_mm_fmsub_ps( + return PpcM128ToPsInline(PpcFmsubPairInline( PpcPsToM128Inline(multiplicand), PpcPsToM128Inline(multiplier), PpcPsToM128Inline(subtractor))); } @@ -498,7 +587,7 @@ inline double PPC_PsMsubNoNiInline(double multiplicand, double multiplier, doubl inline double PPC_PsMaddInline(double multiplicand, double multiplier, double addend) { - return PpcM128ToPsInline(_mm_fmadd_ps( + return PpcM128ToPsInline(PpcFmaddPairInline( PpcPsToM128Inline(multiplicand), PpcPsToM128Inline(multiplier), PpcPsToM128Inline(addend))); } @@ -509,19 +598,19 @@ inline double PPC_PsMaddNoNiInline(double multiplicand, double multiplier, doubl inline double PPC_PsMadds0Inline(double multiplicand, double multiplier, double addend) { - return PpcM128ToPsInline(_mm_fmadd_ps( + return PpcM128ToPsInline(PpcFmaddPairInline( PpcPsToM128Inline(multiplicand), PpcBroadcastPs0Inline(multiplier), PpcPsToM128Inline(addend))); } inline double PPC_PsMadds1Inline(double multiplicand, double multiplier, double addend) { - return PpcM128ToPsInline(_mm_fmadd_ps( + return PpcM128ToPsInline(PpcFmaddPairInline( PpcPsToM128Inline(multiplicand), PpcBroadcastPs1Inline(multiplier), PpcPsToM128Inline(addend))); } inline double PPC_PsNmsubInline(double multiplicand, double multiplier, double subtractor) { - return PpcM128ToPsInline(PpcNegateNonNanLanesInline(_mm_fmsub_ps( + return PpcM128ToPsInline(PpcNegateNonNanLanesInline(PpcFmsubPairInline( PpcPsToM128Inline(multiplicand), PpcPsToM128Inline(multiplier), PpcPsToM128Inline(subtractor)))); } @@ -532,20 +621,20 @@ inline double PPC_PsNmsubNoNiInline(double multiplicand, double multiplier, doub inline double PPC_PsNmaddInline(double multiplicand, double multiplier, double addend) { - return PpcM128ToPsInline(PpcNegateNonNanLanesInline(_mm_fmadd_ps( + return PpcM128ToPsInline(PpcNegateNonNanLanesInline(PpcFmaddPairInline( PpcPsToM128Inline(multiplicand), PpcPsToM128Inline(multiplier), PpcPsToM128Inline(addend)))); } inline double PPC_PsMuls0Inline(double aValue, double cValue) { return PpcFlushPairedForNiInline(PpcM128ToPsInline( - _mm_mul_ps(PpcPsToM128Inline(aValue), PpcBroadcastPs0Inline(cValue)))); + PpcMulPairInline(PpcPsToM128Inline(aValue), PpcBroadcastPs0Inline(cValue)))); } inline double PPC_PsMuls1Inline(double aValue, double cValue) { return PpcFlushPairedForNiInline(PpcM128ToPsInline( - _mm_mul_ps(PpcPsToM128Inline(aValue), PpcBroadcastPs1Inline(cValue)))); + PpcMulPairInline(PpcPsToM128Inline(aValue), PpcBroadcastPs1Inline(cValue)))); } inline PPC_FPR PpcMakePairedResultInline(float ps0, float ps1); @@ -571,50 +660,96 @@ inline double PPC_PsToScalarInline(double value) return static_cast(PpcGetPs0Inline(value)); } -// ps_merge* are pure lane selections (result.ps0 from frA, result.ps1 from frB); with lane 0 -// == ps1 and lane 1 == ps0, two shuffles build the result bit-exact instead of round-tripping -// through the pack helper. +// ps_merge* are pure lane selections (result.ps0 from frA, result.ps1 from frB). On x86 two +// shuffles build the result bit-exact without round-tripping through the pack helper; NEON has +// no equally cheap 2-lane general shuffle, so its branch expresses the exact same selection +// (verified against the x86 comments below, lane for lane) directly in terms of the portable +// Get/Pack accessors instead. inline double PPC_PsMerge00Inline(double aValue, double bValue) { - // lane0 = b.ps0 (b lane 1), lane1 = a.ps0 (a lane 1) + // result.ps0 = a.ps0, result.ps1 = b.ps0 (lane0 = b.ps0/b lane1, lane1 = a.ps0/a lane1) +#if defined(__x86_64__) const __m128 gathered = _mm_shuffle_ps( PpcPsToM128Inline(bValue), PpcPsToM128Inline(aValue), _MM_SHUFFLE(1, 1, 1, 1)); return PpcM128ToPsInline(_mm_shuffle_ps(gathered, gathered, _MM_SHUFFLE(0, 0, 2, 0))); +#elif defined(__aarch64__) + return PpcPackPairedInline(PpcGetPs0Inline(aValue), PpcGetPs0Inline(bValue)); +#endif } inline double PPC_PsMerge01Inline(double aValue, double bValue) { - // lane0 = b.ps1 (b lane 0), lane1 = a.ps0 (a lane 1) + // result.ps0 = a.ps0, result.ps1 = b.ps1 (lane0 = b.ps1/b lane0, lane1 = a.ps0/a lane1) +#if defined(__x86_64__) const __m128 gathered = _mm_shuffle_ps( PpcPsToM128Inline(bValue), PpcPsToM128Inline(aValue), _MM_SHUFFLE(1, 1, 0, 0)); return PpcM128ToPsInline(_mm_shuffle_ps(gathered, gathered, _MM_SHUFFLE(0, 0, 2, 0))); +#elif defined(__aarch64__) + return PpcPackPairedInline(PpcGetPs0Inline(aValue), PpcGetPs1Inline(bValue)); +#endif } inline double PPC_PsMerge10Inline(double aValue, double bValue) { - // lane0 = b.ps0 (b lane 1), lane1 = a.ps1 (a lane 0) + // result.ps0 = a.ps1, result.ps1 = b.ps0 (lane0 = b.ps0/b lane1, lane1 = a.ps1/a lane0) +#if defined(__x86_64__) const __m128 gathered = _mm_shuffle_ps( PpcPsToM128Inline(bValue), PpcPsToM128Inline(aValue), _MM_SHUFFLE(0, 0, 1, 1)); return PpcM128ToPsInline(_mm_shuffle_ps(gathered, gathered, _MM_SHUFFLE(0, 0, 2, 0))); +#elif defined(__aarch64__) + return PpcPackPairedInline(PpcGetPs1Inline(aValue), PpcGetPs0Inline(bValue)); +#endif } inline double PPC_PsMerge11Inline(double aValue, double bValue) { - // lane0 = b.ps1 (b lane 0), lane1 = a.ps1 (a lane 0): plain unpcklps. + // result.ps0 = a.ps1, result.ps1 = b.ps1 (lane0 = b.ps1/b lane0, lane1 = a.ps1/a lane0): + // plain unpcklps on x86. +#if defined(__x86_64__) return PpcM128ToPsInline( _mm_unpacklo_ps(PpcPsToM128Inline(bValue), PpcPsToM128Inline(aValue))); +#elif defined(__aarch64__) + return PpcPackPairedInline(PpcGetPs1Inline(aValue), PpcGetPs1Inline(bValue)); +#endif +} + +inline PpcPairVec PpcAddPairInline(PpcPairVec lhs, PpcPairVec rhs) +{ +#if defined(__x86_64__) + return _mm_add_ps(lhs, rhs); +#elif defined(__aarch64__) + return vadd_f32(lhs, rhs); +#endif +} + +inline PpcPairVec PpcSubPairInline(PpcPairVec lhs, PpcPairVec rhs) +{ +#if defined(__x86_64__) + return _mm_sub_ps(lhs, rhs); +#elif defined(__aarch64__) + return vsub_f32(lhs, rhs); +#endif +} + +inline PpcPairVec PpcDivPairInline(PpcPairVec lhs, PpcPairVec rhs) +{ +#if defined(__x86_64__) + return _mm_div_ps(lhs, rhs); +#elif defined(__aarch64__) + return vdiv_f32(lhs, rhs); +#endif } inline double PPC_PsAddInline(double aValue, double bValue) { return PpcFlushPairedForNiInline( - PpcM128ToPsInline(_mm_add_ps(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue)))); + PpcM128ToPsInline(PpcAddPairInline(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue)))); } inline double PPC_PsAddNoNiInline(double aValue, double bValue) { return PpcM128ToPsInline( - _mm_add_ps(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue))); + PpcAddPairInline(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue))); } inline double PPC_PsSelInline(double lhsValue, double controlValue, double rhsValue) @@ -633,19 +768,19 @@ inline double PPC_PsSelInline(double lhsValue, double controlValue, double rhsVa inline double PPC_PsSubInline(double aValue, double bValue) { return PpcFlushPairedForNiInline( - PpcM128ToPsInline(_mm_sub_ps(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue)))); + PpcM128ToPsInline(PpcSubPairInline(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue)))); } inline double PPC_PsSubNoNiInline(double aValue, double bValue) { return PpcM128ToPsInline( - _mm_sub_ps(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue))); + PpcSubPairInline(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue))); } inline double PPC_PsDivInline(double aValue, double bValue) { return PpcFlushPairedForNiInline( - PpcM128ToPsInline(_mm_div_ps(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue)))); + PpcM128ToPsInline(PpcDivPairInline(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue)))); } inline double PPC_PsNegInline(double value) diff --git a/runtime/include/isa/ppc_isa_fpenv.h b/runtime/include/isa/ppc_isa_fpenv.h index 75a29bc..f3bc7b9 100644 --- a/runtime/include/isa/ppc_isa_fpenv.h +++ b/runtime/include/isa/ppc_isa_fpenv.h @@ -1,18 +1,29 @@ #pragma once // FPSCR[NI] (non-IEEE flush-to-zero) modeled on the host FP environment, plus -// the thread-local mirror of that state the hot paths read instead of MXCSR. +// the thread-local mirror of that state the hot paths read instead of MXCSR/FPCR. #include "ppc_isa_config.h" #include // Software-flushing Gekko's single-precision denormals per op roughly doubled the THP IDCT -// kernel's cycle count, so instead the runtime mirrors guest FPSCR[NI] into host MXCSR FTZ+DAZ -// wherever FPSCR can change (PPC_Mtfs*, fiber context switches, CpuContextScope), making per-op -// flushes free. Accepted deviations (same trade Dolphin makes): FTZ also flushes double -// denormals unlike real NI, and a pre-round-flush edge near FLT_MIN rounds via cvtsd2ss instead. -inline constexpr uint32_t kMkwMxcsrFlushToZeroBits = (1u << 15) | (1u << 6); // FTZ | DAZ - +// kernel's cycle count, so instead the runtime mirrors guest FPSCR[NI] into the host FP control +// register's flush-to-zero bit(s) wherever FPSCR can change (PPC_Mtfs*, fiber context switches, +// CpuContextScope), making per-op flushes free. Accepted deviations (same trade Dolphin makes): +// flush-to-zero also flushes double denormals unlike real NI, and a pre-round-flush edge near +// FLT_MIN rounds via cvtsd2ss (or its AArch64 equivalent) instead. +#if defined(__x86_64__) +// MXCSR: FTZ (bit 15, flushes arithmetic *output* denormals) and DAZ (bit 6, treats denormal +// *inputs* as zero) are two separate bits on x86; both are set/cleared together here. +inline constexpr uint32_t kMkwFpControlFlushToZeroBits = (1u << 15) | (1u << 6); // FTZ | DAZ +#elif defined(__aarch64__) +// FPCR: a single FZ bit (bit 24) flushes both single- and double-precision denormals, both as +// inputs and outputs - actually a cleaner match to the "flush everything" trade above than x86's +// two-bit combo, not an extra deviation. +inline constexpr uint32_t kMkwFpControlFlushToZeroBits = (1u << 24); // FZ +#else +#error "ppc_isa_fpenv.h has no host FP control register mapping for this architecture" +#endif inline thread_local bool g_mkwHostNiActive = false; @@ -22,32 +33,60 @@ inline thread_local bool g_mkwHostNiActive = false; inline constexpr double kMkwNiFlushThreshold = 0x1p-126; // 0x3810000000000000 inline thread_local double g_mkwNiFlushThreshold = 0.0; +// Host FP control register access (MXCSR on x86_64, FPCR on AArch64), abstracted so +// MkwApplyHostNiMode/MkwRestoreHostMxcsr and CpuContextScope (ppc_isa_context.h) don't each need +// their own per-arch branch. +inline uint32_t MkwGetHostFpControl() noexcept +{ +#if defined(__x86_64__) + return _mm_getcsr(); +#elif defined(__aarch64__) + uint64_t fpcr = 0; + __asm__ __volatile__("mrs %0, fpcr" : "=r"(fpcr)); + return static_cast(fpcr); +#endif +} + +inline void MkwSetHostFpControl(uint32_t value) noexcept +{ +#if defined(__x86_64__) + _mm_setcsr(value); +#elif defined(__aarch64__) + // Read-modify-write the full 64-bit FPCR: only the low 32 bits are architecturally defined + // today, but a bare 32-bit write would zero the (reserved) upper half of the real register. + uint64_t fpcr = 0; + __asm__ __volatile__("mrs %0, fpcr" : "=r"(fpcr)); + fpcr = (fpcr & ~static_cast(0xFFFFFFFFu)) | value; + __asm__ __volatile__("msr fpcr, %0" :: "r"(fpcr)); +#endif +} + inline void MkwApplyHostNiMode(uint32_t fpscr) noexcept { - const uint32_t csr = _mm_getcsr(); + const uint32_t csr = MkwGetHostFpControl(); const bool wantNi = (fpscr & 0x4u) != 0; const uint32_t want = wantNi - ? (csr | kMkwMxcsrFlushToZeroBits) - : (csr & ~kMkwMxcsrFlushToZeroBits); + ? (csr | kMkwFpControlFlushToZeroBits) + : (csr & ~kMkwFpControlFlushToZeroBits); if (want != csr) - _mm_setcsr(want); - // `want` has both bits set or both clear, so this is exactly - // `(_mm_getcsr() & kMkwMxcsrFlushToZeroBits) != 0` after the write - the - // mirror cannot disagree with the register even if the incoming CSR held - // only one of the two bits. + MkwSetHostFpControl(want); + // `want` has every flush-to-zero bit set or every one clear, so this is exactly + // `(MkwGetHostFpControl() & kMkwFpControlFlushToZeroBits) != 0` after the write - the + // mirror cannot disagree with the register even if the incoming value held only some of + // those bits. g_mkwHostNiActive = wantNi; g_mkwNiFlushThreshold = wantNi ? kMkwNiFlushThreshold : 0.0; } /// -/// Restores a previously captured MXCSR value and re-derives the mirror from -/// it. Every raw restore has to go through here; a bare _mm_setcsr would leave -/// the mirror describing the FP environment that was just replaced. +/// Restores a previously captured MXCSR/FPCR value and re-derives the mirror from it. Every raw +/// restore has to go through here; a bare MkwSetHostFpControl would leave the mirror describing +/// the FP environment that was just replaced. /// inline void MkwRestoreHostMxcsr(uint32_t csr) noexcept { - _mm_setcsr(csr); - const bool niActive = (csr & kMkwMxcsrFlushToZeroBits) != 0; + MkwSetHostFpControl(csr); + const bool niActive = (csr & kMkwFpControlFlushToZeroBits) != 0; g_mkwHostNiActive = niActive; g_mkwNiFlushThreshold = niActive ? kMkwNiFlushThreshold : 0.0; } diff --git a/runtime/include/isa/ppc_isa_quantized.h b/runtime/include/isa/ppc_isa_quantized.h index 949197a..9f5c807 100644 --- a/runtime/include/isa/ppc_isa_quantized.h +++ b/runtime/include/isa/ppc_isa_quantized.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -38,16 +39,26 @@ inline uint32_t PpcStorePsqFloatBitsInline(uint32_t value) inline uint64_t PpcLoadPairPsqFloatBitsPackedInline(uint64_t value) { +#if defined(__x86_64__) const __m128i lanes = _mm_cvtsi64_si128(static_cast(value)); const __m128i magnitude = _mm_and_si128(lanes, _mm_set1_epi32(0x7FFFFFFF)); const __m128i nanMask = _mm_cmpgt_epi32(magnitude, _mm_set1_epi32(0x7F800000)); const __m128i result = _mm_or_si128( lanes, _mm_and_si128(nanMask, _mm_set1_epi32(0x00400000))); return static_cast(_mm_cvtsi128_si64(result)); +#elif defined(__aarch64__) + // Equivalent to applying PpcLoadPsqFloatBitsInline to each 32-bit lane: the + // x86 body above only ever acts on these same two lanes (the upper 64 bits + // _mm_cvtsi64_si128 zero-fills never survive the final truncating extract). + const uint32_t lo = PpcLoadPsqFloatBitsInline(static_cast(value)); + const uint32_t hi = PpcLoadPsqFloatBitsInline(static_cast(value >> 32)); + return (static_cast(hi) << 32) | lo; +#endif } inline uint64_t PpcStorePairPsqFloatBitsPackedInline(uint64_t value) { +#if defined(__x86_64__) const __m128i lanes = _mm_cvtsi64_si128(static_cast(value)); const __m128i magnitude = _mm_and_si128(lanes, _mm_set1_epi32(0x7FFFFFFF)); const __m128i subnormalMask = _mm_cmplt_epi32(magnitude, _mm_set1_epi32(0x00800000)); @@ -60,8 +71,16 @@ inline uint64_t PpcStorePairPsqFloatBitsPackedInline(uint64_t value) _mm_and_si128(subnormalMask, signedZero), _mm_andnot_si128(subnormalMask, quieted)); return static_cast(_mm_cvtsi128_si64(result)); +#elif defined(__aarch64__) + // Equivalent to applying PpcStorePsqFloatBitsInline to each 32-bit lane; + // same reasoning as the load-side port above. + const uint32_t lo = PpcStorePsqFloatBitsInline(static_cast(value)); + const uint32_t hi = PpcStorePsqFloatBitsInline(static_cast(value >> 32)); + return (static_cast(hi) << 32) | lo; +#endif } +#if defined(__x86_64__) inline __m128i PpcPsqSwapPairBytesInline(__m128i lanes) { const __m128i order = _mm_setr_epi8( @@ -95,21 +114,42 @@ inline __m128i PpcStorePairPsqFloatBitsLanesInline(__m128i lanes) _mm_and_si128(subnormalMask, signedZero), _mm_andnot_si128(subnormalMask, quieted)); } +#endif // defined(__x86_64__) // host -> packed FPR double, guard already proven by the caller. inline double PpcLoadPairPsqFloatFromHostInline(const uint8_t* host) { +#if defined(__x86_64__) const __m128i raw = _mm_loadl_epi64(reinterpret_cast(host)); return PpcM128ToPsInline(_mm_castsi128_ps( PpcLoadPairPsqFloatBitsLanesInline(PpcPsqSwapPairBytesInline(raw)))); +#elif defined(__aarch64__) + // The x86 path's full 8-byte pshufb reversal plus a same-endian load is, + // taken together, exactly a 64-bit byteswap of a plain little-endian load: + // it turns the on-disk [ps0 big-endian][ps1 big-endian] byte layout into a + // native uint64 with low 32 bits = ps1, high 32 bits = ps0 (this file's + // documented packed-double lane convention). + uint64_t raw = 0; + std::memcpy(&raw, host, sizeof(raw)); + const uint64_t swapped = __builtin_bswap64(raw); + return PpcBitCastToDoubleInline(PpcLoadPairPsqFloatBitsPackedInline(swapped)); +#endif } // packed FPR double -> host, guard already proven by the caller. inline void PpcStorePairPsqFloatToHostInline(uint8_t* host, double value) { +#if defined(__x86_64__) const __m128i lanes = PpcStorePairPsqFloatBitsLanesInline( _mm_castps_si128(PpcPsToM128Inline(value))); _mm_storel_epi64(reinterpret_cast<__m128i*>(host), PpcPsqSwapPairBytesInline(lanes)); +#elif defined(__aarch64__) + // Inverse of the load path above: bswap64 is its own inverse, so applying + // it to the quieted packed value reproduces the on-disk big-endian bytes. + const uint64_t quieted = PpcStorePairPsqFloatBitsPackedInline(PpcBitCastToU64Inline(value)); + const uint64_t swapped = __builtin_bswap64(quieted); + std::memcpy(host, &swapped, sizeof(swapped)); +#endif } template @@ -329,6 +369,7 @@ inline uint8_t PpcQuantizePsqU8Scale61Inline(float value) // matching PpcQuantizePsqU8Scale61Inline's !(scaled > 0) rule. inline uint16_t PpcQuantizePairPsqU8Scale61PackedInline(double value) { +#if defined(__x86_64__) const __m128 scaled = _mm_mul_ps(PpcPsToM128Inline(value), _mm_set1_ps(0.125f)); const __m128 nonNegative = _mm_max_ps(scaled, _mm_setzero_ps()); const __m128 clamped = _mm_min_ps(nonNegative, _mm_set1_ps(255.0f)); @@ -338,6 +379,17 @@ inline uint16_t PpcQuantizePairPsqU8Scale61PackedInline(double value) // Native lane 0 is ps1 and lane 1 is ps0. Packing to the low uint16_t // therefore produces the guest-order numeric value (ps0 << 8) | ps1. return static_cast(_mm_cvtsi128_si32(lanes8)); +#elif defined(__aarch64__) + // Equivalent to two calls of the already-portable scalar quantizer above + // (its own !(scaled > 0) rule maps NaN to 0, matching what MAXPS-with-zero + // does on the x86 path per the comment there), packed the same way the + // fallback path just below already does for the non-SIMD case. + PPC_FPR fpr{}; + fpr.d = value; + const uint8_t q0 = PpcQuantizePsqU8Scale61Inline(fpr.paired.ps0); + const uint8_t q1 = PpcQuantizePsqU8Scale61Inline(fpr.paired.ps1); + return static_cast((static_cast(q0) << 8) | static_cast(q1)); +#endif } // Preserve the complete memory/MMIO/executable-write behavior off the leaf diff --git a/runtime/src/guest_flat_memory.cpp b/runtime/src/guest_flat_memory.cpp index ae9f3f7..17780d9 100644 --- a/runtime/src/guest_flat_memory.cpp +++ b/runtime/src/guest_flat_memory.cpp @@ -273,8 +273,12 @@ void EnsureReservation() { std::ostringstream oss; oss << "Unable to reserve the 4 GiB flat guest address space at 0x" << std::hex << reinterpret_cast(requested) << std::dec - << ". Something else in this process already occupies part of the 16 TiB region - " - "an injected library, an overlay or a debugging tool is the usual cause."; + << ". Either something else in this process already occupies that address (an " + "injected library, an overlay or a debugging tool is the usual cause), or this " + "kernel's virtual address space does not reach that high (common on some 32-bit-" + "userspace-compatible or older AArch64 configurations, e.g. a kernel built for " + "39-bit virtual addresses) - in the latter case mmap() silently substitutes an " + "address near the top of the space it does have instead of honoring the request."; throw std::runtime_error(oss.str()); } #endif diff --git a/runtime/src/host_cpu_baseline.cpp b/runtime/src/host_cpu_baseline.cpp index 9d8157e..9a1690c 100644 --- a/runtime/src/host_cpu_baseline.cpp +++ b/runtime/src/host_cpu_baseline.cpp @@ -4,6 +4,13 @@ // build/PCH) and runs from a priority-101 C initializer, ahead of every C++ dynamic initializer // and thus the first AVX2 code that could execute. Keep it free of anything that could pull in // vectorized code: no iostreams, no std::string, no runtime-wide headers. +// +// x86-64-v3 is an x86-specific optional-feature baseline (AVX2/BMI2/FMA and friends are not +// guaranteed present on every x86_64 chip); nothing here applies on AArch64, where ASIMD/NEON is +// mandatory in the base architecture and PublicProducts.cmake never applies an -march=x86-64-v3 +// equivalent flag to begin with. That branch below is a no-op stub, not a port of this check. + +#if defined(__x86_64__) #include #include @@ -214,3 +221,11 @@ extern "C" int MkwHostCpuBaselineInit() { __attribute__((constructor(101))) static void MkwHostCpuBaselineCtor() { MkwHostCpuBaselineInit(); } + +#else // !defined(__x86_64__) + +extern "C" int MkwHostCpuBaselineInit() { + return 0; +} + +#endif // defined(__x86_64__)