add native aarch64 (arm64) support

use -mcpu=native on arm64 targets

build-appimage.sh kernel architecture detection

runtime/include/guest_flat_memory.h

@@ -12,10 +12,26 @@
 namespace GuestFlat {
 
 // Fixed base so the emitted access is `[reg + imm64-in-register]` with no load
-// of a global. 16 TiB: clear of the Windows ASan shadow (32 TiB) and of the
-// usual image/heap placement.
+// of a global.
 inline constexpr uint64_t kGuestSpaceSize = 0x1'0000'0000ull;
+#if defined(__x86_64__)
+// 16 TiB: clear of the Windows ASan shadow (32 TiB) and of the usual image/heap
+// placement.
 inline constexpr uintptr_t kFixedFlatGuestBase = 0x0000'1000'0000'0000ull;
+#elif defined(__aarch64__)
+// 16 TiB (this arch's x86_64 sibling value) is unreachable on any AArch64
+// kernel configured for 39-bit virtual addresses (512 GiB ceiling) - common on
+// older/embedded targets (e.g. this project's own tested Jetson/L4T board, kernel
+// 4.9). There mmap() silently ignores a hint above the ceiling and hands back an
+// address near the top of the real range instead, which this module's caller
+// then rejects as "already occupied". 64 GiB is reachable on every AArch64 VA
+// width in real use (39-bit minimum and up) and was confirmed via direct mmap
+// probing to sit far below where the PIE image, heap, shared libraries and
+// stack actually land (all clustered above ~340 GiB on a 39-bit/512 GiB system).
+inline constexpr uintptr_t kFixedFlatGuestBase = 0x0000'0010'0000'0000ull;
+#else
+#error "guest_flat_memory.h has no fixed flat guest base chosen for this architecture"
+#endif
 
 #define MKW_FLAT_GUEST_BASE (reinterpret_cast<uint8_t*>(GuestFlat::kFixedFlatGuestBase))
 

runtime/include/isa/ppc_isa_config.h

@@ -4,7 +4,13 @@
 #include <cstdint>
 
 #define MKW_RESTRICT __restrict
+#if defined(__x86_64__)
 #include <immintrin.h>
+#elif defined(__aarch64__)
+#include <arm_neon.h>
+#else
+#error "ppc_isa_config.h has no SIMD intrinsics header for this architecture"
+#endif
 
 inline constexpr bool MkwStateFreeAbiEnabled(uint32_t) noexcept
 {

runtime/include/isa/ppc_isa_context.h

@@ -62,7 +62,7 @@ public:
     {
         g_currentCpuContext = ctx;
 
-        savedMxcsr_ = _mm_getcsr();
+        savedMxcsr_ = MkwGetHostFpControl();
         if (ctx != nullptr)
             MkwApplyHostNiMode(ctx->fpscr);
     }

runtime/include/isa/ppc_isa_float.h

@@ -78,37 +78,71 @@ inline void PpcSetPairedFprInline(PPC_FPR& fpr, double packed)
     fpr.d = packed;
 }
 
-// Must stay inside the XMM register domain. Bitcasting through a 64-bit GPR added a movq
+#if defined(__x86_64__)
+using PpcPairVec = __m128;
+#elif defined(__aarch64__)
+// Only 2 lanes are ever meaningful (a PPC paired-single register), so a 2-lane float32x2_t
+// (one 64-bit D register) is a more natural fit than mirroring x86's 128-bit register usage.
+using PpcPairVec = float32x2_t;
+#endif
+
+// Must stay inside the vector register domain. Bitcasting through a 64-bit GPR added a movq
 // domain crossing on every paired-single op (630 in the THP IDCT region alone); a double
-// local already lives in an XMM register, so these casts compile to nothing.
-inline __m128 PpcPsToM128Inline(double value)
+// local already lives in a vector register, so these casts compile to nothing.
+inline PpcPairVec PpcPsToM128Inline(double value)
 {
+#if defined(__x86_64__)
     return _mm_castpd_ps(_mm_set_sd(value));
+#elif defined(__aarch64__)
+    return vreinterpret_f32_f64(vdup_n_f64(value));
+#endif
 }
 
-inline double PpcM128ToPsInline(__m128 value)
+inline double PpcM128ToPsInline(PpcPairVec value)
 {
+#if defined(__x86_64__)
     return _mm_cvtsd_f64(_mm_castps_pd(value));
+#elif defined(__aarch64__)
+    return vget_lane_f64(vreinterpret_f64_f32(value), 0);
+#endif
 }
 
-inline __m128 PpcBroadcastPs0Inline(double value)
+inline PpcPairVec PpcBroadcastPs0Inline(double value)
 {
+#if defined(__x86_64__)
     const __m128 lanes = PpcPsToM128Inline(value);
     return _mm_shuffle_ps(lanes, lanes, _MM_SHUFFLE(1, 1, 1, 1));
+#elif defined(__aarch64__)
+    // ps0 lives in lane 1 (see the lane-accessor comment below).
+    return vdup_lane_f32(PpcPsToM128Inline(value), 1);
+#endif
 }
 
-inline __m128 PpcBroadcastPs1Inline(double value)
+inline PpcPairVec PpcBroadcastPs1Inline(double value)
 {
+#if defined(__x86_64__)
     const __m128 lanes = PpcPsToM128Inline(value);
     return _mm_shuffle_ps(lanes, lanes, _MM_SHUFFLE(0, 0, 0, 0));
+#elif defined(__aarch64__)
+    // ps1 is already lane 0.
+    return vdup_lane_f32(PpcPsToM128Inline(value), 0);
+#endif
 }
 
-inline __m128 PpcNegateNonNanLanesInline(__m128 value)
+inline PpcPairVec PpcNegateNonNanLanesInline(PpcPairVec value)
 {
+#if defined(__x86_64__)
     const __m128 signMask = _mm_castsi128_ps(_mm_set1_epi32(static_cast<int>(0x80000000u)));
     const __m128 negated = _mm_xor_ps(value, signMask);
     const __m128 ordered = _mm_cmpord_ps(value, value);
     return _mm_or_ps(_mm_and_ps(ordered, negated), _mm_andnot_ps(ordered, value));
+#elif defined(__aarch64__)
+    const uint32x2_t signMask = vdup_n_u32(0x80000000u);
+    const float32x2_t negated = vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(value), signMask));
+    // NEON has no direct "ordered compare"; a value compares equal to itself iff it's not NaN.
+    const uint32x2_t ordered = vceq_f32(value, value);
+    return vbsl_f32(ordered, negated, value);
+#endif
 }
 
 // Paired-single lane accessors. The packed double's LOW 32 bits hold ps1 and HIGH 32 bits
@@ -119,20 +153,34 @@ inline __m128 PpcNegateNonNanLanesInline(__m128 value)
 inline float PpcGetPs0Inline(double value)
 {
     // ps0 lives in lane 1; PpcBroadcastPs0Inline already splats it.
+#if defined(__x86_64__)
     return _mm_cvtss_f32(PpcBroadcastPs0Inline(value));
+#elif defined(__aarch64__)
+    return vget_lane_f32(PpcBroadcastPs0Inline(value), 0);
+#endif
 }
 
 inline float PpcGetPs1Inline(double value)
 {
     // ps1 is already lane 0 of the packed representation.
+#if defined(__x86_64__)
     return _mm_cvtss_f32(PpcPsToM128Inline(value));
+#elif defined(__aarch64__)
+    return vget_lane_f32(PpcPsToM128Inline(value), 0);
+#endif
 }
 
 inline double PpcPackPairedInline(float ps0, float ps1)
 {
+#if defined(__x86_64__)
     // _mm_unpacklo_ps(x, y) -> { x[0], y[0], x[1], y[1] }, so lane 0 becomes
     // ps1 and lane 1 becomes ps0, matching the union layout bit for bit.
     return PpcM128ToPsInline(_mm_unpacklo_ps(_mm_set_ss(ps1), _mm_set_ss(ps0)));
+#elif defined(__aarch64__)
+    // Lane 0 = ps1, lane 1 = ps0, matching the union layout bit for bit.
+    const float32x2_t lane0 = vdup_n_f32(ps1);
+    return PpcM128ToPsInline(vset_lane_f32(ps0, lane0, 1));
+#endif
 }
 
 // FPSCR[NI] is modeled by MXCSR FTZ/DAZ, so arithmetic output flushing compiles to nothing.
@@ -148,14 +196,24 @@ inline float PpcForceSingleValueInline(double value)
     // FPSCR[NI] flushes an exact pre-round single-subnormal even when rounding would promote it
     // to the smallest normal. g_mkwNiFlushThreshold (2^-126 active, 0.0 inactive) turns the
     // flush into a branchless mask: a set compare lane keeps just the sign bit, a clear lane
-    // passes the value to CVTSD2SS. DAZ (set exactly when NI is) makes the compare itself treat
-    // a subnormal as zero, matching the mask's answer.
+    // passes the value to the double->float conversion. DAZ/FZ (set exactly when NI is) makes
+    // the compare itself treat a subnormal as zero, matching the mask's answer.
+#if defined(__x86_64__)
     const __m128d v = _mm_set_sd(value);
     const __m128d signMask = _mm_set_sd(-0.0);
     const __m128d magnitude = _mm_andnot_pd(signMask, v);
     const __m128d flush = _mm_cmplt_sd(magnitude, _mm_set_sd(g_mkwNiFlushThreshold));
     const __m128d kept = _mm_andnot_pd(_mm_andnot_pd(signMask, flush), v);
     return static_cast<float>(_mm_cvtsd_f64(kept));
+#elif defined(__aarch64__)
+    const float64x1_t v = vdup_n_f64(value);
+    const uint64x1_t signMask = vdup_n_u64(0x8000000000000000ULL);
+    const float64x1_t magnitude = vreinterpret_f64_u64(vbic_u64(vreinterpret_u64_f64(v), signMask));
+    const uint64x1_t flush = vclt_f64(magnitude, vdup_n_f64(g_mkwNiFlushThreshold));
+    const uint64x1_t signOnly = vand_u64(vreinterpret_u64_f64(v), signMask);
+    const uint64x1_t kept = vbsl_u64(flush, signOnly, vreinterpret_u64_f64(v));
+    return static_cast<float>(vget_lane_f64(vreinterpret_f64_u64(kept), 0));
+#endif
 }
 
 inline float PpcFlushSingleForNiInline(float value)
@@ -468,15 +526,24 @@ inline double PpcFnmsubsInline(double a, double c, double b)
     return static_cast<double>(std::isnan(result) ? result : -result);
 }
 
+inline PpcPairVec PpcMulPairInline(PpcPairVec lhs, PpcPairVec rhs)
+{
+#if defined(__x86_64__)
+    return _mm_mul_ps(lhs, rhs);
+#elif defined(__aarch64__)
+    return vmul_f32(lhs, rhs);
+#endif
+}
+
 inline double PPC_PsMulInline(double lhs, double rhs)
 {
     return PpcFlushPairedForNiInline(
-        PpcM128ToPsInline(_mm_mul_ps(PpcPsToM128Inline(lhs), PpcPsToM128Inline(rhs))));
+        PpcM128ToPsInline(PpcMulPairInline(PpcPsToM128Inline(lhs), PpcPsToM128Inline(rhs))));
 }
 
 inline double PPC_PsMulNoNiInline(double lhs, double rhs)
 {
-    return PpcM128ToPsInline(_mm_mul_ps(PpcPsToM128Inline(lhs), PpcPsToM128Inline(rhs)));
+    return PpcM128ToPsInline(PpcMulPairInline(PpcPsToM128Inline(lhs), PpcPsToM128Inline(rhs)));
 }
 
 // The paired madd family lowers to one hardware FMA. Semantics match the scalar lanes exactly: a
@@ -485,9 +552,31 @@ inline double PPC_PsMulNoNiInline(double lhs, double rhs)
 // PpcNegateNonNanLanesInline. NI flushing is handled by MXCSR (see
 // MkwApplyHostNiMode), so the NI and NoNi entry points are identical here.
 
+// Fused multiply-add/subtract on a pair. NEON's vfma_f32(acc, a, b) = acc + a*b has an
+// accumulator-first operand order, unlike x86's _mm_fmadd_ps(a, b, c) = a*b + c - msub is
+// therefore expressed as an fma against a negated accumulator on both architectures, not a
+// dedicated fms intrinsic, so the two branches stay structurally parallel.
+inline PpcPairVec PpcFmaddPairInline(PpcPairVec multiplicand, PpcPairVec multiplier, PpcPairVec addend)
+{
+#if defined(__x86_64__)
+    return _mm_fmadd_ps(multiplicand, multiplier, addend);
+#elif defined(__aarch64__)
+    return vfma_f32(addend, multiplicand, multiplier);
+#endif
+}
+
+inline PpcPairVec PpcFmsubPairInline(PpcPairVec multiplicand, PpcPairVec multiplier, PpcPairVec subtractor)
+{
+#if defined(__x86_64__)
+    return _mm_fmsub_ps(multiplicand, multiplier, subtractor);
+#elif defined(__aarch64__)
+    return vfma_f32(vneg_f32(subtractor), multiplicand, multiplier);
+#endif
+}
+
 inline double PPC_PsMsubInline(double multiplicand, double multiplier, double subtractor)
 {
-    return PpcM128ToPsInline(_mm_fmsub_ps(
+    return PpcM128ToPsInline(PpcFmsubPairInline(
         PpcPsToM128Inline(multiplicand), PpcPsToM128Inline(multiplier), PpcPsToM128Inline(subtractor)));
 }
 
@@ -498,7 +587,7 @@ inline double PPC_PsMsubNoNiInline(double multiplicand, double multiplier, doubl
 
 inline double PPC_PsMaddInline(double multiplicand, double multiplier, double addend)
 {
-    return PpcM128ToPsInline(_mm_fmadd_ps(
+    return PpcM128ToPsInline(PpcFmaddPairInline(
         PpcPsToM128Inline(multiplicand), PpcPsToM128Inline(multiplier), PpcPsToM128Inline(addend)));
 }
 
@@ -509,19 +598,19 @@ inline double PPC_PsMaddNoNiInline(double multiplicand, double multiplier, doubl
 
 inline double PPC_PsMadds0Inline(double multiplicand, double multiplier, double addend)
 {
-    return PpcM128ToPsInline(_mm_fmadd_ps(
+    return PpcM128ToPsInline(PpcFmaddPairInline(
         PpcPsToM128Inline(multiplicand), PpcBroadcastPs0Inline(multiplier), PpcPsToM128Inline(addend)));
 }
 
 inline double PPC_PsMadds1Inline(double multiplicand, double multiplier, double addend)
 {
-    return PpcM128ToPsInline(_mm_fmadd_ps(
+    return PpcM128ToPsInline(PpcFmaddPairInline(
         PpcPsToM128Inline(multiplicand), PpcBroadcastPs1Inline(multiplier), PpcPsToM128Inline(addend)));
 }
 
 inline double PPC_PsNmsubInline(double multiplicand, double multiplier, double subtractor)
 {
-    return PpcM128ToPsInline(PpcNegateNonNanLanesInline(_mm_fmsub_ps(
+    return PpcM128ToPsInline(PpcNegateNonNanLanesInline(PpcFmsubPairInline(
         PpcPsToM128Inline(multiplicand), PpcPsToM128Inline(multiplier), PpcPsToM128Inline(subtractor))));
 }
 
@@ -532,20 +621,20 @@ inline double PPC_PsNmsubNoNiInline(double multiplicand, double multiplier, doub
 
 inline double PPC_PsNmaddInline(double multiplicand, double multiplier, double addend)
 {
-    return PpcM128ToPsInline(PpcNegateNonNanLanesInline(_mm_fmadd_ps(
+    return PpcM128ToPsInline(PpcNegateNonNanLanesInline(PpcFmaddPairInline(
         PpcPsToM128Inline(multiplicand), PpcPsToM128Inline(multiplier), PpcPsToM128Inline(addend))));
 }
 
 inline double PPC_PsMuls0Inline(double aValue, double cValue)
 {
     return PpcFlushPairedForNiInline(PpcM128ToPsInline(
-        _mm_mul_ps(PpcPsToM128Inline(aValue), PpcBroadcastPs0Inline(cValue))));
+        PpcMulPairInline(PpcPsToM128Inline(aValue), PpcBroadcastPs0Inline(cValue))));
 }
 
 inline double PPC_PsMuls1Inline(double aValue, double cValue)
 {
     return PpcFlushPairedForNiInline(PpcM128ToPsInline(
-        _mm_mul_ps(PpcPsToM128Inline(aValue), PpcBroadcastPs1Inline(cValue))));
+        PpcMulPairInline(PpcPsToM128Inline(aValue), PpcBroadcastPs1Inline(cValue))));
 }
 
 inline PPC_FPR PpcMakePairedResultInline(float ps0, float ps1);
@@ -571,50 +660,96 @@ inline double PPC_PsToScalarInline(double value)
     return static_cast<double>(PpcGetPs0Inline(value));
 }
 
-// ps_merge* are pure lane selections (result.ps0 from frA, result.ps1 from frB); with lane 0
-// == ps1 and lane 1 == ps0, two shuffles build the result bit-exact instead of round-tripping
-// through the pack helper.
+// ps_merge* are pure lane selections (result.ps0 from frA, result.ps1 from frB). On x86 two
+// shuffles build the result bit-exact without round-tripping through the pack helper; NEON has
+// no equally cheap 2-lane general shuffle, so its branch expresses the exact same selection
+// (verified against the x86 comments below, lane for lane) directly in terms of the portable
+// Get/Pack accessors instead.
 inline double PPC_PsMerge00Inline(double aValue, double bValue)
 {
-    // lane0 = b.ps0 (b lane 1), lane1 = a.ps0 (a lane 1)
+    // result.ps0 = a.ps0, result.ps1 = b.ps0 (lane0 = b.ps0/b lane1, lane1 = a.ps0/a lane1)
+#if defined(__x86_64__)
     const __m128 gathered = _mm_shuffle_ps(
         PpcPsToM128Inline(bValue), PpcPsToM128Inline(aValue), _MM_SHUFFLE(1, 1, 1, 1));
     return PpcM128ToPsInline(_mm_shuffle_ps(gathered, gathered, _MM_SHUFFLE(0, 0, 2, 0)));
+#elif defined(__aarch64__)
+    return PpcPackPairedInline(PpcGetPs0Inline(aValue), PpcGetPs0Inline(bValue));
+#endif
 }
 
 inline double PPC_PsMerge01Inline(double aValue, double bValue)
 {
-    // lane0 = b.ps1 (b lane 0), lane1 = a.ps0 (a lane 1)
+    // result.ps0 = a.ps0, result.ps1 = b.ps1 (lane0 = b.ps1/b lane0, lane1 = a.ps0/a lane1)
+#if defined(__x86_64__)
     const __m128 gathered = _mm_shuffle_ps(
         PpcPsToM128Inline(bValue), PpcPsToM128Inline(aValue), _MM_SHUFFLE(1, 1, 0, 0));
     return PpcM128ToPsInline(_mm_shuffle_ps(gathered, gathered, _MM_SHUFFLE(0, 0, 2, 0)));
+#elif defined(__aarch64__)
+    return PpcPackPairedInline(PpcGetPs0Inline(aValue), PpcGetPs1Inline(bValue));
+#endif
 }
 
 inline double PPC_PsMerge10Inline(double aValue, double bValue)
 {
-    // lane0 = b.ps0 (b lane 1), lane1 = a.ps1 (a lane 0)
+    // result.ps0 = a.ps1, result.ps1 = b.ps0 (lane0 = b.ps0/b lane1, lane1 = a.ps1/a lane0)
+#if defined(__x86_64__)
     const __m128 gathered = _mm_shuffle_ps(
         PpcPsToM128Inline(bValue), PpcPsToM128Inline(aValue), _MM_SHUFFLE(0, 0, 1, 1));
     return PpcM128ToPsInline(_mm_shuffle_ps(gathered, gathered, _MM_SHUFFLE(0, 0, 2, 0)));
+#elif defined(__aarch64__)
+    return PpcPackPairedInline(PpcGetPs1Inline(aValue), PpcGetPs0Inline(bValue));
+#endif
 }
 
 inline double PPC_PsMerge11Inline(double aValue, double bValue)
 {
-    // lane0 = b.ps1 (b lane 0), lane1 = a.ps1 (a lane 0): plain unpcklps.
+    // result.ps0 = a.ps1, result.ps1 = b.ps1 (lane0 = b.ps1/b lane0, lane1 = a.ps1/a lane0):
+    // plain unpcklps on x86.
+#if defined(__x86_64__)
     return PpcM128ToPsInline(
         _mm_unpacklo_ps(PpcPsToM128Inline(bValue), PpcPsToM128Inline(aValue)));
+#elif defined(__aarch64__)
+    return PpcPackPairedInline(PpcGetPs1Inline(aValue), PpcGetPs1Inline(bValue));
+#endif
+}
+
+inline PpcPairVec PpcAddPairInline(PpcPairVec lhs, PpcPairVec rhs)
+{
+#if defined(__x86_64__)
+    return _mm_add_ps(lhs, rhs);
+#elif defined(__aarch64__)
+    return vadd_f32(lhs, rhs);
+#endif
+}
+
+inline PpcPairVec PpcSubPairInline(PpcPairVec lhs, PpcPairVec rhs)
+{
+#if defined(__x86_64__)
+    return _mm_sub_ps(lhs, rhs);
+#elif defined(__aarch64__)
+    return vsub_f32(lhs, rhs);
+#endif
+}
+
+inline PpcPairVec PpcDivPairInline(PpcPairVec lhs, PpcPairVec rhs)
+{
+#if defined(__x86_64__)
+    return _mm_div_ps(lhs, rhs);
+#elif defined(__aarch64__)
+    return vdiv_f32(lhs, rhs);
+#endif
 }
 
 inline double PPC_PsAddInline(double aValue, double bValue)
 {
     return PpcFlushPairedForNiInline(
-        PpcM128ToPsInline(_mm_add_ps(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue))));
+        PpcM128ToPsInline(PpcAddPairInline(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue))));
 }
 
 inline double PPC_PsAddNoNiInline(double aValue, double bValue)
 {
     return PpcM128ToPsInline(
-        _mm_add_ps(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue)));
+        PpcAddPairInline(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue)));
 }
 
 inline double PPC_PsSelInline(double lhsValue, double controlValue, double rhsValue)
@@ -633,19 +768,19 @@ inline double PPC_PsSelInline(double lhsValue, double controlValue, double rhsVa
 inline double PPC_PsSubInline(double aValue, double bValue)
 {
     return PpcFlushPairedForNiInline(
-        PpcM128ToPsInline(_mm_sub_ps(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue))));
+        PpcM128ToPsInline(PpcSubPairInline(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue))));
 }
 
 inline double PPC_PsSubNoNiInline(double aValue, double bValue)
 {
     return PpcM128ToPsInline(
-        _mm_sub_ps(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue)));
+        PpcSubPairInline(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue)));
 }
 
 inline double PPC_PsDivInline(double aValue, double bValue)
 {
     return PpcFlushPairedForNiInline(
-        PpcM128ToPsInline(_mm_div_ps(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue))));
+        PpcM128ToPsInline(PpcDivPairInline(PpcPsToM128Inline(aValue), PpcPsToM128Inline(bValue))));
 }
 
 inline double PPC_PsNegInline(double value)

runtime/include/isa/ppc_isa_fpenv.h

@@ -1,18 +1,29 @@
 #pragma once
 // FPSCR[NI] (non-IEEE flush-to-zero) modeled on the host FP environment, plus
-// the thread-local mirror of that state the hot paths read instead of MXCSR.
+// the thread-local mirror of that state the hot paths read instead of MXCSR/FPCR.
 
 #include "ppc_isa_config.h"
 
 #include <cstdint>
 
 // Software-flushing Gekko's single-precision denormals per op roughly doubled the THP IDCT
-// kernel's cycle count, so instead the runtime mirrors guest FPSCR[NI] into host MXCSR FTZ+DAZ
-// wherever FPSCR can change (PPC_Mtfs*, fiber context switches, CpuContextScope), making per-op
-// flushes free. Accepted deviations (same trade Dolphin makes): FTZ also flushes double
-// denormals unlike real NI, and a pre-round-flush edge near FLT_MIN rounds via cvtsd2ss instead.
-inline constexpr uint32_t kMkwMxcsrFlushToZeroBits = (1u << 15) | (1u << 6); // FTZ | DAZ
-
+// kernel's cycle count, so instead the runtime mirrors guest FPSCR[NI] into the host FP control
+// register's flush-to-zero bit(s) wherever FPSCR can change (PPC_Mtfs*, fiber context switches,
+// CpuContextScope), making per-op flushes free. Accepted deviations (same trade Dolphin makes):
+// flush-to-zero also flushes double denormals unlike real NI, and a pre-round-flush edge near
+// FLT_MIN rounds via cvtsd2ss (or its AArch64 equivalent) instead.
+#if defined(__x86_64__)
+// MXCSR: FTZ (bit 15, flushes arithmetic *output* denormals) and DAZ (bit 6, treats denormal
+// *inputs* as zero) are two separate bits on x86; both are set/cleared together here.
+inline constexpr uint32_t kMkwFpControlFlushToZeroBits = (1u << 15) | (1u << 6); // FTZ | DAZ
+#elif defined(__aarch64__)
+// FPCR: a single FZ bit (bit 24) flushes both single- and double-precision denormals, both as
+// inputs and outputs - actually a cleaner match to the "flush everything" trade above than x86's
+// two-bit combo, not an extra deviation.
+inline constexpr uint32_t kMkwFpControlFlushToZeroBits = (1u << 24); // FZ
+#else
+#error "ppc_isa_fpenv.h has no host FP control register mapping for this architecture"
+#endif
 
 inline thread_local bool g_mkwHostNiActive = false;
 
@@ -22,32 +33,60 @@ inline thread_local bool g_mkwHostNiActive = false;
 inline constexpr double kMkwNiFlushThreshold = 0x1p-126;  // 0x3810000000000000
 inline thread_local double g_mkwNiFlushThreshold = 0.0;
 
+// Host FP control register access (MXCSR on x86_64, FPCR on AArch64), abstracted so
+// MkwApplyHostNiMode/MkwRestoreHostMxcsr and CpuContextScope (ppc_isa_context.h) don't each need
+// their own per-arch branch.
+inline uint32_t MkwGetHostFpControl() noexcept
+{
+#if defined(__x86_64__)
+    return _mm_getcsr();
+#elif defined(__aarch64__)
+    uint64_t fpcr = 0;
+    __asm__ __volatile__("mrs %0, fpcr" : "=r"(fpcr));
+    return static_cast<uint32_t>(fpcr);
+#endif
+}
+
+inline void MkwSetHostFpControl(uint32_t value) noexcept
+{
+#if defined(__x86_64__)
+    _mm_setcsr(value);
+#elif defined(__aarch64__)
+    // Read-modify-write the full 64-bit FPCR: only the low 32 bits are architecturally defined
+    // today, but a bare 32-bit write would zero the (reserved) upper half of the real register.
+    uint64_t fpcr = 0;
+    __asm__ __volatile__("mrs %0, fpcr" : "=r"(fpcr));
+    fpcr = (fpcr & ~static_cast<uint64_t>(0xFFFFFFFFu)) | value;
+    __asm__ __volatile__("msr fpcr, %0" :: "r"(fpcr));
+#endif
+}
+
 inline void MkwApplyHostNiMode(uint32_t fpscr) noexcept
 {
-    const uint32_t csr = _mm_getcsr();
+    const uint32_t csr = MkwGetHostFpControl();
     const bool wantNi = (fpscr & 0x4u) != 0;
     const uint32_t want = wantNi
-        ? (csr | kMkwMxcsrFlushToZeroBits)
-        : (csr & ~kMkwMxcsrFlushToZeroBits);
+        ? (csr | kMkwFpControlFlushToZeroBits)
+        : (csr & ~kMkwFpControlFlushToZeroBits);
     if (want != csr)
-        _mm_setcsr(want);
-    // `want` has both bits set or both clear, so this is exactly
-    // `(_mm_getcsr() & kMkwMxcsrFlushToZeroBits) != 0` after the write - the
-    // mirror cannot disagree with the register even if the incoming CSR held
-    // only one of the two bits.
+        MkwSetHostFpControl(want);
+    // `want` has every flush-to-zero bit set or every one clear, so this is exactly
+    // `(MkwGetHostFpControl() & kMkwFpControlFlushToZeroBits) != 0` after the write - the
+    // mirror cannot disagree with the register even if the incoming value held only some of
+    // those bits.
     g_mkwHostNiActive = wantNi;
     g_mkwNiFlushThreshold = wantNi ? kMkwNiFlushThreshold : 0.0;
 }
 
 /// <summary>
-/// Restores a previously captured MXCSR value and re-derives the mirror from
-/// it. Every raw restore has to go through here; a bare _mm_setcsr would leave
-/// the mirror describing the FP environment that was just replaced.
+/// Restores a previously captured MXCSR/FPCR value and re-derives the mirror from it. Every raw
+/// restore has to go through here; a bare MkwSetHostFpControl would leave the mirror describing
+/// the FP environment that was just replaced.
 /// </summary>
 inline void MkwRestoreHostMxcsr(uint32_t csr) noexcept
 {
-    _mm_setcsr(csr);
-    const bool niActive = (csr & kMkwMxcsrFlushToZeroBits) != 0;
+    MkwSetHostFpControl(csr);
+    const bool niActive = (csr & kMkwFpControlFlushToZeroBits) != 0;
     g_mkwHostNiActive = niActive;
     g_mkwNiFlushThreshold = niActive ? kMkwNiFlushThreshold : 0.0;
 }

runtime/include/isa/ppc_isa_quantized.h

@@ -12,6 +12,7 @@
 #include <cmath>
 #include <cstdint>
 #include <cstdlib>
+#include <cstring>
 #include <limits>
 #include <type_traits>
 #include <utility>
@@ -38,16 +39,26 @@ inline uint32_t PpcStorePsqFloatBitsInline(uint32_t value)
 
 inline uint64_t PpcLoadPairPsqFloatBitsPackedInline(uint64_t value)
 {
+#if defined(__x86_64__)
     const __m128i lanes = _mm_cvtsi64_si128(static_cast<long long>(value));
     const __m128i magnitude = _mm_and_si128(lanes, _mm_set1_epi32(0x7FFFFFFF));
     const __m128i nanMask = _mm_cmpgt_epi32(magnitude, _mm_set1_epi32(0x7F800000));
     const __m128i result = _mm_or_si128(
         lanes, _mm_and_si128(nanMask, _mm_set1_epi32(0x00400000)));
     return static_cast<uint64_t>(_mm_cvtsi128_si64(result));
+#elif defined(__aarch64__)
+    // Equivalent to applying PpcLoadPsqFloatBitsInline to each 32-bit lane: the
+    // x86 body above only ever acts on these same two lanes (the upper 64 bits
+    // _mm_cvtsi64_si128 zero-fills never survive the final truncating extract).
+    const uint32_t lo = PpcLoadPsqFloatBitsInline(static_cast<uint32_t>(value));
+    const uint32_t hi = PpcLoadPsqFloatBitsInline(static_cast<uint32_t>(value >> 32));
+    return (static_cast<uint64_t>(hi) << 32) | lo;
+#endif
 }
 
 inline uint64_t PpcStorePairPsqFloatBitsPackedInline(uint64_t value)
 {
+#if defined(__x86_64__)
     const __m128i lanes = _mm_cvtsi64_si128(static_cast<long long>(value));
     const __m128i magnitude = _mm_and_si128(lanes, _mm_set1_epi32(0x7FFFFFFF));
     const __m128i subnormalMask = _mm_cmplt_epi32(magnitude, _mm_set1_epi32(0x00800000));
@@ -60,8 +71,16 @@ inline uint64_t PpcStorePairPsqFloatBitsPackedInline(uint64_t value)
         _mm_and_si128(subnormalMask, signedZero),
         _mm_andnot_si128(subnormalMask, quieted));
     return static_cast<uint64_t>(_mm_cvtsi128_si64(result));
+#elif defined(__aarch64__)
+    // Equivalent to applying PpcStorePsqFloatBitsInline to each 32-bit lane;
+    // same reasoning as the load-side port above.
+    const uint32_t lo = PpcStorePsqFloatBitsInline(static_cast<uint32_t>(value));
+    const uint32_t hi = PpcStorePsqFloatBitsInline(static_cast<uint32_t>(value >> 32));
+    return (static_cast<uint64_t>(hi) << 32) | lo;
+#endif
 }
 
+#if defined(__x86_64__)
 inline __m128i PpcPsqSwapPairBytesInline(__m128i lanes)
 {
     const __m128i order = _mm_setr_epi8(
@@ -95,21 +114,42 @@ inline __m128i PpcStorePairPsqFloatBitsLanesInline(__m128i lanes)
         _mm_and_si128(subnormalMask, signedZero),
         _mm_andnot_si128(subnormalMask, quieted));
 }
+#endif // defined(__x86_64__)
 
 // host -> packed FPR double, guard already proven by the caller.
 inline double PpcLoadPairPsqFloatFromHostInline(const uint8_t* host)
 {
+#if defined(__x86_64__)
     const __m128i raw = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(host));
     return PpcM128ToPsInline(_mm_castsi128_ps(
         PpcLoadPairPsqFloatBitsLanesInline(PpcPsqSwapPairBytesInline(raw))));
+#elif defined(__aarch64__)
+    // The x86 path's full 8-byte pshufb reversal plus a same-endian load is,
+    // taken together, exactly a 64-bit byteswap of a plain little-endian load:
+    // it turns the on-disk [ps0 big-endian][ps1 big-endian] byte layout into a
+    // native uint64 with low 32 bits = ps1, high 32 bits = ps0 (this file's
+    // documented packed-double lane convention).
+    uint64_t raw = 0;
+    std::memcpy(&raw, host, sizeof(raw));
+    const uint64_t swapped = __builtin_bswap64(raw);
+    return PpcBitCastToDoubleInline(PpcLoadPairPsqFloatBitsPackedInline(swapped));
+#endif
 }
 
 // packed FPR double -> host, guard already proven by the caller.
 inline void PpcStorePairPsqFloatToHostInline(uint8_t* host, double value)
 {
+#if defined(__x86_64__)
     const __m128i lanes = PpcStorePairPsqFloatBitsLanesInline(
         _mm_castps_si128(PpcPsToM128Inline(value)));
     _mm_storel_epi64(reinterpret_cast<__m128i*>(host), PpcPsqSwapPairBytesInline(lanes));
+#elif defined(__aarch64__)
+    // Inverse of the load path above: bswap64 is its own inverse, so applying
+    // it to the quieted packed value reproduces the on-disk big-endian bytes.
+    const uint64_t quieted = PpcStorePairPsqFloatBitsPackedInline(PpcBitCastToU64Inline(value));
+    const uint64_t swapped = __builtin_bswap64(quieted);
+    std::memcpy(host, &swapped, sizeof(swapped));
+#endif
 }
 
 template <typename SignedType>
@@ -329,6 +369,7 @@ inline uint8_t PpcQuantizePsqU8Scale61Inline(float value)
 // matching PpcQuantizePsqU8Scale61Inline's !(scaled > 0) rule.
 inline uint16_t PpcQuantizePairPsqU8Scale61PackedInline(double value)
 {
+#if defined(__x86_64__)
     const __m128 scaled = _mm_mul_ps(PpcPsToM128Inline(value), _mm_set1_ps(0.125f));
     const __m128 nonNegative = _mm_max_ps(scaled, _mm_setzero_ps());
     const __m128 clamped = _mm_min_ps(nonNegative, _mm_set1_ps(255.0f));
@@ -338,6 +379,17 @@ inline uint16_t PpcQuantizePairPsqU8Scale61PackedInline(double value)
     // Native lane 0 is ps1 and lane 1 is ps0. Packing to the low uint16_t
     // therefore produces the guest-order numeric value (ps0 << 8) | ps1.
     return static_cast<uint16_t>(_mm_cvtsi128_si32(lanes8));
+#elif defined(__aarch64__)
+    // Equivalent to two calls of the already-portable scalar quantizer above
+    // (its own !(scaled > 0) rule maps NaN to 0, matching what MAXPS-with-zero
+    // does on the x86 path per the comment there), packed the same way the
+    // fallback path just below already does for the non-SIMD case.
+    PPC_FPR fpr{};
+    fpr.d = value;
+    const uint8_t q0 = PpcQuantizePsqU8Scale61Inline(fpr.paired.ps0);
+    const uint8_t q1 = PpcQuantizePsqU8Scale61Inline(fpr.paired.ps1);
+    return static_cast<uint16_t>((static_cast<uint16_t>(q0) << 8) | static_cast<uint16_t>(q1));
+#endif
 }
 
 // Preserve the complete memory/MMIO/executable-write behavior off the leaf