#include "isa/ppc_isa_quantized.h" #if defined(__FAST_MATH__) || __FINITE_MATH_ONLY__ #error "PSQ helpers require strict PPC floating-point options" #endif template MKW_PPC_NO_INLINE MKW_PPC_COLD double PPC_PsqLStateFallback(uint32_t gqr, uint32_t addr) { static_assert(W <= 1u); const uint32_t type = (gqr >> 16) & 0x7u; const uint32_t scale = (gqr >> 24) & 0x3Fu; if constexpr (W == 0u) { switch (type) { case 0u: return Stack ? PpcLoadPairPsqFloatStackInline(addr) : PpcLoadPairPsqFloatFastInline(addr); case 4u: return Stack ? PpcLoadPairPsqIntegerStackInline(addr, scale) : PpcLoadPairPsqIntegerFastInline(addr, scale); case 5u: return Stack ? PpcLoadPairPsqIntegerStackInline(addr, scale) : PpcLoadPairPsqIntegerFastInline(addr, scale); case 6u: return Stack ? PpcLoadPairPsqIntegerStackInline(addr, scale) : PpcLoadPairPsqIntegerFastInline(addr, scale); case 7u: return Stack ? PpcLoadPairPsqIntegerStackInline(addr, scale) : PpcLoadPairPsqIntegerFastInline(addr, scale); default: std::abort(); } } else { switch (type) { case 0u: return Stack ? PpcLoadSinglePsqFloatStackInline(addr) : PpcLoadSinglePsqFloatFastInline(addr); case 4u: return Stack ? PpcLoadSinglePsqQuantizedStackInline(addr, scale) : PpcLoadSinglePsqQuantizedFastInline(addr, scale); case 5u: return Stack ? PpcLoadSinglePsqQuantizedStackInline(addr, scale) : PpcLoadSinglePsqQuantizedFastInline(addr, scale); case 6u: return Stack ? PpcLoadSinglePsqQuantizedStackInline(addr, scale) : PpcLoadSinglePsqQuantizedFastInline(addr, scale); case 7u: return Stack ? PpcLoadSinglePsqQuantizedStackInline(addr, scale) : PpcLoadSinglePsqQuantizedFastInline(addr, scale); default: std::abort(); } } } template MKW_PPC_NO_INLINE MKW_PPC_COLD void PPC_PsqStStateFallback(uint32_t gqr, uint32_t addr, double value) { static_assert(W <= 1u); const uint32_t type = gqr & 0x7u; const uint32_t scale = (gqr >> 8) & 0x3Fu; if constexpr (W == 0u) { switch (type) { case 0u: Stack ? PpcStorePairPsqFloatStackInline(addr, value) : PpcStorePairPsqFloatFastInline(addr, value); return; case 4u: if constexpr (Stack) PpcStorePairPsqQuantizedStackInline(addr, value, scale); else PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; case 5u: if constexpr (Stack) PpcStorePairPsqQuantizedStackInline(addr, value, scale); else PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; case 6u: if constexpr (Stack) PpcStorePairPsqQuantizedStackInline(addr, value, scale); else PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; case 7u: if constexpr (Stack) PpcStorePairPsqQuantizedStackInline(addr, value, scale); else PpcStorePairPsqQuantizedFastInline(addr, value, scale); return; default: std::abort(); } } else { switch (type) { case 0u: if constexpr (Stack) PpcStoreSinglePsqFloatStackInline(addr, value); else PpcStoreSinglePsqFloatFastInline(addr, value); return; case 4u: if constexpr (Stack) PpcStoreSinglePsqQuantizedStackInline(addr, value, scale); else PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; case 5u: if constexpr (Stack) PpcStoreSinglePsqQuantizedStackInline(addr, value, scale); else PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; case 6u: if constexpr (Stack) PpcStoreSinglePsqQuantizedStackInline(addr, value, scale); else PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; case 7u: if constexpr (Stack) PpcStoreSinglePsqQuantizedStackInline(addr, value, scale); else PpcStoreSinglePsqQuantizedFastInline(addr, value, scale); return; default: std::abort(); } } } template MKW_PPC_NO_INLINE MKW_PPC_COLD double PPC_PsqLResolvedStateFallback( uint32_t gqr, uint8_t* resolvedHost, uint32_t offset, uint32_t addr) { static_assert(W <= 1u); if (!resolvedHost) [[unlikely]] return PPC_PsqLStateInline(gqr, addr); const uint32_t type = (gqr >> 16) & 0x7u; const uint32_t scale = (gqr >> 24) & 0x3Fu; if constexpr (W == 0u) { switch (type) { case 0u: return PpcLoadPairPsqFloatResolvedInline(resolvedHost, offset, addr); case 4u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, scale); case 5u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, scale); case 6u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, scale); case 7u: return PpcLoadPairPsqIntegerResolvedInline(resolvedHost, offset, addr, scale); default: std::abort(); } } else { switch (type) { case 0u: return PpcLoadSinglePsqFloatResolvedInline(resolvedHost, offset, addr); case 4u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, scale); case 5u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, scale); case 6u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, scale); case 7u: return PpcLoadSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, scale); default: std::abort(); } } } template MKW_PPC_NO_INLINE MKW_PPC_COLD void PPC_PsqStResolvedStateFallback( uint32_t gqr, uint8_t* resolvedHost, uint32_t offset, uint32_t addr, double value) { static_assert(W <= 1u); if (!resolvedHost) [[unlikely]] { PPC_PsqStStateInline(gqr, addr, value); return; } const uint32_t type = gqr & 0x7u; const uint32_t scale = (gqr >> 8) & 0x3Fu; if constexpr (W == 0u) { switch (type) { case 0u: PpcStorePairPsqFloatResolvedInline(resolvedHost, offset, addr, value); return; case 4u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; case 5u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; case 6u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; case 7u: PpcStorePairPsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; default: std::abort(); } } else { switch (type) { case 0u: PpcStoreSinglePsqFloatResolvedInline(resolvedHost, offset, addr, value); return; case 4u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; case 5u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; case 6u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; case 7u: PpcStoreSinglePsqQuantizedResolvedInline(resolvedHost, offset, addr, value, scale); return; default: std::abort(); } } } template double PPC_PsqLStateFallback<0u, false>(uint32_t, uint32_t); template void PPC_PsqStStateFallback<0u, false>(uint32_t, uint32_t, double); template double PPC_PsqLStateFallback<0u, true>(uint32_t, uint32_t); template void PPC_PsqStStateFallback<0u, true>(uint32_t, uint32_t, double); template double PPC_PsqLResolvedStateFallback<0u>(uint32_t, uint8_t*, uint32_t, uint32_t); template void PPC_PsqStResolvedStateFallback<0u>(uint32_t, uint8_t*, uint32_t, uint32_t, double); template double PPC_PsqLStateFallback<1u, false>(uint32_t, uint32_t); template void PPC_PsqStStateFallback<1u, false>(uint32_t, uint32_t, double); template double PPC_PsqLStateFallback<1u, true>(uint32_t, uint32_t); template void PPC_PsqStStateFallback<1u, true>(uint32_t, uint32_t, double); template double PPC_PsqLResolvedStateFallback<1u>(uint32_t, uint8_t*, uint32_t, uint32_t); template void PPC_PsqStResolvedStateFallback<1u>(uint32_t, uint8_t*, uint32_t, uint32_t, double);