From c57016c272937205e45855b74de6590f3b2b39ae Mon Sep 17 00:00:00 2001 From: salh Date: Mon, 20 Apr 2026 17:23:49 +0300 Subject: [PATCH] Track vendored simde x86 headers --- thirdparty/rexglue-sdk/.gitignore | 2 + .../thirdparty/simde/simde/x86/aes.h | 417 + .../thirdparty/simde/simde/x86/avx.h | 6267 ++++++++ .../thirdparty/simde/simde/x86/avx2.h | 5758 ++++++++ .../thirdparty/simde/simde/x86/avx512.h | 149 + .../simde/simde/x86/avx512/2intersect.h | 249 + .../simde/simde/x86/avx512/4dpwssd.h | 67 + .../simde/simde/x86/avx512/4dpwssds.h | 67 + .../thirdparty/simde/simde/x86/avx512/abs.h | 580 + .../thirdparty/simde/simde/x86/avx512/add.h | 625 + .../thirdparty/simde/simde/x86/avx512/adds.h | 529 + .../thirdparty/simde/simde/x86/avx512/and.h | 305 + .../simde/simde/x86/avx512/andnot.h | 193 + .../thirdparty/simde/simde/x86/avx512/avg.h | 258 + .../simde/simde/x86/avx512/bitshuffle.h | 202 + .../thirdparty/simde/simde/x86/avx512/blend.h | 293 + .../simde/simde/x86/avx512/broadcast.h | 897 ++ .../thirdparty/simde/simde/x86/avx512/cast.h | 357 + .../thirdparty/simde/simde/x86/avx512/cmp.h | 1714 +++ .../thirdparty/simde/simde/x86/avx512/cmpeq.h | 241 + .../thirdparty/simde/simde/x86/avx512/cmpge.h | 1434 ++ .../thirdparty/simde/simde/x86/avx512/cmpgt.h | 212 + .../thirdparty/simde/simde/x86/avx512/cmple.h | 1432 ++ .../thirdparty/simde/simde/x86/avx512/cmplt.h | 123 + .../simde/simde/x86/avx512/cmpneq.h | 490 + .../simde/simde/x86/avx512/compress.h | 755 + .../simde/simde/x86/avx512/conflict.h | 351 + .../simde/simde/x86/avx512/copysign.h | 86 + .../thirdparty/simde/simde/x86/avx512/cvt.h | 402 + .../thirdparty/simde/simde/x86/avx512/cvts.h | 781 + .../thirdparty/simde/simde/x86/avx512/cvtt.h | 130 + .../thirdparty/simde/simde/x86/avx512/cvtus.h | 67 + .../thirdparty/simde/simde/x86/avx512/dbsad.h | 388 + .../thirdparty/simde/simde/x86/avx512/div.h | 162 + .../simde/simde/x86/avx512/dpbf16.h | 281 + .../simde/simde/x86/avx512/dpbusd.h | 292 + .../simde/simde/x86/avx512/dpbusds.h | 344 + .../simde/simde/x86/avx512/dpwssd.h | 269 + .../simde/simde/x86/avx512/dpwssds.h | 299 + .../simde/simde/x86/avx512/expand.h | 97 + .../simde/simde/x86/avx512/extract.h | 267 + .../simde/simde/x86/avx512/fixupimm.h | 900 ++ .../simde/simde/x86/avx512/fixupimm_round.h | 687 + .../simde/simde/x86/avx512/flushsubnormal.h | 91 + .../thirdparty/simde/simde/x86/avx512/fmadd.h | 136 + .../thirdparty/simde/simde/x86/avx512/fmsub.h | 276 + .../simde/simde/x86/avx512/fnmadd.h | 108 + .../simde/simde/x86/avx512/fnmsub.h | 108 + .../simde/simde/x86/avx512/fpclass.h | 99 + .../simde/simde/x86/avx512/gather.h | 312 + .../simde/simde/x86/avx512/insert.h | 490 + .../thirdparty/simde/simde/x86/avx512/kand.h | 53 + .../thirdparty/simde/simde/x86/avx512/knot.h | 106 + .../simde/simde/x86/avx512/kshift.h | 152 + .../thirdparty/simde/simde/x86/avx512/kxor.h | 107 + .../thirdparty/simde/simde/x86/avx512/load.h | 115 + .../thirdparty/simde/simde/x86/avx512/loadu.h | 297 + .../thirdparty/simde/simde/x86/avx512/lzcnt.h | 220 + .../thirdparty/simde/simde/x86/avx512/madd.h | 157 + .../simde/simde/x86/avx512/maddubs.h | 159 + .../thirdparty/simde/simde/x86/avx512/max.h | 611 + .../thirdparty/simde/simde/x86/avx512/min.h | 611 + .../thirdparty/simde/simde/x86/avx512/mov.h | 865 ++ .../simde/simde/x86/avx512/mov_mask.h | 372 + .../thirdparty/simde/simde/x86/avx512/movm.h | 460 + .../thirdparty/simde/simde/x86/avx512/mul.h | 279 + .../thirdparty/simde/simde/x86/avx512/mulhi.h | 65 + .../simde/simde/x86/avx512/mulhrs.h | 65 + .../thirdparty/simde/simde/x86/avx512/mullo.h | 169 + .../simde/simde/x86/avx512/multishift.h | 170 + .../simde/simde/x86/avx512/negate.h | 88 + .../thirdparty/simde/simde/x86/avx512/or.h | 308 + .../thirdparty/simde/simde/x86/avx512/packs.h | 122 + .../simde/simde/x86/avx512/packus.h | 122 + .../simde/simde/x86/avx512/permutex.h | 101 + .../simde/simde/x86/avx512/permutex2var.h | 1645 +++ .../simde/simde/x86/avx512/permutexvar.h | 1194 ++ .../simde/simde/x86/avx512/popcnt.h | 1346 ++ .../thirdparty/simde/simde/x86/avx512/range.h | 745 + .../simde/simde/x86/avx512/range_round.h | 686 + .../thirdparty/simde/simde/x86/avx512/rcp.h | 65 + .../simde/simde/x86/avx512/reduce.h | 355 + .../thirdparty/simde/simde/x86/avx512/rol.h | 410 + .../thirdparty/simde/simde/x86/avx512/rolv.h | 415 + .../thirdparty/simde/simde/x86/avx512/ror.h | 410 + .../thirdparty/simde/simde/x86/avx512/rorv.h | 391 + .../thirdparty/simde/simde/x86/avx512/round.h | 282 + .../simde/simde/x86/avx512/roundscale.h | 616 + .../simde/simde/x86/avx512/roundscale_round.h | 690 + .../thirdparty/simde/simde/x86/avx512/sad.h | 77 + .../simde/simde/x86/avx512/scalef.h | 389 + .../thirdparty/simde/simde/x86/avx512/set.h | 572 + .../thirdparty/simde/simde/x86/avx512/set1.h | 352 + .../thirdparty/simde/simde/x86/avx512/set4.h | 140 + .../simde/simde/x86/avx512/setone.h | 72 + .../thirdparty/simde/simde/x86/avx512/setr.h | 144 + .../thirdparty/simde/simde/x86/avx512/setr4.h | 140 + .../simde/simde/x86/avx512/setzero.h | 105 + .../thirdparty/simde/simde/x86/avx512/shldv.h | 157 + .../simde/simde/x86/avx512/shuffle.h | 417 + .../thirdparty/simde/simde/x86/avx512/sll.h | 247 + .../thirdparty/simde/simde/x86/avx512/slli.h | 179 + .../thirdparty/simde/simde/x86/avx512/sllv.h | 122 + .../thirdparty/simde/simde/x86/avx512/sqrt.h | 127 + .../thirdparty/simde/simde/x86/avx512/sra.h | 81 + .../thirdparty/simde/simde/x86/avx512/srai.h | 96 + .../thirdparty/simde/simde/x86/avx512/srav.h | 67 + .../thirdparty/simde/simde/x86/avx512/srl.h | 216 + .../thirdparty/simde/simde/x86/avx512/srli.h | 180 + .../thirdparty/simde/simde/x86/avx512/srlv.h | 282 + .../thirdparty/simde/simde/x86/avx512/store.h | 93 + .../simde/simde/x86/avx512/storeu.h | 218 + .../thirdparty/simde/simde/x86/avx512/sub.h | 351 + .../thirdparty/simde/simde/x86/avx512/subs.h | 222 + .../simde/simde/x86/avx512/ternarylogic.h | 3769 +++++ .../thirdparty/simde/simde/x86/avx512/test.h | 232 + .../thirdparty/simde/simde/x86/avx512/testn.h | 63 + .../thirdparty/simde/simde/x86/avx512/types.h | 821 ++ .../simde/simde/x86/avx512/unpackhi.h | 753 + .../simde/simde/x86/avx512/unpacklo.h | 752 + .../thirdparty/simde/simde/x86/avx512/xor.h | 319 + .../simde/simde/x86/avx512/xorsign.h | 72 + .../thirdparty/simde/simde/x86/clmul.h | 387 + .../thirdparty/simde/simde/x86/f16c.h | 172 + .../thirdparty/simde/simde/x86/fma.h | 732 + .../thirdparty/simde/simde/x86/gfni.h | 1295 ++ .../thirdparty/simde/simde/x86/mmx.h | 2398 +++ .../thirdparty/simde/simde/x86/sse.h | 4830 ++++++ .../thirdparty/simde/simde/x86/sse2.h | 7741 ++++++++++ .../thirdparty/simde/simde/x86/sse3.h | 515 + .../thirdparty/simde/simde/x86/sse4.1.h | 2367 +++ .../thirdparty/simde/simde/x86/sse4.2.h | 381 + .../thirdparty/simde/simde/x86/ssse3.h | 1057 ++ .../thirdparty/simde/simde/x86/svml.h | 12129 ++++++++++++++++ .../thirdparty/simde/simde/x86/xop.h | 3740 +++++ 135 files changed, 95634 insertions(+) create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/aes.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx2.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/2intersect.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/4dpwssd.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/4dpwssds.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/abs.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/add.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/adds.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/and.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/andnot.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/avg.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/bitshuffle.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/blend.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/broadcast.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cast.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmp.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpeq.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpge.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpgt.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmple.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmplt.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpneq.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/compress.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/conflict.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/copysign.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvt.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvts.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvtt.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvtus.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dbsad.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/div.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpbf16.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpbusd.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpbusds.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpwssd.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpwssds.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/expand.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/extract.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fixupimm.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fixupimm_round.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/flushsubnormal.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fmadd.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fmsub.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fnmadd.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fnmsub.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fpclass.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/gather.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/insert.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/kand.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/knot.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/kshift.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/kxor.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/load.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/loadu.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/lzcnt.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/madd.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/maddubs.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/max.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/min.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mov.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mov_mask.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/movm.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mul.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mulhi.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mulhrs.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mullo.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/multishift.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/negate.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/or.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/packs.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/packus.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/permutex.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/permutex2var.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/permutexvar.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/popcnt.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/range.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/range_round.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rcp.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/reduce.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rol.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rolv.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/ror.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rorv.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/round.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/roundscale.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/roundscale_round.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sad.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/scalef.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/set.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/set1.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/set4.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setone.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setr.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setr4.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setzero.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/shldv.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/shuffle.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sll.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/slli.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sllv.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sqrt.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sra.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srai.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srav.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srl.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srli.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srlv.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/store.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/storeu.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sub.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/subs.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/ternarylogic.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/test.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/testn.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/types.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/unpackhi.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/unpacklo.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/xor.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/xorsign.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/clmul.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/f16c.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/fma.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/gfni.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/mmx.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse2.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse3.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse4.1.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse4.2.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/ssse3.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/svml.h create mode 100644 thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/xop.h diff --git a/thirdparty/rexglue-sdk/.gitignore b/thirdparty/rexglue-sdk/.gitignore index 1efd8cfa..fcb6c67e 100644 --- a/thirdparty/rexglue-sdk/.gitignore +++ b/thirdparty/rexglue-sdk/.gitignore @@ -24,6 +24,8 @@ mono_crash.* [Rr]eleases/ x64/ x86/ +!thirdparty/simde/simde/x86/ +!thirdparty/simde/simde/x86/** !thirdparty/FFmpeg/libavutil/x86/ !thirdparty/FFmpeg/libavutil/x86/** !thirdparty/FFmpeg/libavcodec/x86/ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/aes.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/aes.h new file mode 100644 index 00000000..1d5b0492 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/aes.h @@ -0,0 +1,417 @@ +/* MIT License + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +#if !defined(SIMDE_X86_AES_H) +#define SIMDE_X86_AES_H + +/* + * Advanced Encryption Standard + * @author Dani Huertas + * @email huertas.dani@gmail.com + * + * Based on the document FIPS PUB 197 + */ + +#include "sse2.h" + +/* + * Multiplication in GF(2^8) + * http://en.wikipedia.org/wiki/Finite_field_arithmetic + * Irreducible polynomial m(x) = x8 + x4 + x3 + x + 1 + * + * NOTE: This function can be easily replaced with a look up table for a speed + * boost, at the expense of an increase in memory size. + +SIMDE_FUNCTION_ATTRIBUTES +uint8_t gmult(uint8_t a, uint8_t b) { + uint8_t p = 0, i = 0, hbs = 0; + + for (i = 0; i < 8; i++) { + if (b & 1) { + p ^= a; + } + + hbs = a & 0x80; + a <<= 1; + if (hbs) a ^= 0x1b; // 0000 0001 0001 1011 + b >>= 1; + } + + return (uint8_t)p; +} + */ + +#if !(defined(SIMDE_ARM_NEON_A32V7_NATIVE) && defined(SIMDE_ARCH_ARM_CRYPTO)) + +#include "../simde-aes.h" + +/* + * Transformation in the Cipher and Inverse Cipher in which a Round + * Key is added to the State using an XOR operation. The length of a + * Round Key equals the size of the State (i.e., for Nb = 4, the Round + * Key length equals 128 bits/16 bytes). + */ +SIMDE_FUNCTION_ATTRIBUTES +void simde_x_aes_add_round_key(uint8_t *state, simde__m128i_private w, uint8_t r) { + + int Nb = simde_x_aes_Nb; + uint8_t c; + + for (c = 0; c < Nb; c++) { + state[Nb*0+c] = state[Nb*0+c]^w.u8[4*Nb*r+4*c+0]; + state[Nb*1+c] = state[Nb*1+c]^w.u8[4*Nb*r+4*c+1]; + state[Nb*2+c] = state[Nb*2+c]^w.u8[4*Nb*r+4*c+2]; + state[Nb*3+c] = state[Nb*3+c]^w.u8[4*Nb*r+4*c+3]; + } +} + +/* + * Transformation in the Cipher that takes all of the columns of the + * State and mixes their data (independently of one another) to + * produce new columns. + */ +SIMDE_FUNCTION_ATTRIBUTES +void simde_x_aes_mix_columns(uint8_t *state) { + + int Nb = simde_x_aes_Nb; + // uint8_t k[] = {0x02, 0x01, 0x01, 0x03}; // a(x) = {02} + {01}x + {01}x2 + {03}x3 + uint8_t i, j, col[4], res[4]; + + for (j = 0; j < Nb; j++) { + for (i = 0; i < 4; i++) { + col[i] = state[Nb*i+j]; + } + + //coef_mult(k, col, res); + simde_x_aes_coef_mult_lookup(0, col, res); + + for (i = 0; i < 4; i++) { + state[Nb*i+j] = res[i]; + } + } +} + +/* + * Transformation in the Inverse Cipher that is the inverse of + * MixColumns(). + */ +SIMDE_FUNCTION_ATTRIBUTES +void simde_x_aes_inv_mix_columns(uint8_t *state) { + + int Nb = simde_x_aes_Nb; + // uint8_t k[] = {0x0e, 0x09, 0x0d, 0x0b}; // a(x) = {0e} + {09}x + {0d}x2 + {0b}x3 + uint8_t i, j, col[4], res[4]; + + for (j = 0; j < Nb; j++) { + for (i = 0; i < 4; i++) { + col[i] = state[Nb*i+j]; + } + + //coef_mult(k, col, res); + simde_x_aes_coef_mult_lookup(4, col, res); + + for (i = 0; i < 4; i++) { + state[Nb*i+j] = res[i]; + } + } +} + +/* + * Transformation in the Cipher that processes the State by cyclically + * shifting the last three rows of the State by different offsets. + */ +SIMDE_FUNCTION_ATTRIBUTES +void simde_x_aes_shift_rows(uint8_t *state) { + + int Nb = simde_x_aes_Nb; + uint8_t i, k, s, tmp; + + for (i = 1; i < 4; i++) { + // shift(1,4)=1; shift(2,4)=2; shift(3,4)=3 + // shift(r, 4) = r; + s = 0; + while (s < i) { + tmp = state[Nb*i+0]; + + for (k = 1; k < Nb; k++) { + state[Nb*i+k-1] = state[Nb*i+k]; + } + + state[Nb*i+Nb-1] = tmp; + s++; + } + } +} + +/* + * Transformation in the Inverse Cipher that is the inverse of + * ShiftRows(). + */ +SIMDE_FUNCTION_ATTRIBUTES +void simde_x_aes_inv_shift_rows(uint8_t *state) { + + uint8_t Nb = simde_x_aes_Nb; + uint8_t i, k, s, tmp; + + for (i = 1; i < 4; i++) { + s = 0; + while (s < i) { + tmp = state[Nb*i+Nb-1]; + + for (k = Nb-1; k > 0; k--) { + state[Nb*i+k] = state[Nb*i+k-1]; + } + + state[Nb*i+0] = tmp; + s++; + } + } +} + +/* + * Transformation in the Cipher that processes the State using a non + * linear byte substitution table (S-box) that operates on each of the + * State bytes independently. + */ +SIMDE_FUNCTION_ATTRIBUTES +void simde_x_aes_sub_bytes(uint8_t *state) { + + int Nb = simde_x_aes_Nb; + uint8_t i, j; + + for (i = 0; i < 4; i++) { + for (j = 0; j < Nb; j++) { + // s_box row: yyyy ---- + // s_box col: ---- xxxx + // s_box[16*(yyyy) + xxxx] == s_box[yyyyxxxx] + state[Nb*i+j] = simde_x_aes_s_box[state[Nb*i+j]]; + } + } +} + +/* + * Transformation in the Inverse Cipher that is the inverse of + * SubBytes(). + */ +SIMDE_FUNCTION_ATTRIBUTES +void simde_x_aes_inv_sub_bytes(uint8_t *state) { + + int Nb = simde_x_aes_Nb; + uint8_t i, j; + + for (i = 0; i < 4; i++) { + for (j = 0; j < Nb; j++) { + state[Nb*i+j] = simde_x_aes_inv_s_box[state[Nb*i+j]]; + } + } +} + +/* + * Performs the AES cipher operation + */ +SIMDE_FUNCTION_ATTRIBUTES +void simde_x_aes_enc(simde__m128i_private in, simde__m128i_private *out, simde__m128i_private w, int is_last) { + + int Nb = simde_x_aes_Nb; + uint8_t state[4*simde_x_aes_Nb]; + uint8_t r = 0, i, j; + + for (i = 0; i < 4; i++) { + for (j = 0; j < Nb; j++) { + state[Nb*i+j] = in.u8[i+4*j]; + } + } + + simde_x_aes_sub_bytes(state); + simde_x_aes_shift_rows(state); + + if (!is_last) + simde_x_aes_mix_columns(state); + + simde_x_aes_add_round_key(state, w, r); + + for (i = 0; i < 4; i++) { + for (j = 0; j < Nb; j++) { + out->u8[i+4*j] = state[Nb*i+j]; + } + } +} + +/* + * Performs the AES inverse cipher operation + */ +SIMDE_FUNCTION_ATTRIBUTES +void simde_x_aes_dec(simde__m128i_private in, simde__m128i_private *out, simde__m128i_private w, int is_last) { + + int Nb = simde_x_aes_Nb; + uint8_t state[4*simde_x_aes_Nb]; + uint8_t r = 0, i, j; + + for (i = 0; i < 4; i++) { + for (j = 0; j < Nb; j++) { + state[Nb*i+j] = in.u8[i+4*j]; + } + } + + simde_x_aes_inv_shift_rows(state); + simde_x_aes_inv_sub_bytes(state); + + if (!is_last) + simde_x_aes_inv_mix_columns(state); + + simde_x_aes_add_round_key(state, w, r); + + for (i = 0; i < 4; i++) { + for (j = 0; j < Nb; j++) { + out->u8[i+4*j] = state[Nb*i+j]; + } + } +} +#endif // if !(defined(SIMDE_ARM_NEON_A32V7_NATIVE) && defined(SIMDE_ARCH_ARM_CRYPTO)) + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i simde_mm_aesenc_si128(simde__m128i a, simde__m128i round_key) { + #if defined(SIMDE_X86_AES_NATIVE) + return _mm_aesenc_si128(a, round_key); + #else + simde__m128i_private result_; + simde__m128i_private a_ = simde__m128i_to_private(a); + simde__m128i_private round_key_ = simde__m128i_to_private(round_key); + #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARCH_ARM_CRYPTO) + result_.neon_u8 = veorq_u8( + vaesmcq_u8(vaeseq_u8(a_.neon_u8, vdupq_n_u8(0))), + round_key_.neon_u8); + #else + simde_x_aes_enc(a_, &result_, round_key_, 0); + #endif + return simde__m128i_from_private(result_); + #endif +} +#if defined(SIMDE_X86_AES_ENABLE_NATIVE_ALIASES) + #define _mm_aesenc_si128(a, b) simde_mm_aesenc_si128(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i simde_mm_aesdec_si128(simde__m128i a, simde__m128i round_key) { + #if defined(SIMDE_X86_AES_NATIVE) + return _mm_aesdec_si128(a, round_key); + #else + simde__m128i_private result_; + simde__m128i_private a_ = simde__m128i_to_private(a); + simde__m128i_private round_key_ = simde__m128i_to_private(round_key); + #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARCH_ARM_CRYPTO) + result_.neon_u8 = veorq_u8( + vaesimcq_u8(vaesdq_u8(a_.neon_u8, vdupq_n_u8(0))), + round_key_.neon_u8); + #else + simde_x_aes_dec(a_, &result_, round_key_, 0); + #endif + return simde__m128i_from_private(result_); + #endif +} +#if defined(SIMDE_X86_AES_ENABLE_NATIVE_ALIASES) + #define _mm_aesdec_si128(a, b) simde_mm_aesdec_si128(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i simde_mm_aesenclast_si128(simde__m128i a, simde__m128i round_key) { + #if defined(SIMDE_X86_AES_NATIVE) + return _mm_aesenclast_si128(a, round_key); + #else + simde__m128i_private result_; + simde__m128i_private a_ = simde__m128i_to_private(a); + simde__m128i_private round_key_ = simde__m128i_to_private(round_key); + #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARCH_ARM_CRYPTO) + result_.neon_u8 = vaeseq_u8(a_.neon_u8, vdupq_n_u8(0)); + result_.neon_i32 = veorq_s32(result_.neon_i32, round_key_.neon_i32); // _mm_xor_si128 + #else + simde_x_aes_enc(a_, &result_, round_key_, 1); + #endif + return simde__m128i_from_private(result_); + #endif +} +#if defined(SIMDE_X86_AES_ENABLE_NATIVE_ALIASES) + #define _mm_aesenclast_si128(a, b) simde_mm_aesenclast_si128(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i simde_mm_aesdeclast_si128(simde__m128i a, simde__m128i round_key) { + #if defined(SIMDE_X86_AES_NATIVE) + return _mm_aesdeclast_si128(a, round_key); + #else + simde__m128i_private result_; + simde__m128i_private a_ = simde__m128i_to_private(a); + simde__m128i_private round_key_ = simde__m128i_to_private(round_key); + #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARCH_ARM_CRYPTO) + result_.neon_u8 = veorq_u8( + vaesdq_u8(a_.neon_u8, vdupq_n_u8(0)), + round_key_.neon_u8); + #else + simde_x_aes_dec(a_, &result_, round_key_, 1); + #endif + return simde__m128i_from_private(result_); + #endif +} +#if defined(SIMDE_X86_AES_ENABLE_NATIVE_ALIASES) + #define _mm_aesdeclast_si128(a, b) simde_mm_aesdeclast_si128(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i simde_mm_aesimc_si128(simde__m128i a) { + #if defined(SIMDE_X86_AES_NATIVE) + return _mm_aesimc_si128(a); + #else + simde__m128i_private result_ = simde__m128i_to_private(simde_mm_setzero_si128()); + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_ARCH_ARM_CRYPTO) + result_.neon_u8 = vaesimcq_u8(a_.neon_u8); + #else + int Nb = simde_x_aes_Nb; + // uint8_t k[] = {0x0e, 0x09, 0x0d, 0x0b}; // a(x) = {0e} + {09}x + {0d}x2 + {0b}x3 + uint8_t i, j, col[4], res[4]; + + for (j = 0; j < Nb; j++) { + for (i = 0; i < 4; i++) { + col[i] = a_.u8[Nb*j+i]; + } + + //coef_mult(k, col, res); + simde_x_aes_coef_mult_lookup(4, col, res); + + for (i = 0; i < 4; i++) { + result_.u8[Nb*j+i] = res[i]; + } + } + #endif + return simde__m128i_from_private(result_); + #endif +} +#if defined(SIMDE_X86_AES_ENABLE_NATIVE_ALIASES) + #define _mm_aesimc_si128(a) simde_mm_aesimc_si128(a) +#endif + +#undef simde_x_aes_Nb + +#endif /* !defined(SIMDE_X86_AES_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx.h new file mode 100644 index 00000000..2314f955 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx.h @@ -0,0 +1,6267 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2018-2020 Evan Nemerson + * 2020 Michael R. Crusoe + */ + +#include "sse.h" +#if !defined(SIMDE_X86_AVX_H) +#define SIMDE_X86_AVX_H + +#include "sse4.2.h" +#include "../simde-f16.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +typedef union { + #if defined(SIMDE_VECTOR_SUBSCRIPT) + SIMDE_ALIGN_TO_32 int8_t i8 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int16_t i16 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int32_t i32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int64_t i64 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint8_t u8 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint16_t u16 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint32_t u32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint64_t u64 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_ALIGN_TO_32 simde_int128 i128 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 simde_uint128 u128 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + #endif + SIMDE_ALIGN_TO_32 simde_float32 f32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 simde_float64 f64 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int_fast32_t i32f SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint_fast32_t u32f SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + #else + SIMDE_ALIGN_TO_32 int8_t i8[32]; + SIMDE_ALIGN_TO_32 int16_t i16[16]; + SIMDE_ALIGN_TO_32 int32_t i32[8]; + SIMDE_ALIGN_TO_32 int64_t i64[4]; + SIMDE_ALIGN_TO_32 uint8_t u8[32]; + SIMDE_ALIGN_TO_32 uint16_t u16[16]; + SIMDE_ALIGN_TO_32 uint32_t u32[8]; + SIMDE_ALIGN_TO_32 uint64_t u64[4]; + SIMDE_ALIGN_TO_32 int_fast32_t i32f[32 / sizeof(int_fast32_t)]; + SIMDE_ALIGN_TO_32 uint_fast32_t u32f[32 / sizeof(uint_fast32_t)]; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_ALIGN_TO_32 simde_int128 i128[2]; + SIMDE_ALIGN_TO_32 simde_uint128 u128[2]; + #endif + SIMDE_ALIGN_TO_32 simde_float32 f32[8]; + SIMDE_ALIGN_TO_32 simde_float64 f64[4]; + #endif + + SIMDE_ALIGN_TO_32 simde__m128_private m128_private[2]; + SIMDE_ALIGN_TO_32 simde__m128 m128[2]; + + #if defined(SIMDE_X86_AVX_NATIVE) + SIMDE_ALIGN_TO_32 __m256 n; + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) altivec_u8[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) altivec_u16[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed char) altivec_i8[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed short) altivec_i16[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(int) altivec_i32[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(float) altivec_f32[2]; + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long) altivec_u64[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(long long) altivec_i64[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(double) altivec_f64[2]; + #endif + #endif +} simde__m256_private; + +typedef union { + #if defined(SIMDE_VECTOR_SUBSCRIPT) + SIMDE_ALIGN_TO_32 int8_t i8 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int16_t i16 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int32_t i32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int64_t i64 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint8_t u8 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint16_t u16 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint32_t u32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint64_t u64 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_ALIGN_TO_32 simde_int128 i128 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 simde_uint128 u128 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + #endif + SIMDE_ALIGN_TO_32 simde_float32 f32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 simde_float64 f64 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int_fast32_t i32f SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint_fast32_t u32f SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + #else + SIMDE_ALIGN_TO_32 int8_t i8[32]; + SIMDE_ALIGN_TO_32 int16_t i16[16]; + SIMDE_ALIGN_TO_32 int32_t i32[8]; + SIMDE_ALIGN_TO_32 int64_t i64[4]; + SIMDE_ALIGN_TO_32 uint8_t u8[32]; + SIMDE_ALIGN_TO_32 uint16_t u16[16]; + SIMDE_ALIGN_TO_32 uint32_t u32[8]; + SIMDE_ALIGN_TO_32 uint64_t u64[4]; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_ALIGN_TO_32 simde_int128 i128[2]; + SIMDE_ALIGN_TO_32 simde_uint128 u128[2]; + #endif + SIMDE_ALIGN_TO_32 simde_float32 f32[8]; + SIMDE_ALIGN_TO_32 simde_float64 f64[4]; + SIMDE_ALIGN_TO_32 int_fast32_t i32f[32 / sizeof(int_fast32_t)]; + SIMDE_ALIGN_TO_32 uint_fast32_t u32f[32 / sizeof(uint_fast32_t)]; + #endif + + SIMDE_ALIGN_TO_32 simde__m128d_private m128d_private[2]; + SIMDE_ALIGN_TO_32 simde__m128d m128d[2]; + + #if defined(SIMDE_X86_AVX_NATIVE) + SIMDE_ALIGN_TO_32 __m256d n; + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) altivec_u8[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) altivec_u16[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed char) altivec_i8[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed short) altivec_i16[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed int) altivec_i32[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(float) altivec_f32[2]; + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long) altivec_u64[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed long long) altivec_i64[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(double) altivec_f64[2]; + #endif + #endif +} simde__m256d_private; + +typedef union { + #if defined(SIMDE_VECTOR_SUBSCRIPT) + SIMDE_ALIGN_TO_32 int8_t i8 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int16_t i16 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int32_t i32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int64_t i64 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint8_t u8 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint16_t u16 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint32_t u32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint64_t u64 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_ALIGN_TO_32 simde_int128 i128 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 simde_uint128 u128 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + #endif + #if defined(SIMDE_FLOAT16_VECTOR) + SIMDE_ALIGN_TO_32 simde_float16 f16 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + #else + SIMDE_ALIGN_TO_32 simde_float16 f16[16]; + #endif + SIMDE_ALIGN_TO_32 simde_float32 f32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 simde_float64 f64 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int_fast32_t i32f SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint_fast32_t u32f SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + #else + SIMDE_ALIGN_TO_32 int8_t i8[32]; + SIMDE_ALIGN_TO_32 int16_t i16[16]; + SIMDE_ALIGN_TO_32 int32_t i32[8]; + SIMDE_ALIGN_TO_32 int64_t i64[4]; + SIMDE_ALIGN_TO_32 uint8_t u8[32]; + SIMDE_ALIGN_TO_32 uint16_t u16[16]; + SIMDE_ALIGN_TO_32 uint32_t u32[8]; + SIMDE_ALIGN_TO_32 uint64_t u64[4]; + SIMDE_ALIGN_TO_32 int_fast32_t i32f[32 / sizeof(int_fast32_t)]; + SIMDE_ALIGN_TO_32 uint_fast32_t u32f[32 / sizeof(uint_fast32_t)]; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_ALIGN_TO_32 simde_int128 i128[2]; + SIMDE_ALIGN_TO_32 simde_uint128 u128[2]; + #endif + SIMDE_ALIGN_TO_32 simde_float16 f16[16]; + SIMDE_ALIGN_TO_32 simde_float32 f32[8]; + SIMDE_ALIGN_TO_32 simde_float64 f64[4]; + #endif + + SIMDE_ALIGN_TO_32 simde__m128i_private m128i_private[2]; + SIMDE_ALIGN_TO_32 simde__m128i m128i[2]; + + #if defined(SIMDE_X86_AVX_NATIVE) + SIMDE_ALIGN_TO_32 __m256i n; + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) altivec_u8[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) altivec_u16[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed char) altivec_i8[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed short) altivec_i16[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed int) altivec_i32[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(float) altivec_f32[2]; + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long) altivec_u64[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed long long) altivec_i64[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(double) altivec_f64[2]; + #endif + #endif +} simde__m256i_private; + +#if defined(SIMDE_X86_AVX_NATIVE) + typedef __m256 simde__m256; + typedef __m256i simde__m256i; + typedef __m256d simde__m256d; +#elif defined(SIMDE_VECTOR_SUBSCRIPT) + typedef simde_float32 simde__m256 SIMDE_ALIGN_TO_32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + typedef int_fast32_t simde__m256i SIMDE_ALIGN_TO_32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + typedef simde_float64 simde__m256d SIMDE_ALIGN_TO_32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; +#else + typedef simde__m256_private simde__m256; + typedef simde__m256i_private simde__m256i; + typedef simde__m256d_private simde__m256d; +#endif + +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #if !defined(HEDLEY_INTEL_VERSION) && !defined(_AVXINTRIN_H_INCLUDED) && !defined(__AVXINTRIN_H) && !defined(_CMP_EQ_OQ) + typedef simde__m256 __m256; + typedef simde__m256i __m256i; + typedef simde__m256d __m256d; + #else + #undef __m256 + #define __m256 simde__m256 + #undef __m256i + #define __m256i simde__m256i + #undef __m256d + #define __m256d simde__m256d + #endif +#endif + +HEDLEY_STATIC_ASSERT(32 == sizeof(simde__m256), "simde__m256 size incorrect"); +HEDLEY_STATIC_ASSERT(32 == sizeof(simde__m256_private), "simde__m256_private size incorrect"); +HEDLEY_STATIC_ASSERT(32 == sizeof(simde__m256i), "simde__m256i size incorrect"); +HEDLEY_STATIC_ASSERT(32 == sizeof(simde__m256i_private), "simde__m256i_private size incorrect"); +HEDLEY_STATIC_ASSERT(32 == sizeof(simde__m256d), "simde__m256d size incorrect"); +HEDLEY_STATIC_ASSERT(32 == sizeof(simde__m256d_private), "simde__m256d_private size incorrect"); +#if defined(SIMDE_CHECK_ALIGNMENT) && defined(SIMDE_ALIGN_OF) +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m256) == 32, "simde__m256 is not 32-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m256_private) == 32, "simde__m256_private is not 32-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m256i) == 32, "simde__m256i is not 32-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m256i_private) == 32, "simde__m256i_private is not 32-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m256d) == 32, "simde__m256d is not 32-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m256d_private) == 32, "simde__m256d_private is not 32-byte aligned"); +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde__m256_from_private(simde__m256_private v) { + simde__m256 r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256_private +simde__m256_to_private(simde__m256 v) { + simde__m256_private r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde__m256i_from_private(simde__m256i_private v) { + simde__m256i r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i_private +simde__m256i_to_private(simde__m256i v) { + simde__m256i_private r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde__m256d_from_private(simde__m256d_private v) { + simde__m256d r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d_private +simde__m256d_to_private(simde__m256d v) { + simde__m256d_private r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +#define SIMDE_CMP_EQ_OQ 0 +#define SIMDE_CMP_LT_OS 1 +#define SIMDE_CMP_LE_OS 2 +#define SIMDE_CMP_UNORD_Q 3 +#define SIMDE_CMP_NEQ_UQ 4 +#define SIMDE_CMP_NLT_US 5 +#define SIMDE_CMP_NLE_US 6 +#define SIMDE_CMP_ORD_Q 7 +#define SIMDE_CMP_EQ_UQ 8 +#define SIMDE_CMP_NGE_US 9 +#define SIMDE_CMP_NGT_US 10 +#define SIMDE_CMP_FALSE_OQ 11 +#define SIMDE_CMP_NEQ_OQ 12 +#define SIMDE_CMP_GE_OS 13 +#define SIMDE_CMP_GT_OS 14 +#define SIMDE_CMP_TRUE_UQ 15 +#define SIMDE_CMP_EQ_OS 16 +#define SIMDE_CMP_LT_OQ 17 +#define SIMDE_CMP_LE_OQ 18 +#define SIMDE_CMP_UNORD_S 19 +#define SIMDE_CMP_NEQ_US 20 +#define SIMDE_CMP_NLT_UQ 21 +#define SIMDE_CMP_NLE_UQ 22 +#define SIMDE_CMP_ORD_S 23 +#define SIMDE_CMP_EQ_US 24 +#define SIMDE_CMP_NGE_UQ 25 +#define SIMDE_CMP_NGT_UQ 26 +#define SIMDE_CMP_FALSE_OS 27 +#define SIMDE_CMP_NEQ_OS 28 +#define SIMDE_CMP_GE_OQ 29 +#define SIMDE_CMP_GT_OQ 30 +#define SIMDE_CMP_TRUE_US 31 + +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) && !defined(_CMP_EQ_OQ) +#define _CMP_EQ_OQ SIMDE_CMP_EQ_OQ +#define _CMP_LT_OS SIMDE_CMP_LT_OS +#define _CMP_LE_OS SIMDE_CMP_LE_OS +#define _CMP_UNORD_Q SIMDE_CMP_UNORD_Q +#define _CMP_NEQ_UQ SIMDE_CMP_NEQ_UQ +#define _CMP_NLT_US SIMDE_CMP_NLT_US +#define _CMP_NLE_US SIMDE_CMP_NLE_US +#define _CMP_ORD_Q SIMDE_CMP_ORD_Q +#define _CMP_EQ_UQ SIMDE_CMP_EQ_UQ +#define _CMP_NGE_US SIMDE_CMP_NGE_US +#define _CMP_NGT_US SIMDE_CMP_NGT_US +#define _CMP_FALSE_OQ SIMDE_CMP_FALSE_OQ +#define _CMP_NEQ_OQ SIMDE_CMP_NEQ_OQ +#define _CMP_GE_OS SIMDE_CMP_GE_OS +#define _CMP_GT_OS SIMDE_CMP_GT_OS +#define _CMP_TRUE_UQ SIMDE_CMP_TRUE_UQ +#define _CMP_EQ_OS SIMDE_CMP_EQ_OS +#define _CMP_LT_OQ SIMDE_CMP_LT_OQ +#define _CMP_LE_OQ SIMDE_CMP_LE_OQ +#define _CMP_UNORD_S SIMDE_CMP_UNORD_S +#define _CMP_NEQ_US SIMDE_CMP_NEQ_US +#define _CMP_NLT_UQ SIMDE_CMP_NLT_UQ +#define _CMP_NLE_UQ SIMDE_CMP_NLE_UQ +#define _CMP_ORD_S SIMDE_CMP_ORD_S +#define _CMP_EQ_US SIMDE_CMP_EQ_US +#define _CMP_NGE_UQ SIMDE_CMP_NGE_UQ +#define _CMP_NGT_UQ SIMDE_CMP_NGT_UQ +#define _CMP_FALSE_OS SIMDE_CMP_FALSE_OS +#define _CMP_NEQ_OS SIMDE_CMP_NEQ_OS +#define _CMP_GE_OQ SIMDE_CMP_GE_OQ +#define _CMP_GT_OQ SIMDE_CMP_GT_OQ +#define _CMP_TRUE_US SIMDE_CMP_TRUE_US +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_castps_pd (simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_castps_pd(a); + #else + return *HEDLEY_REINTERPRET_CAST(simde__m256d*, &a); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_castps_pd + #define _mm256_castps_pd(a) simde_mm256_castps_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_castps_si256 (simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_castps_si256(a); + #else + return *HEDLEY_REINTERPRET_CAST(simde__m256i*, &a); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_castps_si256 + #define _mm256_castps_si256(a) simde_mm256_castps_si256(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_castsi256_pd (simde__m256i a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_castsi256_pd(a); + #else + return *HEDLEY_REINTERPRET_CAST(simde__m256d*, &a); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_castsi256_pd + #define _mm256_castsi256_pd(a) simde_mm256_castsi256_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_castsi256_ps (simde__m256i a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_castsi256_ps(a); + #else + return *HEDLEY_REINTERPRET_CAST(simde__m256*, &a); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_castsi256_ps + #define _mm256_castsi256_ps(a) simde_mm256_castsi256_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_castpd_ps (simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_castpd_ps(a); + #else + return *HEDLEY_REINTERPRET_CAST(simde__m256*, &a); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_castpd_ps + #define _mm256_castpd_ps(a) simde_mm256_castpd_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_castpd_si256 (simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_castpd_si256(a); + #else + return *HEDLEY_REINTERPRET_CAST(simde__m256i*, &a); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_castpd_si256 + #define _mm256_castpd_si256(a) simde_mm256_castpd_si256(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_setzero_si256 (void) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_setzero_si256(); + #else + simde__m256i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_setzero_si128(); + r_.m128i[1] = simde_mm_setzero_si128(); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = 0; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_setzero_si256 + #define _mm256_setzero_si256() simde_mm256_setzero_si256() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_setzero_ps (void) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_setzero_ps(); + #else + return simde_mm256_castsi256_ps(simde_mm256_setzero_si256()); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_setzero_ps + #define _mm256_setzero_ps() simde_mm256_setzero_ps() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_setzero_pd (void) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_setzero_pd(); + #else + return simde_mm256_castsi256_pd(simde_mm256_setzero_si256()); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_setzero_pd + #define _mm256_setzero_pd() simde_mm256_setzero_pd() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_x_mm256_not_ps(simde__m256 a) { + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = ~a_.i32; + #elif SIMDE_NATURAL_VECTOR_SIZE_GE(128) + r_.m128[0] = simde_x_mm_not_ps(a_.m128[0]); + r_.m128[1] = simde_x_mm_not_ps(a_.m128[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ~(a_.i32[i]); + } + #endif + + return simde__m256_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_x_mm256_select_ps(simde__m256 a, simde__m256 b, simde__m256 mask) { + /* This function is for when you want to blend two elements together + * according to a mask. It is similar to _mm256_blendv_ps, except that + * it is undefined whether the blend is based on the highest bit in + * each lane (like blendv) or just bitwise operations. This allows + * us to implement the function efficiently everywhere. + * + * Basically, you promise that all the lanes in mask are either 0 or + * ~0. */ + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_blendv_ps(a, b, mask); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b), + mask_ = simde__m256_to_private(mask); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 ^ ((a_.i32 ^ b_.i32) & mask_.i32); + #elif SIMDE_NATURAL_VECTOR_SIZE_GE(128) + r_.m128[0] = simde_x_mm_select_ps(a_.m128[0], b_.m128[0], mask_.m128[0]); + r_.m128[1] = simde_x_mm_select_ps(a_.m128[1], b_.m128[1], mask_.m128[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] ^ ((a_.i32[i] ^ b_.i32[i]) & mask_.i32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_x_mm256_not_pd(simde__m256d a) { + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = ~a_.i64; + #elif SIMDE_NATURAL_VECTOR_SIZE_GE(128) + r_.m128d[0] = simde_x_mm_not_pd(a_.m128d[0]); + r_.m128d[1] = simde_x_mm_not_pd(a_.m128d[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = ~(a_.i64[i]); + } + #endif + + return simde__m256d_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_x_mm256_select_pd(simde__m256d a, simde__m256d b, simde__m256d mask) { + /* This function is for when you want to blend two elements together + * according to a mask. It is similar to _mm256_blendv_pd, except that + * it is undefined whether the blend is based on the highest bit in + * each lane (like blendv) or just bitwise operations. This allows + * us to implement the function efficiently everywhere. + * + * Basically, you promise that all the lanes in mask are either 0 or + * ~0. */ + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_blendv_pd(a, b, mask); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b), + mask_ = simde__m256d_to_private(mask); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 ^ ((a_.i64 ^ b_.i64) & mask_.i64); + #elif SIMDE_NATURAL_VECTOR_SIZE_GE(128) + r_.m128d[0] = simde_x_mm_select_pd(a_.m128d[0], b_.m128d[0], mask_.m128d[0]); + r_.m128d[1] = simde_x_mm_select_pd(a_.m128d[1], b_.m128d[1], mask_.m128d[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] ^ ((a_.i64[i] ^ b_.i64[i]) & mask_.i64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_setone_si256 (void) { + simde__m256i_private r_; + +#if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + __typeof__(r_.i32f) rv = { 0, }; + r_.i32f = ~rv; +#elif defined(SIMDE_X86_AVX2_NATIVE) + __m256i t = _mm256_setzero_si256(); + r_.n = _mm256_cmpeq_epi32(t, t); +#else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = ~HEDLEY_STATIC_CAST(int_fast32_t, 0); + } +#endif + + return simde__m256i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_x_mm256_setone_ps (void) { + return simde_mm256_castsi256_ps(simde_x_mm256_setone_si256()); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_x_mm256_setone_pd (void) { + return simde_mm256_castsi256_pd(simde_x_mm256_setone_si256()); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_set_epi8 (int8_t e31, int8_t e30, int8_t e29, int8_t e28, + int8_t e27, int8_t e26, int8_t e25, int8_t e24, + int8_t e23, int8_t e22, int8_t e21, int8_t e20, + int8_t e19, int8_t e18, int8_t e17, int8_t e16, + int8_t e15, int8_t e14, int8_t e13, int8_t e12, + int8_t e11, int8_t e10, int8_t e9, int8_t e8, + int8_t e7, int8_t e6, int8_t e5, int8_t e4, + int8_t e3, int8_t e2, int8_t e1, int8_t e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_set_epi8(e31, e30, e29, e28, e27, e26, e25, e24, + e23, e22, e21, e20, e19, e18, e17, e16, + e15, e14, e13, e12, e11, e10, e9, e8, + e7, e6, e5, e4, e3, e2, e1, e0); + #else + simde__m256i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_set_epi8( + e15, e14, e13, e12, e11, e10, e9, e8, + e7, e6, e5, e4, e3, e2, e1, e0); + r_.m128i[1] = simde_mm_set_epi8( + e31, e30, e29, e28, e27, e26, e25, e24, + e23, e22, e21, e20, e19, e18, e17, e16); + #else + r_.i8[ 0] = e0; + r_.i8[ 1] = e1; + r_.i8[ 2] = e2; + r_.i8[ 3] = e3; + r_.i8[ 4] = e4; + r_.i8[ 5] = e5; + r_.i8[ 6] = e6; + r_.i8[ 7] = e7; + r_.i8[ 8] = e8; + r_.i8[ 9] = e9; + r_.i8[10] = e10; + r_.i8[11] = e11; + r_.i8[12] = e12; + r_.i8[13] = e13; + r_.i8[14] = e14; + r_.i8[15] = e15; + r_.i8[16] = e16; + r_.i8[17] = e17; + r_.i8[18] = e18; + r_.i8[19] = e19; + r_.i8[20] = e20; + r_.i8[21] = e21; + r_.i8[22] = e22; + r_.i8[23] = e23; + r_.i8[24] = e24; + r_.i8[25] = e25; + r_.i8[26] = e26; + r_.i8[27] = e27; + r_.i8[28] = e28; + r_.i8[29] = e29; + r_.i8[30] = e30; + r_.i8[31] = e31; + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set_epi8 + #define _mm256_set_epi8(e31, e30, e29, e28, e27, e26, e25, e24, e23, e22, e21, e20, e19, e18, e17, e16, e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) \ + simde_mm256_set_epi8(e31, e30, e29, e28, e27, e26, e25, e24, e23, e22, e21, e20, e19, e18, e17, e16, e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_set_epi16 (int16_t e15, int16_t e14, int16_t e13, int16_t e12, + int16_t e11, int16_t e10, int16_t e9, int16_t e8, + int16_t e7, int16_t e6, int16_t e5, int16_t e4, + int16_t e3, int16_t e2, int16_t e1, int16_t e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_set_epi16(e15, e14, e13, e12, e11, e10, e9, e8, + e7, e6, e5, e4, e3, e2, e1, e0); + #else + simde__m256i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_set_epi16( e7, e6, e5, e4, e3, e2, e1, e0); + r_.m128i[1] = simde_mm_set_epi16(e15, e14, e13, e12, e11, e10, e9, e8); + #else + r_.i16[ 0] = e0; + r_.i16[ 1] = e1; + r_.i16[ 2] = e2; + r_.i16[ 3] = e3; + r_.i16[ 4] = e4; + r_.i16[ 5] = e5; + r_.i16[ 6] = e6; + r_.i16[ 7] = e7; + r_.i16[ 8] = e8; + r_.i16[ 9] = e9; + r_.i16[10] = e10; + r_.i16[11] = e11; + r_.i16[12] = e12; + r_.i16[13] = e13; + r_.i16[14] = e14; + r_.i16[15] = e15; + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set_epi16 + #define _mm256_set_epi16(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) \ + simde_mm256_set_epi16(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_set_epi32 (int32_t e7, int32_t e6, int32_t e5, int32_t e4, + int32_t e3, int32_t e2, int32_t e1, int32_t e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_set_epi32(e7, e6, e5, e4, e3, e2, e1, e0); + #else + simde__m256i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_set_epi32(e3, e2, e1, e0); + r_.m128i[1] = simde_mm_set_epi32(e7, e6, e5, e4); + #else + r_.i32[ 0] = e0; + r_.i32[ 1] = e1; + r_.i32[ 2] = e2; + r_.i32[ 3] = e3; + r_.i32[ 4] = e4; + r_.i32[ 5] = e5; + r_.i32[ 6] = e6; + r_.i32[ 7] = e7; + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set_epi32 + #define _mm256_set_epi32(e7, e6, e5, e4, e3, e2, e1, e0) \ + simde_mm256_set_epi32(e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_set_epi64x (int64_t e3, int64_t e2, int64_t e1, int64_t e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_set_epi64x(e3, e2, e1, e0); + #else + simde__m256i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_set_epi64x(e1, e0); + r_.m128i[1] = simde_mm_set_epi64x(e3, e2); + #else + r_.i64[0] = e0; + r_.i64[1] = e1; + r_.i64[2] = e2; + r_.i64[3] = e3; + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set_epi64x + #define _mm256_set_epi64x(e3, e2, e1, e0) simde_mm256_set_epi64x(e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_set_epu8 (uint8_t e31, uint8_t e30, uint8_t e29, uint8_t e28, + uint8_t e27, uint8_t e26, uint8_t e25, uint8_t e24, + uint8_t e23, uint8_t e22, uint8_t e21, uint8_t e20, + uint8_t e19, uint8_t e18, uint8_t e17, uint8_t e16, + uint8_t e15, uint8_t e14, uint8_t e13, uint8_t e12, + uint8_t e11, uint8_t e10, uint8_t e9, uint8_t e8, + uint8_t e7, uint8_t e6, uint8_t e5, uint8_t e4, + uint8_t e3, uint8_t e2, uint8_t e1, uint8_t e0) { + simde__m256i_private r_; + + r_.u8[ 0] = e0; + r_.u8[ 1] = e1; + r_.u8[ 2] = e2; + r_.u8[ 3] = e3; + r_.u8[ 4] = e4; + r_.u8[ 5] = e5; + r_.u8[ 6] = e6; + r_.u8[ 7] = e7; + r_.u8[ 8] = e8; + r_.u8[ 9] = e9; + r_.u8[10] = e10; + r_.u8[11] = e11; + r_.u8[12] = e12; + r_.u8[13] = e13; + r_.u8[14] = e14; + r_.u8[15] = e15; + r_.u8[16] = e16; + r_.u8[17] = e17; + r_.u8[18] = e18; + r_.u8[19] = e19; + r_.u8[20] = e20; + r_.u8[20] = e20; + r_.u8[21] = e21; + r_.u8[22] = e22; + r_.u8[23] = e23; + r_.u8[24] = e24; + r_.u8[25] = e25; + r_.u8[26] = e26; + r_.u8[27] = e27; + r_.u8[28] = e28; + r_.u8[29] = e29; + r_.u8[30] = e30; + r_.u8[31] = e31; + + return simde__m256i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_set_epu16 (uint16_t e15, uint16_t e14, uint16_t e13, uint16_t e12, + uint16_t e11, uint16_t e10, uint16_t e9, uint16_t e8, + uint16_t e7, uint16_t e6, uint16_t e5, uint16_t e4, + uint16_t e3, uint16_t e2, uint16_t e1, uint16_t e0) { + simde__m256i_private r_; + + r_.u16[ 0] = e0; + r_.u16[ 1] = e1; + r_.u16[ 2] = e2; + r_.u16[ 3] = e3; + r_.u16[ 4] = e4; + r_.u16[ 5] = e5; + r_.u16[ 6] = e6; + r_.u16[ 7] = e7; + r_.u16[ 8] = e8; + r_.u16[ 9] = e9; + r_.u16[10] = e10; + r_.u16[11] = e11; + r_.u16[12] = e12; + r_.u16[13] = e13; + r_.u16[14] = e14; + r_.u16[15] = e15; + + return simde__m256i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_set_epu32 (uint32_t e7, uint32_t e6, uint32_t e5, uint32_t e4, + uint32_t e3, uint32_t e2, uint32_t e1, uint32_t e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_set_epi32(HEDLEY_STATIC_CAST(int32_t, e7), HEDLEY_STATIC_CAST(int32_t, e6), HEDLEY_STATIC_CAST(int32_t, e5), HEDLEY_STATIC_CAST(int32_t, e4), + HEDLEY_STATIC_CAST(int32_t, e3), HEDLEY_STATIC_CAST(int32_t, e2), HEDLEY_STATIC_CAST(int32_t, e1), HEDLEY_STATIC_CAST(int32_t, e0)); + #else + simde__m256i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_set_epi32(HEDLEY_STATIC_CAST(int32_t, e3), HEDLEY_STATIC_CAST(int32_t, e2), HEDLEY_STATIC_CAST(int32_t, e1), HEDLEY_STATIC_CAST(int32_t, e0)); + r_.m128i[1] = simde_mm_set_epi32(HEDLEY_STATIC_CAST(int32_t, e7), HEDLEY_STATIC_CAST(int32_t, e6), HEDLEY_STATIC_CAST(int32_t, e5), HEDLEY_STATIC_CAST(int32_t, e4)); + #else + r_.u32[ 0] = e0; + r_.u32[ 1] = e1; + r_.u32[ 2] = e2; + r_.u32[ 3] = e3; + r_.u32[ 4] = e4; + r_.u32[ 5] = e5; + r_.u32[ 6] = e6; + r_.u32[ 7] = e7; + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_set_epu64x (uint64_t e3, uint64_t e2, uint64_t e1, uint64_t e0) { + simde__m256i_private r_; + + r_.u64[0] = e0; + r_.u64[1] = e1; + r_.u64[2] = e2; + r_.u64[3] = e3; + + return simde__m256i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_set_ps (simde_float32 e7, simde_float32 e6, simde_float32 e5, simde_float32 e4, + simde_float32 e3, simde_float32 e2, simde_float32 e1, simde_float32 e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_set_ps(e7, e6, e5, e4, e3, e2, e1, e0); + #else + simde__m256_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_set_ps(e3, e2, e1, e0); + r_.m128[1] = simde_mm_set_ps(e7, e6, e5, e4); + #else + r_.f32[0] = e0; + r_.f32[1] = e1; + r_.f32[2] = e2; + r_.f32[3] = e3; + r_.f32[4] = e4; + r_.f32[5] = e5; + r_.f32[6] = e6; + r_.f32[7] = e7; + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set_ps + #define _mm256_set_ps(e7, e6, e5, e4, e3, e2, e1, e0) \ + simde_mm256_set_ps(e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_set_pd (simde_float64 e3, simde_float64 e2, simde_float64 e1, simde_float64 e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_set_pd(e3, e2, e1, e0); + #else + simde__m256d_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_set_pd(e1, e0); + r_.m128d[1] = simde_mm_set_pd(e3, e2); + #else + r_.f64[0] = e0; + r_.f64[1] = e1; + r_.f64[2] = e2; + r_.f64[3] = e3; + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set_pd + #define _mm256_set_pd(e3, e2, e1, e0) \ + simde_mm256_set_pd(e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_set_m128 (simde__m128 e1, simde__m128 e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_insertf128_ps(_mm256_castps128_ps256(e0), e1, 1); + #else + simde__m256_private r_; + simde__m128_private + e1_ = simde__m128_to_private(e1), + e0_ = simde__m128_to_private(e0); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128_private[0] = e0_; + r_.m128_private[1] = e1_; + #elif defined(SIMDE_HAVE_INT128_) + r_.i128[0] = e0_.i128[0]; + r_.i128[1] = e1_.i128[0]; + #else + r_.i64[0] = e0_.i64[0]; + r_.i64[1] = e0_.i64[1]; + r_.i64[2] = e1_.i64[0]; + r_.i64[3] = e1_.i64[1]; + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set_m128 + #define _mm256_set_m128(e1, e0) simde_mm256_set_m128(e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_set_m128d (simde__m128d e1, simde__m128d e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_insertf128_pd(_mm256_castpd128_pd256(e0), e1, 1); + #else + simde__m256d_private r_; + simde__m128d_private + e1_ = simde__m128d_to_private(e1), + e0_ = simde__m128d_to_private(e0); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d_private[0] = e0_; + r_.m128d_private[1] = e1_; + #else + r_.i64[0] = e0_.i64[0]; + r_.i64[1] = e0_.i64[1]; + r_.i64[2] = e1_.i64[0]; + r_.i64[3] = e1_.i64[1]; + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set_m128d + #define _mm256_set_m128d(e1, e0) simde_mm256_set_m128d(e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_set_m128i (simde__m128i e1, simde__m128i e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_insertf128_si256(_mm256_castsi128_si256(e0), e1, 1); + #else + simde__m256i_private r_; + simde__m128i_private + e1_ = simde__m128i_to_private(e1), + e0_ = simde__m128i_to_private(e0); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i_private[0] = e0_; + r_.m128i_private[1] = e1_; + #else + r_.i64[0] = e0_.i64[0]; + r_.i64[1] = e0_.i64[1]; + r_.i64[2] = e1_.i64[0]; + r_.i64[3] = e1_.i64[1]; + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set_m128i + #define _mm256_set_m128i(e1, e0) simde_mm256_set_m128i(e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_set1_epi8 (int8_t a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_set1_epi8(a); + #else + simde__m256i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_set1_epi8(a); + r_.m128i[1] = simde_mm_set1_epi8(a); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set1_epi8 + #define _mm256_set1_epi8(a) simde_mm256_set1_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_set1_epi16 (int16_t a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_set1_epi16(a); + #else + simde__m256i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_set1_epi16(a); + r_.m128i[1] = simde_mm_set1_epi16(a); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set1_epi16 + #define _mm256_set1_epi16(a) simde_mm256_set1_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_set1_epi32 (int32_t a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_set1_epi32(a); + #else + simde__m256i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_set1_epi32(a); + r_.m128i[1] = simde_mm_set1_epi32(a); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set1_epi32 + #define _mm256_set1_epi32(a) simde_mm256_set1_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_set1_epi64x (int64_t a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_set1_epi64x(a); + #else + simde__m256i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_set1_epi64x(a); + r_.m128i[1] = simde_mm_set1_epi64x(a); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set1_epi64x + #define _mm256_set1_epi64x(a) simde_mm256_set1_epi64x(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_set1_ps (simde_float32 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_set1_ps(a); + #else + simde__m256_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_set1_ps(a); + r_.m128[1] = simde_mm_set1_ps(a); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set1_ps + #define _mm256_set1_ps(a) simde_mm256_set1_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_set1_pd (simde_float64 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_set1_pd(a); + #else + simde__m256d_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_set1_pd(a); + r_.m128d[1] = simde_mm_set1_pd(a); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_set1_pd + #define _mm256_set1_pd(a) simde_mm256_set1_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_deinterleaveeven_epi16 (simde__m256i a, simde__m256i b) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_x_mm_deinterleaveeven_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_x_mm_deinterleaveeven_epi16(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i16 = SIMDE_SHUFFLE_VECTOR_(16, 32, a_.i16, b_.i16, 0, 2, 4, 6, 16, 18, 20, 22, 8, 10, 12, 14, 24, 26, 28, 30); + #else + const size_t halfway_point = (sizeof(r_.i16) / sizeof(r_.i16[0])) / 2; + const size_t quarter_point = (sizeof(r_.i16) / sizeof(r_.i16[0])) / 4; + for (size_t i = 0 ; i < quarter_point ; i++) { + r_.i16[i] = a_.i16[2 * i]; + r_.i16[i + quarter_point] = b_.i16[2 * i]; + r_.i16[halfway_point + i] = a_.i16[halfway_point + 2 * i]; + r_.i16[halfway_point + i + quarter_point] = b_.i16[halfway_point + 2 * i]; + } + #endif + + return simde__m256i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_deinterleaveodd_epi16 (simde__m256i a, simde__m256i b) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_x_mm_deinterleaveodd_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_x_mm_deinterleaveodd_epi16(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i16 = SIMDE_SHUFFLE_VECTOR_(16, 32, a_.i16, b_.i16, 1, 3, 5, 7, 17, 19, 21, 23, 9, 11, 13, 15, 25, 27, 29, 31); + #else + const size_t halfway_point = (sizeof(r_.i16) / sizeof(r_.i16[0])) / 2; + const size_t quarter_point = (sizeof(r_.i16) / sizeof(r_.i16[0])) / 4; + for (size_t i = 0 ; i < quarter_point ; i++) { + r_.i16[i] = a_.i16[2 * i + 1]; + r_.i16[i + quarter_point] = b_.i16[2 * i + 1]; + r_.i16[halfway_point + i] = a_.i16[halfway_point + 2 * i + 1]; + r_.i16[halfway_point + i + quarter_point] = b_.i16[halfway_point + 2 * i + 1]; + } + #endif + + return simde__m256i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_deinterleaveeven_epi32 (simde__m256i a, simde__m256i b) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_x_mm_deinterleaveeven_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_x_mm_deinterleaveeven_epi32(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i32 = SIMDE_SHUFFLE_VECTOR_(32, 32, a_.i32, b_.i32, 0, 2, 8, 10, 4, 6, 12, 14); + #else + const size_t halfway_point = (sizeof(r_.i32) / sizeof(r_.i32[0])) / 2; + const size_t quarter_point = (sizeof(r_.i32) / sizeof(r_.i32[0])) / 4; + for (size_t i = 0 ; i < quarter_point ; i++) { + r_.i32[i] = a_.i32[2 * i]; + r_.i32[i + quarter_point] = b_.i32[2 * i]; + r_.i32[halfway_point + i] = a_.i32[halfway_point + 2 * i]; + r_.i32[halfway_point + i + quarter_point] = b_.i32[halfway_point + 2 * i]; + } + #endif + + return simde__m256i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_deinterleaveodd_epi32 (simde__m256i a, simde__m256i b) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_x_mm_deinterleaveodd_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_x_mm_deinterleaveodd_epi32(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i32 = SIMDE_SHUFFLE_VECTOR_(32, 32, a_.i32, b_.i32, 1, 3, 9, 11, 5, 7, 13, 15); + #else + const size_t halfway_point = (sizeof(r_.i32) / sizeof(r_.i32[0])) / 2; + const size_t quarter_point = (sizeof(r_.i32) / sizeof(r_.i32[0])) / 4; + for (size_t i = 0 ; i < quarter_point ; i++) { + r_.i32[i] = a_.i32[2 * i + 1]; + r_.i32[i + quarter_point] = b_.i32[2 * i + 1]; + r_.i32[halfway_point + i] = a_.i32[halfway_point + 2 * i + 1]; + r_.i32[halfway_point + i + quarter_point] = b_.i32[halfway_point + 2 * i + 1]; + } + #endif + + return simde__m256i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_x_mm256_deinterleaveeven_ps (simde__m256 a, simde__m256 b) { + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_x_mm_deinterleaveeven_ps(a_.m128[0], b_.m128[0]); + r_.m128[1] = simde_x_mm_deinterleaveeven_ps(a_.m128[1], b_.m128[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 32, a_.f32, b_.f32, 0, 2, 8, 10, 4, 6, 12, 14); + #else + const size_t halfway_point = (sizeof(r_.f32) / sizeof(r_.f32[0])) / 2; + const size_t quarter_point = (sizeof(r_.f32) / sizeof(r_.f32[0])) / 4; + for (size_t i = 0 ; i < quarter_point ; i++) { + r_.f32[i] = a_.f32[2 * i]; + r_.f32[i + quarter_point] = b_.f32[2 * i]; + r_.f32[halfway_point + i] = a_.f32[halfway_point + 2 * i]; + r_.f32[halfway_point + i + quarter_point] = b_.f32[halfway_point + 2 * i]; + } + #endif + + return simde__m256_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_x_mm256_deinterleaveodd_ps (simde__m256 a, simde__m256 b) { + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_x_mm_deinterleaveodd_ps(a_.m128[0], b_.m128[0]); + r_.m128[1] = simde_x_mm_deinterleaveodd_ps(a_.m128[1], b_.m128[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 32, a_.f32, b_.f32, 1, 3, 9, 11, 5, 7, 13, 15); + #else + const size_t halfway_point = (sizeof(r_.f32) / sizeof(r_.f32[0])) / 2; + const size_t quarter_point = (sizeof(r_.f32) / sizeof(r_.f32[0])) / 4; + for (size_t i = 0 ; i < quarter_point ; i++) { + r_.f32[i] = a_.f32[2 * i + 1]; + r_.f32[i + quarter_point] = b_.f32[2 * i + 1]; + r_.f32[halfway_point + i] = a_.f32[halfway_point + 2 * i + 1]; + r_.f32[halfway_point + i + quarter_point] = b_.f32[halfway_point + 2 * i + 1]; + } + #endif + + return simde__m256_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_x_mm256_deinterleaveeven_pd (simde__m256d a, simde__m256d b) { + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_x_mm_deinterleaveeven_pd(a_.m128d[0], b_.m128d[0]); + r_.m128d[1] = simde_x_mm_deinterleaveeven_pd(a_.m128d[1], b_.m128d[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 32, a_.f64, b_.f64, 0, 4, 2, 6); + #else + const size_t halfway_point = (sizeof(r_.f64) / sizeof(r_.f64[0])) / 2; + const size_t quarter_point = (sizeof(r_.f64) / sizeof(r_.f64[0])) / 4; + for (size_t i = 0 ; i < quarter_point ; i++) { + r_.f64[i] = a_.f64[2 * i]; + r_.f64[i + quarter_point] = b_.f64[2 * i]; + r_.f64[halfway_point + i] = a_.f64[halfway_point + 2 * i]; + r_.f64[halfway_point + i + quarter_point] = b_.f64[halfway_point + 2 * i]; + } + #endif + + return simde__m256d_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_x_mm256_deinterleaveodd_pd (simde__m256d a, simde__m256d b) { + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_x_mm_deinterleaveodd_pd(a_.m128d[0], b_.m128d[0]); + r_.m128d[1] = simde_x_mm_deinterleaveodd_pd(a_.m128d[1], b_.m128d[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 32, a_.f64, b_.f64, 1, 5, 3, 7); + #else + const size_t halfway_point = (sizeof(r_.f64) / sizeof(r_.f64[0])) / 2; + const size_t quarter_point = (sizeof(r_.f64) / sizeof(r_.f64[0])) / 4; + for (size_t i = 0 ; i < quarter_point ; i++) { + r_.f64[i] = a_.f64[2 * i + 1]; + r_.f64[i + quarter_point] = b_.f64[2 * i + 1]; + r_.f64[halfway_point + i] = a_.f64[halfway_point + 2 * i + 1]; + r_.f64[halfway_point + i + quarter_point] = b_.f64[halfway_point + 2 * i + 1]; + } + #endif + + return simde__m256d_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_x_mm256_abs_ps(simde__m256 a) { + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_fabsf(a_.f32[i]); + } + return simde__m256_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_x_mm256_abs_pd(simde__m256d a) { + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_fabs(a_.f64[i]); + } + return simde__m256d_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_add_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_add_ps(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_add_ps(a_.m128[0], b_.m128[0]); + r_.m128[1] = simde_mm_add_ps(a_.m128[1], b_.m128[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = a_.f32 + b_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[i] + b_.f32[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_add_ps + #define _mm256_add_ps(a, b) simde_mm256_add_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_hadd_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_hadd_ps(a, b); + #else + return simde_mm256_add_ps(simde_x_mm256_deinterleaveeven_ps(a, b), simde_x_mm256_deinterleaveodd_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_hadd_ps + #define _mm256_hadd_ps(a, b) simde_mm256_hadd_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_add_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_add_pd(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_add_pd(a_.m128d[0], b_.m128d[0]); + r_.m128d[1] = simde_mm_add_pd(a_.m128d[1], b_.m128d[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = a_.f64 + b_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[i] + b_.f64[i]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_add_pd + #define _mm256_add_pd(a, b) simde_mm256_add_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_hadd_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_hadd_pd(a, b); + #else + return simde_mm256_add_pd(simde_x_mm256_deinterleaveeven_pd(a, b), simde_x_mm256_deinterleaveodd_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_hadd_pd + #define _mm256_hadd_pd(a, b) simde_mm256_hadd_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_addsub_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_addsub_ps(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_addsub_ps(a_.m128[0], b_.m128[0]); + r_.m128[1] = simde_mm_addsub_ps(a_.m128[1], b_.m128[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i += 2) { + r_.f32[ i ] = a_.f32[ i ] - b_.f32[ i ]; + r_.f32[i + 1] = a_.f32[i + 1] + b_.f32[i + 1]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_addsub_ps + #define _mm256_addsub_ps(a, b) simde_mm256_addsub_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_addsub_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_addsub_pd(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_addsub_pd(a_.m128d[0], b_.m128d[0]); + r_.m128d[1] = simde_mm_addsub_pd(a_.m128d[1], b_.m128d[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i += 2) { + r_.f64[ i ] = a_.f64[ i ] - b_.f64[ i ]; + r_.f64[i + 1] = a_.f64[i + 1] + b_.f64[i + 1]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_addsub_pd + #define _mm256_addsub_pd(a, b) simde_mm256_addsub_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_and_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_and_ps(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_and_ps(a_.m128[0], b_.m128[0]); + r_.m128[1] = simde_mm_and_ps(a_.m128[1], b_.m128[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f & b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] & b_.i32f[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_and_ps + #define _mm256_and_ps(a, b) simde_mm256_and_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_and_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_and_pd(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_and_pd(a_.m128d[0], b_.m128d[0]); + r_.m128d[1] = simde_mm_and_pd(a_.m128d[1], b_.m128d[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f & b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] & b_.i32f[i]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_and_pd + #define _mm256_and_pd(a, b) simde_mm256_and_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_andnot_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_andnot_ps(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_andnot_ps(a_.m128[0], b_.m128[0]); + r_.m128[1] = simde_mm_andnot_ps(a_.m128[1], b_.m128[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = ~a_.i32f & b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = ~a_.i32f[i] & b_.i32f[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_andnot_ps + #define _mm256_andnot_ps(a, b) simde_mm256_andnot_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_andnot_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_andnot_pd(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_andnot_pd(a_.m128d[0], b_.m128d[0]); + r_.m128d[1] = simde_mm_andnot_pd(a_.m128d[1], b_.m128d[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = ~a_.i32f & b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = ~a_.i32f[i] & b_.i32f[i]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_andnot_pd + #define _mm256_andnot_pd(a, b) simde_mm256_andnot_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_blend_ps (simde__m256 a, simde__m256 b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = ((imm8 >> i) & 1) ? b_.f32[i] : a_.f32[i]; + } + + return simde__m256_from_private(r_); +} +#if defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm256_blend_ps(a, b, imm8) _mm256_blend_ps(a, b, imm8) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) +# define simde_mm256_blend_ps(a, b, imm8) \ + simde_mm256_set_m128( \ + simde_mm_blend_ps(simde_mm256_extractf128_ps(a, 1), simde_mm256_extractf128_ps(b, 1), (imm8) >> 4), \ + simde_mm_blend_ps(simde_mm256_extractf128_ps(a, 0), simde_mm256_extractf128_ps(b, 0), (imm8) & 0x0F)) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_blend_ps + #define _mm256_blend_ps(a, b, imm8) simde_mm256_blend_ps(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_blend_pd (simde__m256d a, simde__m256d b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = ((imm8 >> i) & 1) ? b_.f64[i] : a_.f64[i]; + } + return simde__m256d_from_private(r_); +} +#if defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm256_blend_pd(a, b, imm8) _mm256_blend_pd(a, b, imm8) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) +# define simde_mm256_blend_pd(a, b, imm8) \ + simde_mm256_set_m128d( \ + simde_mm_blend_pd(simde_mm256_extractf128_pd(a, 1), simde_mm256_extractf128_pd(b, 1), (imm8) >> 2), \ + simde_mm_blend_pd(simde_mm256_extractf128_pd(a, 0), simde_mm256_extractf128_pd(b, 0), (imm8) & 3)) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_blend_pd + #define _mm256_blend_pd(a, b, imm8) simde_mm256_blend_pd(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_blendv_ps (simde__m256 a, simde__m256 b, simde__m256 mask) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_blendv_ps(a, b, mask); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b), + mask_ = simde__m256_to_private(mask); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_blendv_ps(a_.m128[0], b_.m128[0], mask_.m128[0]); + r_.m128[1] = simde_mm_blendv_ps(a_.m128[1], b_.m128[1], mask_.m128[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.f32[i] = (mask_.u32[i] & (UINT32_C(1) << 31)) ? b_.f32[i] : a_.f32[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_blendv_ps + #define _mm256_blendv_ps(a, b, imm8) simde_mm256_blendv_ps(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_blendv_pd (simde__m256d a, simde__m256d b, simde__m256d mask) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_blendv_pd(a, b, mask); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b), + mask_ = simde__m256d_to_private(mask); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_blendv_pd(a_.m128d[0], b_.m128d[0], mask_.m128d[0]); + r_.m128d[1] = simde_mm_blendv_pd(a_.m128d[1], b_.m128d[1], mask_.m128d[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.f64[i] = (mask_.u64[i] & (UINT64_C(1) << 63)) ? b_.f64[i] : a_.f64[i]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_blendv_pd + #define _mm256_blendv_pd(a, b, imm8) simde_mm256_blendv_pd(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_broadcast_pd (simde__m128d const * mem_addr) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_broadcast_pd(mem_addr); + #else + simde__m256d_private r_; + + simde__m128d tmp = simde_mm_loadu_pd(HEDLEY_REINTERPRET_CAST(simde_float64 const*, mem_addr)); + r_.m128d[0] = tmp; + r_.m128d[1] = tmp; + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_broadcast_pd + #define _mm256_broadcast_pd(mem_addr) simde_mm256_broadcast_pd(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_broadcast_ps (simde__m128 const * mem_addr) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_broadcast_ps(mem_addr); + #else + simde__m256_private r_; + + simde__m128 tmp = simde_mm_loadu_ps(HEDLEY_REINTERPRET_CAST(simde_float32 const*, mem_addr)); + r_.m128[0] = tmp; + r_.m128[1] = tmp; + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_broadcast_ps + #define _mm256_broadcast_ps(mem_addr) simde_mm256_broadcast_ps(HEDLEY_REINTERPRET_CAST(simde__m128 const*, mem_addr)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_broadcast_sd (simde_float64 const * a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_broadcast_sd(a); + #else + return simde_mm256_set1_pd(*a); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_broadcast_sd + #define _mm256_broadcast_sd(mem_addr) simde_mm256_broadcast_sd(HEDLEY_REINTERPRET_CAST(double const*, mem_addr)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_broadcast_ss (simde_float32 const * a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm_broadcast_ss(a); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return simde__m128_from_wasm_v128(wasm_v128_load32_splat(a)); + #else + return simde_mm_set1_ps(*a); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_broadcast_ss + #define _mm_broadcast_ss(mem_addr) simde_mm_broadcast_ss(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_broadcast_ss (simde_float32 const * a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_broadcast_ss(a); + #else + return simde_mm256_set1_ps(*a); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_broadcast_ss + #define _mm256_broadcast_ss(mem_addr) simde_mm256_broadcast_ss(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_castpd128_pd256 (simde__m128d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_castpd128_pd256(a); + #else + simde__m256d_private r_; + simde__m128d_private a_ = simde__m128d_to_private(a); + + r_.m128d_private[0] = a_; + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_castpd128_pd256 + #define _mm256_castpd128_pd256(a) simde_mm256_castpd128_pd256(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm256_castpd256_pd128 (simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_castpd256_pd128(a); + #else + simde__m256d_private a_ = simde__m256d_to_private(a); + return a_.m128d[0]; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_castpd256_pd128 + #define _mm256_castpd256_pd128(a) simde_mm256_castpd256_pd128(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_castps128_ps256 (simde__m128 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_castps128_ps256(a); + #else + simde__m256_private r_; + simde__m128_private a_ = simde__m128_to_private(a); + + r_.m128_private[0] = a_; + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_castps128_ps256 + #define _mm256_castps128_ps256(a) simde_mm256_castps128_ps256(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm256_castps256_ps128 (simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_castps256_ps128(a); + #else + simde__m256_private a_ = simde__m256_to_private(a); + return a_.m128[0]; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_castps256_ps128 + #define _mm256_castps256_ps128(a) simde_mm256_castps256_ps128(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_castsi128_si256 (simde__m128i a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_castsi128_si256(a); + #else + simde__m256i_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + r_.m128i_private[0] = a_; + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_castsi128_si256 + #define _mm256_castsi128_si256(a) simde_mm256_castsi128_si256(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm256_castsi256_si128 (simde__m256i a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_castsi256_si128(a); + #else + simde__m256i_private a_ = simde__m256i_to_private(a); + return a_.m128i[0]; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_castsi256_si128 + #define _mm256_castsi256_si128(a) simde_mm256_castsi256_si128(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_round_ps (simde__m256 a, const int rounding) { + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + switch (rounding & ~SIMDE_MM_FROUND_NO_EXC) { + #if defined(simde_math_nearbyintf) + case SIMDE_MM_FROUND_CUR_DIRECTION: + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_nearbyintf(a_.f32[i]); + } + break; + #endif + + #if defined(simde_math_roundf) + case SIMDE_MM_FROUND_TO_NEAREST_INT: + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_roundf(a_.f32[i]); + } + break; + #endif + + #if defined(simde_math_floorf) + case SIMDE_MM_FROUND_TO_NEG_INF: + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_floorf(a_.f32[i]); + } + break; + #endif + + #if defined(simde_math_ceilf) + case SIMDE_MM_FROUND_TO_POS_INF: + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_ceilf(a_.f32[i]); + } + break; + #endif + + #if defined(simde_math_truncf) + case SIMDE_MM_FROUND_TO_ZERO: + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_truncf(a_.f32[i]); + } + break; + #endif + + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm256_undefined_ps()); + } + + return simde__m256_from_private(r_); +} +#if defined(SIMDE_X86_AVX_NATIVE) + #define simde_mm256_round_ps(a, rounding) _mm256_round_ps(a, rounding) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm256_round_ps(a, rounding) SIMDE_STATEMENT_EXPR_(({ \ + simde__m256_private \ + simde_mm256_round_ps_r_ = simde__m256_to_private(simde_mm256_setzero_ps()), \ + simde_mm256_round_ps_a_ = simde__m256_to_private(a); \ + \ + for (size_t simde_mm256_round_ps_i = 0 ; simde_mm256_round_ps_i < (sizeof(simde_mm256_round_ps_r_.m128) / sizeof(simde_mm256_round_ps_r_.m128[0])) ; simde_mm256_round_ps_i++) { \ + simde_mm256_round_ps_r_.m128[simde_mm256_round_ps_i] = simde_mm_round_ps(simde_mm256_round_ps_a_.m128[simde_mm256_round_ps_i], rounding); \ + } \ + \ + simde__m256_from_private(simde_mm256_round_ps_r_); \ + })) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_round_ps + #define _mm256_round_ps(a, rounding) simde_mm256_round_ps(a, rounding) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_round_pd (simde__m256d a, const int rounding) { + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + switch (rounding & ~SIMDE_MM_FROUND_NO_EXC) { + #if defined(simde_math_nearbyint) + case SIMDE_MM_FROUND_CUR_DIRECTION: + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_nearbyint(a_.f64[i]); + } + break; + #endif + + #if defined(simde_math_round) + case SIMDE_MM_FROUND_TO_NEAREST_INT: + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_round(a_.f64[i]); + } + break; + #endif + + #if defined(simde_math_floor) + case SIMDE_MM_FROUND_TO_NEG_INF: + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_floor(a_.f64[i]); + } + break; + #endif + + #if defined(simde_math_ceil) + case SIMDE_MM_FROUND_TO_POS_INF: + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_ceil(a_.f64[i]); + } + break; + #endif + + #if defined(simde_math_trunc) + case SIMDE_MM_FROUND_TO_ZERO: + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_trunc(a_.f64[i]); + } + break; + #endif + + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm256_undefined_pd()); + } + + return simde__m256d_from_private(r_); +} +#if defined(SIMDE_X86_AVX_NATIVE) + #define simde_mm256_round_pd(a, rounding) _mm256_round_pd(a, rounding) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm256_round_pd(a, rounding) SIMDE_STATEMENT_EXPR_(({ \ + simde__m256d_private \ + simde_mm256_round_pd_r_ = simde__m256d_to_private(simde_mm256_setzero_pd()), \ + simde_mm256_round_pd_a_ = simde__m256d_to_private(a); \ + \ + for (size_t simde_mm256_round_pd_i = 0 ; simde_mm256_round_pd_i < (sizeof(simde_mm256_round_pd_r_.m128d) / sizeof(simde_mm256_round_pd_r_.m128d[0])) ; simde_mm256_round_pd_i++) { \ + simde_mm256_round_pd_r_.m128d[simde_mm256_round_pd_i] = simde_mm_round_pd(simde_mm256_round_pd_a_.m128d[simde_mm256_round_pd_i], rounding); \ + } \ + \ + simde__m256d_from_private(simde_mm256_round_pd_r_); \ + })) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_round_pd + #define _mm256_round_pd(a, rounding) simde_mm256_round_pd(a, rounding) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_ceil_pd (simde__m256d a) { + return simde_mm256_round_pd(a, SIMDE_MM_FROUND_TO_POS_INF); +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_ceil_pd + #define _mm256_ceil_pd(a) simde_mm256_ceil_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_ceil_ps (simde__m256 a) { + return simde_mm256_round_ps(a, SIMDE_MM_FROUND_TO_POS_INF); +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_ceil_ps + #define _mm256_ceil_ps(a) simde_mm256_ceil_ps(a) +#endif + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DIAGNOSTIC_DISABLE_FLOAT_EQUAL + +/* This implementation does not support signaling NaNs (yet?) */ +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmp_pd (simde__m128d a, simde__m128d b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 31) { + switch (imm8) { + case SIMDE_CMP_EQ_UQ: + case SIMDE_CMP_EQ_US: + return simde_mm_or_pd(simde_mm_cmpunord_pd(a, b), simde_mm_cmpeq_pd(a, b)); + break; + case SIMDE_CMP_EQ_OQ: + case SIMDE_CMP_EQ_OS: + return simde_mm_cmpeq_pd(a, b); + break; + case SIMDE_CMP_NGE_US: + case SIMDE_CMP_NGE_UQ: + return simde_x_mm_not_pd(simde_mm_cmpge_pd(a, b)); + break; + case SIMDE_CMP_LT_OS: + case SIMDE_CMP_LT_OQ: + return simde_mm_cmplt_pd(a, b); + break; + case SIMDE_CMP_NGT_US: + case SIMDE_CMP_NGT_UQ: + return simde_x_mm_not_pd(simde_mm_cmpgt_pd(a, b)); + break; + case SIMDE_CMP_LE_OS: + case SIMDE_CMP_LE_OQ: + return simde_mm_cmple_pd(a, b); + break; + case SIMDE_CMP_NEQ_UQ: + case SIMDE_CMP_NEQ_US: + return simde_mm_cmpneq_pd(a, b); + break; + case SIMDE_CMP_NEQ_OQ: + case SIMDE_CMP_NEQ_OS: + return simde_mm_and_pd(simde_mm_cmpord_pd(a, b), simde_mm_cmpneq_pd(a, b)); + break; + case SIMDE_CMP_NLT_US: + case SIMDE_CMP_NLT_UQ: + return simde_x_mm_not_pd(simde_mm_cmplt_pd(a, b)); + break; + case SIMDE_CMP_GE_OS: + case SIMDE_CMP_GE_OQ: + return simde_mm_cmpge_pd(a, b); + break; + case SIMDE_CMP_NLE_US: + case SIMDE_CMP_NLE_UQ: + return simde_x_mm_not_pd(simde_mm_cmple_pd(a, b)); + break; + case SIMDE_CMP_GT_OS: + case SIMDE_CMP_GT_OQ: + return simde_mm_cmpgt_pd(a, b); + break; + case SIMDE_CMP_FALSE_OQ: + case SIMDE_CMP_FALSE_OS: + return simde_mm_setzero_pd(); + break; + case SIMDE_CMP_TRUE_UQ: + case SIMDE_CMP_TRUE_US: + return simde_x_mm_setone_pd(); + break; + case SIMDE_CMP_UNORD_Q: + case SIMDE_CMP_UNORD_S: + return simde_mm_cmpunord_pd(a, b); + break; + case SIMDE_CMP_ORD_Q: + case SIMDE_CMP_ORD_S: + return simde_mm_cmpord_pd(a, b); + break; + } + + HEDLEY_UNREACHABLE_RETURN(simde_mm_setzero_pd()); +} +#if defined(__clang__) && defined(__AVX512DQ__) + #define simde_mm_cmp_pd(a, b, imm8) (__extension__ ({ \ + simde__m128d simde_mm_cmp_pd_r; \ + switch (imm8) { \ + case SIMDE_CMP_FALSE_OQ: \ + case SIMDE_CMP_FALSE_OS: \ + simde_mm_cmp_pd_r = simde_mm_setzero_pd(); \ + break; \ + case SIMDE_CMP_TRUE_UQ: \ + case SIMDE_CMP_TRUE_US: \ + simde_mm_cmp_pd_r = simde_x_mm_setone_pd(); \ + break; \ + default: \ + simde_mm_cmp_pd_r = simde_mm_cmp_pd(a, b, imm8); \ + break; \ + } \ + simde_mm_cmp_pd_r; \ + })) +#elif defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm_cmp_pd(a, b, imm8) _mm_cmp_pd(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_cmp_pd + #define _mm_cmp_pd(a, b, imm8) simde_mm_cmp_pd(a, b, imm8) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmp_ps (simde__m128 a, simde__m128 b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 31) { + switch (imm8) { + case SIMDE_CMP_EQ_UQ: + case SIMDE_CMP_EQ_US: + return simde_mm_or_ps(simde_mm_cmpunord_ps(a, b), simde_mm_cmpeq_ps(a, b)); + break; + case SIMDE_CMP_EQ_OQ: + case SIMDE_CMP_EQ_OS: + return simde_mm_cmpeq_ps(a, b); + break; + case SIMDE_CMP_NGE_US: + case SIMDE_CMP_NGE_UQ: + return simde_x_mm_not_ps(simde_mm_cmpge_ps(a, b)); + break; + case SIMDE_CMP_LT_OS: + case SIMDE_CMP_LT_OQ: + return simde_mm_cmplt_ps(a, b); + break; + case SIMDE_CMP_NGT_US: + case SIMDE_CMP_NGT_UQ: + return simde_x_mm_not_ps(simde_mm_cmpgt_ps(a, b)); + break; + case SIMDE_CMP_LE_OS: + case SIMDE_CMP_LE_OQ: + return simde_mm_cmple_ps(a, b); + break; + case SIMDE_CMP_NEQ_UQ: + case SIMDE_CMP_NEQ_US: + return simde_mm_cmpneq_ps(a, b); + break; + case SIMDE_CMP_NEQ_OQ: + case SIMDE_CMP_NEQ_OS: + return simde_mm_and_ps(simde_mm_cmpord_ps(a, b), simde_mm_cmpneq_ps(a, b)); + break; + case SIMDE_CMP_NLT_US: + case SIMDE_CMP_NLT_UQ: + return simde_x_mm_not_ps(simde_mm_cmplt_ps(a, b)); + break; + case SIMDE_CMP_GE_OS: + case SIMDE_CMP_GE_OQ: + return simde_mm_cmpge_ps(a, b); + break; + case SIMDE_CMP_NLE_US: + case SIMDE_CMP_NLE_UQ: + return simde_x_mm_not_ps(simde_mm_cmple_ps(a, b)); + break; + case SIMDE_CMP_GT_OS: + case SIMDE_CMP_GT_OQ: + return simde_mm_cmpgt_ps(a, b); + break; + case SIMDE_CMP_FALSE_OQ: + case SIMDE_CMP_FALSE_OS: + return simde_mm_setzero_ps(); + break; + case SIMDE_CMP_TRUE_UQ: + case SIMDE_CMP_TRUE_US: + return simde_x_mm_setone_ps(); + break; + case SIMDE_CMP_UNORD_Q: + case SIMDE_CMP_UNORD_S: + return simde_mm_cmpunord_ps(a, b); + break; + case SIMDE_CMP_ORD_Q: + case SIMDE_CMP_ORD_S: + return simde_mm_cmpord_ps(a, b); + break; + } + + HEDLEY_UNREACHABLE_RETURN(simde_mm_setzero_ps()); +} +/* Prior to 9.0 clang has problems with _mm{,256}_cmp_{ps,pd} for all four of the true/false + * comparisons, but only when AVX-512 is enabled. */ +#if defined(__clang__) && defined(__AVX512DQ__) + #define simde_mm_cmp_ps(a, b, imm8) (__extension__ ({ \ + simde__m128 simde_mm_cmp_ps_r; \ + switch (imm8) { \ + case SIMDE_CMP_FALSE_OQ: \ + case SIMDE_CMP_FALSE_OS: \ + simde_mm_cmp_ps_r = simde_mm_setzero_ps(); \ + break; \ + case SIMDE_CMP_TRUE_UQ: \ + case SIMDE_CMP_TRUE_US: \ + simde_mm_cmp_ps_r = simde_x_mm_setone_ps(); \ + break; \ + default: \ + simde_mm_cmp_ps_r = simde_mm_cmp_ps(a, b, imm8); \ + break; \ + } \ + simde_mm_cmp_ps_r; \ + })) +#elif defined(SIMDE_X86_AVX_NATIVE) + #define simde_mm_cmp_ps(a, b, imm8) _mm_cmp_ps(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_cmp_ps + #define _mm_cmp_ps(a, b, imm8) simde_mm_cmp_ps(a, b, imm8) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmp_sd (simde__m128d a, simde__m128d b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 31) { + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + switch (imm8) { + case SIMDE_CMP_EQ_OQ: + case SIMDE_CMP_EQ_OS: + a_.i64[0] = (a_.f64[0] == b_.f64[0]) ? ~INT64_C(0) : INT64_C(0); + break; + + case SIMDE_CMP_LT_OQ: + case SIMDE_CMP_LT_OS: + a_.i64[0] = (a_.f64[0] < b_.f64[0]) ? ~INT64_C(0) : INT64_C(0); + break; + + case SIMDE_CMP_LE_OQ: + case SIMDE_CMP_LE_OS: + a_.i64[0] = (a_.f64[0] <= b_.f64[0]) ? ~INT64_C(0) : INT64_C(0); + break; + + case SIMDE_CMP_UNORD_Q: + case SIMDE_CMP_UNORD_S: + a_.i64[0] = ((a_.f64[0] != a_.f64[0]) || (b_.f64[0] != b_.f64[0])) ? ~INT64_C(0) : INT64_C(0); + break; + + case SIMDE_CMP_NEQ_UQ: + case SIMDE_CMP_NEQ_US: + a_.i64[0] = ((a_.f64[0] == a_.f64[0]) & (b_.f64[0] == b_.f64[0]) & (a_.f64[0] != b_.f64[0])) ? ~INT64_C(0) : INT64_C(0); + break; + + case SIMDE_CMP_NEQ_OQ: + case SIMDE_CMP_NEQ_OS: + a_.i64[0] = ((a_.f64[0] == a_.f64[0]) & (b_.f64[0] == b_.f64[0]) & (a_.f64[0] != b_.f64[0])) ? ~INT64_C(0) : INT64_C(0); + break; + + case SIMDE_CMP_NLT_UQ: + case SIMDE_CMP_NLT_US: + a_.i64[0] = !(a_.f64[0] < b_.f64[0]) ? ~INT64_C(0) : INT64_C(0); + break; + + case SIMDE_CMP_NLE_UQ: + case SIMDE_CMP_NLE_US: + a_.i64[0] = !(a_.f64[0] <= b_.f64[0]) ? ~INT64_C(0) : INT64_C(0); + break; + + case SIMDE_CMP_ORD_Q: + case SIMDE_CMP_ORD_S: + a_.i64[0] = ((a_.f64[0] == a_.f64[0]) & (b_.f64[0] == b_.f64[0])) ? ~INT64_C(0) : INT64_C(0); + break; + + case SIMDE_CMP_EQ_UQ: + case SIMDE_CMP_EQ_US: + a_.i64[0] = ((a_.f64[0] != a_.f64[0]) | (b_.f64[0] != b_.f64[0]) | (a_.f64[0] == b_.f64[0])) ? ~INT64_C(0) : INT64_C(0); + break; + + case SIMDE_CMP_NGE_UQ: + case SIMDE_CMP_NGE_US: + a_.i64[0] = !(a_.f64[0] >= b_.f64[0]) ? ~INT64_C(0) : INT64_C(0); + break; + + case SIMDE_CMP_NGT_UQ: + case SIMDE_CMP_NGT_US: + a_.i64[0] = !(a_.f64[0] > b_.f64[0]) ? ~INT64_C(0) : INT64_C(0); + break; + + case SIMDE_CMP_FALSE_OQ: + case SIMDE_CMP_FALSE_OS: + a_.i64[0] = INT64_C(0); + break; + + case SIMDE_CMP_GE_OQ: + case SIMDE_CMP_GE_OS: + a_.i64[0] = (a_.f64[0] >= b_.f64[0]) ? ~INT64_C(0) : INT64_C(0); + break; + + case SIMDE_CMP_GT_OQ: + case SIMDE_CMP_GT_OS: + a_.i64[0] = (a_.f64[0] > b_.f64[0]) ? ~INT64_C(0) : INT64_C(0); + break; + + case SIMDE_CMP_TRUE_UQ: + case SIMDE_CMP_TRUE_US: + a_.i64[0] = ~INT64_C(0); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde__m128d_from_private(a_); +} +#if defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm_cmp_sd(a, b, imm8) _mm_cmp_sd(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_cmp_sd + #define _mm_cmp_sd(a, b, imm8) simde_mm_cmp_sd(a, b, imm8) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmp_ss (simde__m128 a, simde__m128 b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 31) { + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + switch (imm8) { + case SIMDE_CMP_EQ_OQ: + case SIMDE_CMP_EQ_OS: + a_.i32[0] = (a_.f32[0] == b_.f32[0]) ? ~INT32_C(0) : INT32_C(0); + break; + + case SIMDE_CMP_LT_OQ: + case SIMDE_CMP_LT_OS: + a_.i32[0] = (a_.f32[0] < b_.f32[0]) ? ~INT32_C(0) : INT32_C(0); + break; + + case SIMDE_CMP_LE_OQ: + case SIMDE_CMP_LE_OS: + a_.i32[0] = (a_.f32[0] <= b_.f32[0]) ? ~INT32_C(0) : INT32_C(0); + break; + + case SIMDE_CMP_UNORD_Q: + case SIMDE_CMP_UNORD_S: + a_.i32[0] = ((a_.f32[0] != a_.f32[0]) || (b_.f32[0] != b_.f32[0])) ? ~INT32_C(0) : INT32_C(0); + break; + + case SIMDE_CMP_NEQ_UQ: + case SIMDE_CMP_NEQ_US: + a_.i32[0] = ((a_.f32[0] == a_.f32[0]) & (b_.f32[0] == b_.f32[0]) & (a_.f32[0] != b_.f32[0])) ? ~INT32_C(0) : INT32_C(0); + break; + + case SIMDE_CMP_NEQ_OQ: + case SIMDE_CMP_NEQ_OS: + a_.i32[0] = ((a_.f32[0] == a_.f32[0]) & (b_.f32[0] == b_.f32[0]) & (a_.f32[0] != b_.f32[0])) ? ~INT32_C(0) : INT32_C(0); + break; + + case SIMDE_CMP_NLT_UQ: + case SIMDE_CMP_NLT_US: + a_.i32[0] = !(a_.f32[0] < b_.f32[0]) ? ~INT32_C(0) : INT32_C(0); + break; + + case SIMDE_CMP_NLE_UQ: + case SIMDE_CMP_NLE_US: + a_.i32[0] = !(a_.f32[0] <= b_.f32[0]) ? ~INT32_C(0) : INT32_C(0); + break; + + case SIMDE_CMP_ORD_Q: + case SIMDE_CMP_ORD_S: + a_.i32[0] = ((a_.f32[0] == a_.f32[0]) & (b_.f32[0] == b_.f32[0])) ? ~INT32_C(0) : INT32_C(0); + break; + + case SIMDE_CMP_EQ_UQ: + case SIMDE_CMP_EQ_US: + a_.i32[0] = ((a_.f32[0] != a_.f32[0]) | (b_.f32[0] != b_.f32[0]) | (a_.f32[0] == b_.f32[0])) ? ~INT32_C(0) : INT32_C(0); + break; + + case SIMDE_CMP_NGE_UQ: + case SIMDE_CMP_NGE_US: + a_.i32[0] = !(a_.f32[0] >= b_.f32[0]) ? ~INT32_C(0) : INT32_C(0); + break; + + case SIMDE_CMP_NGT_UQ: + case SIMDE_CMP_NGT_US: + a_.i32[0] = !(a_.f32[0] > b_.f32[0]) ? ~INT32_C(0) : INT32_C(0); + break; + + case SIMDE_CMP_FALSE_OQ: + case SIMDE_CMP_FALSE_OS: + a_.i32[0] = INT32_C(0); + break; + + case SIMDE_CMP_GE_OQ: + case SIMDE_CMP_GE_OS: + a_.i32[0] = (a_.f32[0] >= b_.f32[0]) ? ~INT32_C(0) : INT32_C(0); + break; + + case SIMDE_CMP_GT_OQ: + case SIMDE_CMP_GT_OS: + a_.i32[0] = (a_.f32[0] > b_.f32[0]) ? ~INT32_C(0) : INT32_C(0); + break; + + case SIMDE_CMP_TRUE_UQ: + case SIMDE_CMP_TRUE_US: + a_.i32[0] = ~INT32_C(0); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde__m128_from_private(a_); +} +#if defined(SIMDE_X86_AVX_NATIVE) + #define simde_mm_cmp_ss(a, b, imm8) _mm_cmp_ss(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_cmp_ss + #define _mm_cmp_ss(a, b, imm8) simde_mm_cmp_ss(a, b, imm8) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__m256d +#if defined(__clang__) && defined(__AVX512DQ__) +simde_mm256_cmp_pd_internal_ +#else +simde_mm256_cmp_pd +#endif +(simde__m256d a, simde__m256d b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 31) { + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + switch (imm8) { + case SIMDE_CMP_EQ_OQ: + case SIMDE_CMP_EQ_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 == b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = (a_.f64[i] == b_.f64[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_LT_OQ: + case SIMDE_CMP_LT_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 < b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = (a_.f64[i] < b_.f64[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_LE_OQ: + case SIMDE_CMP_LE_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 <= b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = (a_.f64[i] <= b_.f64[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_UNORD_Q: + case SIMDE_CMP_UNORD_S: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 != a_.f64) | (b_.f64 != b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = ((a_.f64[i] != a_.f64[i]) || (b_.f64[i] != b_.f64[i])) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NEQ_UQ: + case SIMDE_CMP_NEQ_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 != b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = (a_.f64[i] != b_.f64[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NEQ_OQ: + case SIMDE_CMP_NEQ_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 == a_.f64) & (b_.f64 == b_.f64) & (a_.f64 != b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = ((a_.f64[i] == a_.f64[i]) & (b_.f64[i] == b_.f64[i]) & (a_.f64[i] != b_.f64[i])) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NLT_UQ: + case SIMDE_CMP_NLT_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), ~(a_.f64 < b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = !(a_.f64[i] < b_.f64[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NLE_UQ: + case SIMDE_CMP_NLE_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), ~(a_.f64 <= b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = !(a_.f64[i] <= b_.f64[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_ORD_Q: + case SIMDE_CMP_ORD_S: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), ((a_.f64 == a_.f64) & (b_.f64 == b_.f64))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = ((a_.f64[i] == a_.f64[i]) & (b_.f64[i] == b_.f64[i])) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_EQ_UQ: + case SIMDE_CMP_EQ_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 != a_.f64) | (b_.f64 != b_.f64) | (a_.f64 == b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = ((a_.f64[i] != a_.f64[i]) | (b_.f64[i] != b_.f64[i]) | (a_.f64[i] == b_.f64[i])) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NGE_UQ: + case SIMDE_CMP_NGE_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), ~(a_.f64 >= b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = !(a_.f64[i] >= b_.f64[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NGT_UQ: + case SIMDE_CMP_NGT_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), ~(a_.f64 > b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = !(a_.f64[i] > b_.f64[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_FALSE_OQ: + case SIMDE_CMP_FALSE_OS: + r_ = simde__m256d_to_private(simde_mm256_setzero_pd()); + break; + + case SIMDE_CMP_GE_OQ: + case SIMDE_CMP_GE_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 >= b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = (a_.f64[i] >= b_.f64[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_GT_OQ: + case SIMDE_CMP_GT_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 > b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = (a_.f64[i] > b_.f64[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_TRUE_UQ: + case SIMDE_CMP_TRUE_US: + r_ = simde__m256d_to_private(simde_x_mm256_setone_pd()); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde__m256d_from_private(r_); +} +#if defined(__clang__) && defined(__AVX512DQ__) + #define simde_mm256_cmp_pd(a, b, imm8) (__extension__ ({ \ + simde__m256d simde_mm256_cmp_pd_r; \ + switch (imm8) { \ + case SIMDE_CMP_FALSE_OQ: \ + case SIMDE_CMP_FALSE_OS: \ + simde_mm256_cmp_pd_r = simde_mm256_setzero_pd(); \ + break; \ + case SIMDE_CMP_TRUE_UQ: \ + case SIMDE_CMP_TRUE_US: \ + simde_mm256_cmp_pd_r = simde_x_mm256_setone_pd(); \ + break; \ + default: \ + simde_mm256_cmp_pd_r = simde_mm256_cmp_pd_internal_(a, b, imm8); \ + break; \ + } \ + simde_mm256_cmp_pd_r; \ + })) +#elif defined(SIMDE_X86_AVX_NATIVE) + #define simde_mm256_cmp_pd(a, b, imm8) _mm256_cmp_pd(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmp_pd + #define _mm256_cmp_pd(a, b, imm8) simde_mm256_cmp_pd(a, b, imm8) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__m256 +#if defined(__clang__) && defined(__AVX512DQ__) +simde_mm256_cmp_ps_internal_ +#else +simde_mm256_cmp_ps +#endif +(simde__m256 a, simde__m256 b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 31) { + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + switch (imm8) { + case SIMDE_CMP_EQ_OQ: + case SIMDE_CMP_EQ_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 == b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = (a_.f32[i] == b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_LT_OQ: + case SIMDE_CMP_LT_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 < b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = (a_.f32[i] < b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_LE_OQ: + case SIMDE_CMP_LE_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 <= b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = (a_.f32[i] <= b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_UNORD_Q: + case SIMDE_CMP_UNORD_S: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 != a_.f32) | (b_.f32 != b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = ((a_.f32[i] != a_.f32[i]) || (b_.f32[i] != b_.f32[i])) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NEQ_UQ: + case SIMDE_CMP_NEQ_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 != b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = (a_.f32[i] != b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NEQ_OQ: + case SIMDE_CMP_NEQ_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 == a_.f32) & (b_.f32 == b_.f32) & (a_.f32 != b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = ((a_.f32[i] == a_.f32[i]) & (b_.f32[i] == b_.f32[i]) & (a_.f32[i] != b_.f32[i])) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NLT_UQ: + case SIMDE_CMP_NLT_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), ~(a_.f32 < b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = !(a_.f32[i] < b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NLE_UQ: + case SIMDE_CMP_NLE_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), ~(a_.f32 <= b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = !(a_.f32[i] <= b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_ORD_Q: + case SIMDE_CMP_ORD_S: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), ((a_.f32 == a_.f32) & (b_.f32 == b_.f32))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = ((a_.f32[i] == a_.f32[i]) & (b_.f32[i] == b_.f32[i])) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_EQ_UQ: + case SIMDE_CMP_EQ_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 != a_.f32) | (b_.f32 != b_.f32) | (a_.f32 == b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = ((a_.f32[i] != a_.f32[i]) | (b_.f32[i] != b_.f32[i]) | (a_.f32[i] == b_.f32[i])) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NGE_UQ: + case SIMDE_CMP_NGE_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), ~(a_.f32 >= b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = !(a_.f32[i] >= b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NGT_UQ: + case SIMDE_CMP_NGT_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), ~(a_.f32 > b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = !(a_.f32[i] > b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_FALSE_OQ: + case SIMDE_CMP_FALSE_OS: + r_ = simde__m256_to_private(simde_mm256_setzero_ps()); + break; + + case SIMDE_CMP_GE_OQ: + case SIMDE_CMP_GE_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 >= b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = (a_.f32[i] >= b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_GT_OQ: + case SIMDE_CMP_GT_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 > b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = (a_.f32[i] > b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_TRUE_UQ: + case SIMDE_CMP_TRUE_US: + r_ = simde__m256_to_private(simde_x_mm256_setone_ps()); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde__m256_from_private(r_); +} +#if defined(__clang__) && defined(__AVX512DQ__) + #define simde_mm256_cmp_ps(a, b, imm8) (__extension__ ({ \ + simde__m256 simde_mm256_cmp_ps_r; \ + switch (imm8) { \ + case SIMDE_CMP_FALSE_OQ: \ + case SIMDE_CMP_FALSE_OS: \ + simde_mm256_cmp_ps_r = simde_mm256_setzero_ps(); \ + break; \ + case SIMDE_CMP_TRUE_UQ: \ + case SIMDE_CMP_TRUE_US: \ + simde_mm256_cmp_ps_r = simde_x_mm256_setone_ps(); \ + break; \ + default: \ + simde_mm256_cmp_ps_r = simde_mm256_cmp_ps_internal_(a, b, imm8); \ + break; \ + } \ + simde_mm256_cmp_ps_r; \ + })) +#elif defined(SIMDE_X86_AVX_NATIVE) + #define simde_mm256_cmp_ps(a, b, imm8) _mm256_cmp_ps(a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) && SIMDE_NATURAL_VECTOR_SIZE_LE(128) + #define simde_mm256_cmp_ps(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m256_private \ + simde_mm256_cmp_ps_r_ = simde__m256_to_private(simde_mm256_setzero_ps()), \ + simde_mm256_cmp_ps_a_ = simde__m256_to_private((a)), \ + simde_mm256_cmp_ps_b_ = simde__m256_to_private((b)); \ + \ + for (size_t i = 0 ; i < (sizeof(simde_mm256_cmp_ps_r_.m128) / sizeof(simde_mm256_cmp_ps_r_.m128[0])) ; i++) { \ + simde_mm256_cmp_ps_r_.m128[i] = simde_mm_cmp_ps(simde_mm256_cmp_ps_a_.m128[i], simde_mm256_cmp_ps_b_.m128[i], (imm8)); \ + } \ + \ + simde__m256_from_private(simde_mm256_cmp_ps_r_); \ + })) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmp_ps + #define _mm256_cmp_ps(a, b, imm8) simde_mm256_cmp_ps(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_x_mm256_copysign_ps(simde__m256 dest, simde__m256 src) { + simde__m256_private + r_, + dest_ = simde__m256_to_private(dest), + src_ = simde__m256_to_private(src); + + #if defined(simde_math_copysignf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_copysignf(dest_.f32[i], src_.f32[i]); + } + #else + simde__m256 sgnbit = simde_mm256_xor_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.0)), simde_mm256_set1_ps(-SIMDE_FLOAT32_C(0.0))); + return simde_mm256_xor_ps(simde_mm256_and_ps(sgnbit, src), simde_mm256_andnot_ps(sgnbit, dest)); + #endif + + return simde__m256_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_x_mm256_copysign_pd(simde__m256d dest, simde__m256d src) { + simde__m256d_private + r_, + dest_ = simde__m256d_to_private(dest), + src_ = simde__m256d_to_private(src); + + #if defined(simde_math_copysign) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_copysign(dest_.f64[i], src_.f64[i]); + } + #else + simde__m256d sgnbit = simde_mm256_xor_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.0)), simde_mm256_set1_pd(-SIMDE_FLOAT64_C(0.0))); + return simde_mm256_xor_pd(simde_mm256_and_pd(sgnbit, src), simde_mm256_andnot_pd(sgnbit, dest)); + #endif + + return simde__m256d_from_private(r_); +} + +HEDLEY_DIAGNOSTIC_POP /* -Wfloat-equal */ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_cvtepi32_pd (simde__m128i a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cvtepi32_pd(a); + #else + simde__m256d_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = HEDLEY_STATIC_CAST(simde_float64, a_.i32[i]); + } + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtepi32_pd + #define _mm256_cvtepi32_pd(a) simde_mm256_cvtepi32_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 + simde_mm256_cvtepi32_ps (simde__m256i a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cvtepi32_ps(a); + #else + simde__m256_private r_; + simde__m256i_private a_ = simde__m256i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = HEDLEY_STATIC_CAST(simde_float32, a_.i32[i]); + } + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtepi32_ps + #define _mm256_cvtepi32_ps(a) simde_mm256_cvtepi32_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm256_cvtpd_epi32 (simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cvtpd_epi32(a); + #else + simde__m128i_private r_; + simde__m256d_private a_ = simde__m256d_to_private(a); + + #if defined(simde_math_nearbyint) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + r_.i32[i] = SIMDE_CONVERT_FTOI(int32_t, simde_math_nearbyint(a_.f64[i])); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtpd_epi32 + #define _mm256_cvtpd_epi32(a) simde_mm256_cvtpd_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm256_cvtpd_ps (simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cvtpd_ps(a); + #else + simde__m128_private r_; + simde__m256d_private a_ = simde__m256d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = HEDLEY_STATIC_CAST(simde_float32, a_.f64[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtpd_ps + #define _mm256_cvtpd_ps(a) simde_mm256_cvtpd_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cvtps_epi32 (simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cvtps_epi32(a); + #else + simde__m256i_private r_; + simde__m256_private a_ = simde__m256_to_private(a); + + #if defined(simde_math_nearbyintf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + r_.i32[i] = SIMDE_CONVERT_FTOI(int32_t, simde_math_nearbyintf(a_.f32[i])); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtps_epi32 + #define _mm256_cvtps_epi32(a) simde_mm256_cvtps_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_cvtps_pd (simde__m128 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cvtps_pd(a); + #else + simde__m256d_private r_; + simde__m128_private a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + r_.f64[i] = HEDLEY_STATIC_CAST(double, a_.f32[i]); + } + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtps_pd + #define _mm256_cvtps_pd(a) simde_mm256_cvtps_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde_float64 +simde_mm256_cvtsd_f64 (simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) && ( \ + SIMDE_DETECT_CLANG_VERSION_CHECK(3,9,0) || \ + HEDLEY_GCC_VERSION_CHECK(7,0,0) || \ + HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + HEDLEY_MSVC_VERSION_CHECK(19,14,0)) + return _mm256_cvtsd_f64(a); + #else + simde__m256d_private a_ = simde__m256d_to_private(a); + return a_.f64[0]; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtsd_f64 + #define _mm256_cvtsd_f64(a) simde_mm256_cvtsd_f64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm256_cvtsi256_si32 (simde__m256i a) { + #if defined(SIMDE_X86_AVX_NATIVE) && ( \ + SIMDE_DETECT_CLANG_VERSION_CHECK(3,9,0) || \ + HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + HEDLEY_MSVC_VERSION_CHECK(19,14,0)) + return _mm256_cvtsi256_si32(a); + #else + simde__m256i_private a_ = simde__m256i_to_private(a); + return a_.i32[0]; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtsi256_si32 + #define _mm256_cvtsi256_si32(a) simde_mm256_cvtsi256_si32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde_float32 +simde_mm256_cvtss_f32 (simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) && ( \ + SIMDE_DETECT_CLANG_VERSION_CHECK(3,9,0) || \ + HEDLEY_GCC_VERSION_CHECK(7,0,0) || \ + HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + HEDLEY_MSVC_VERSION_CHECK(19,14,0)) + return _mm256_cvtss_f32(a); + #else + simde__m256_private a_ = simde__m256_to_private(a); + return a_.f32[0]; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtss_f32 + #define _mm256_cvtss_f32(a) simde_mm256_cvtss_f32(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm256_cvttpd_epi32 (simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cvttpd_epi32(a); + #else + simde__m128i_private r_; + simde__m256d_private a_ = simde__m256d_to_private(a); + + #if defined(simde_math_trunc) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + r_.i32[i] = SIMDE_CONVERT_FTOI(int32_t, simde_math_trunc(a_.f64[i])); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvttpd_epi32 + #define _mm256_cvttpd_epi32(a) simde_mm256_cvttpd_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cvttps_epi32 (simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cvttps_epi32(a); + #else + simde__m256i_private r_; + simde__m256_private a_ = simde__m256_to_private(a); + + #if defined(simde_math_truncf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + r_.i32[i] = SIMDE_CONVERT_FTOI(int32_t, simde_math_truncf(a_.f32[i])); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvttps_epi32 + #define _mm256_cvttps_epi32(a) simde_mm256_cvttps_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_div_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_div_ps(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_div_ps(a_.m128[0], b_.m128[0]); + r_.m128[1] = simde_mm_div_ps(a_.m128[1], b_.m128[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = a_.f32 / b_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[i] / b_.f32[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_div_ps + #define _mm256_div_ps(a, b) simde_mm256_div_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_div_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_div_pd(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_div_pd(a_.m128d[0], b_.m128d[0]); + r_.m128d[1] = simde_mm_div_pd(a_.m128d[1], b_.m128d[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = a_.f64 / b_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[i] / b_.f64[i]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_div_pd + #define _mm256_div_pd(a, b) simde_mm256_div_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm256_extractf128_pd (simde__m256d a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m256d_private a_ = simde__m256d_to_private(a); + return a_.m128d[imm8]; +} +#if defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm256_extractf128_pd(a, imm8) _mm256_extractf128_pd(a, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_extractf128_pd + #define _mm256_extractf128_pd(a, imm8) simde_mm256_extractf128_pd(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm256_extractf128_ps (simde__m256 a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m256_private a_ = simde__m256_to_private(a); + return a_.m128[imm8]; +} +#if defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm256_extractf128_ps(a, imm8) _mm256_extractf128_ps(a, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_extractf128_ps + #define _mm256_extractf128_ps(a, imm8) simde_mm256_extractf128_ps(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm256_extractf128_si256 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m256i_private a_ = simde__m256i_to_private(a); + return a_.m128i[imm8]; +} +#if defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm256_extractf128_si256(a, imm8) _mm256_extractf128_si256(a, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_extractf128_si256 + #define _mm256_extractf128_si256(a, imm8) simde_mm256_extractf128_si256(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_floor_pd (simde__m256d a) { + return simde_mm256_round_pd(a, SIMDE_MM_FROUND_TO_NEG_INF); +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_floor_pd + #define _mm256_floor_pd(a) simde_mm256_floor_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_floor_ps (simde__m256 a) { + return simde_mm256_round_ps(a, SIMDE_MM_FROUND_TO_NEG_INF); +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_floor_ps + #define _mm256_floor_ps(a) simde_mm256_floor_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_insert_epi8 (simde__m256i a, int8_t i, const int index) + SIMDE_REQUIRE_RANGE(index, 0, 31) { + simde__m256i_private a_ = simde__m256i_to_private(a); + + a_.i8[index] = i; + + return simde__m256i_from_private(a_); +} +#if defined(SIMDE_X86_AVX_NATIVE) && \ + (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,10,0)) + #define simde_mm256_insert_epi8(a, i, index) _mm256_insert_epi8(a, i, index) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_insert_epi8 + #define _mm256_insert_epi8(a, i, index) simde_mm256_insert_epi8(a, i, index) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_insert_epi16 (simde__m256i a, int16_t i, const int index) + SIMDE_REQUIRE_RANGE(index, 0, 15) { + simde__m256i_private a_ = simde__m256i_to_private(a); + + a_.i16[index] = i; + + return simde__m256i_from_private(a_); +} +#if defined(SIMDE_X86_AVX_NATIVE) && \ + (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,10,0)) + #define simde_mm256_insert_epi16(a, i, index) _mm256_insert_epi16(a, i, index) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_insert_epi16 + #define _mm256_insert_epi16(a, i, imm8) simde_mm256_insert_epi16(a, i, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_insert_epi32 (simde__m256i a, int32_t i, const int index) + SIMDE_REQUIRE_RANGE(index, 0, 7) { + simde__m256i_private a_ = simde__m256i_to_private(a); + + a_.i32[index] = i; + + return simde__m256i_from_private(a_); +} +#if defined(SIMDE_X86_AVX_NATIVE) && \ + (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,10,0)) + #define simde_mm256_insert_epi32(a, i, index) _mm256_insert_epi32(a, i, index) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_insert_epi32 + #define _mm256_insert_epi32(a, i, index) simde_mm256_insert_epi32(a, i, index) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_insert_epi64 (simde__m256i a, int64_t i, const int index) + SIMDE_REQUIRE_RANGE(index, 0, 3) { + simde__m256i_private a_ = simde__m256i_to_private(a); + + a_.i64[index] = i; + + return simde__m256i_from_private(a_); +} +#if defined(SIMDE_X86_AVX_NATIVE) && defined(SIMDE_ARCH_AMD64) && \ + (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) && \ + SIMDE_DETECT_CLANG_VERSION_CHECK(3,7,0) + #define simde_mm256_insert_epi64(a, i, index) _mm256_insert_epi64(a, i, index) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) + #undef _mm256_insert_epi64 + #define _mm256_insert_epi64(a, i, index) simde_mm256_insert_epi64(a, i, index) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d simde_mm256_insertf128_pd(simde__m256d a, simde__m128d b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m256d_private a_ = simde__m256d_to_private(a); + simde__m128d_private b_ = simde__m128d_to_private(b); + + a_.m128d_private[imm8] = b_; + + return simde__m256d_from_private(a_); +} +#if defined(SIMDE_X86_AVX_NATIVE) + #define simde_mm256_insertf128_pd(a, b, imm8) _mm256_insertf128_pd(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_insertf128_pd + #define _mm256_insertf128_pd(a, b, imm8) simde_mm256_insertf128_pd(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 simde_mm256_insertf128_ps(simde__m256 a, simde__m128 b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m256_private a_ = simde__m256_to_private(a); + simde__m128_private b_ = simde__m128_to_private(b); + + a_.m128_private[imm8] = b_; + + return simde__m256_from_private(a_); +} +#if defined(SIMDE_X86_AVX_NATIVE) + #define simde_mm256_insertf128_ps(a, b, imm8) _mm256_insertf128_ps(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_insertf128_ps + #define _mm256_insertf128_ps(a, b, imm8) simde_mm256_insertf128_ps(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i simde_mm256_insertf128_si256(simde__m256i a, simde__m128i b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m256i_private a_ = simde__m256i_to_private(a); + simde__m128i_private b_ = simde__m128i_to_private(b); + + a_.m128i_private[imm8] = b_; + + return simde__m256i_from_private(a_); +} +#if defined(SIMDE_X86_AVX_NATIVE) + #define simde_mm256_insertf128_si256(a, b, imm8) _mm256_insertf128_si256(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_insertf128_si256 + #define _mm256_insertf128_si256(a, b, imm8) simde_mm256_insertf128_si256(a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm256_dp_ps(a, b, imm8) _mm256_dp_ps(a, b, imm8) +#else +# define simde_mm256_dp_ps(a, b, imm8) \ + simde_mm256_set_m128( \ + simde_mm_dp_ps(simde_mm256_extractf128_ps(a, 1), simde_mm256_extractf128_ps(b, 1), imm8), \ + simde_mm_dp_ps(simde_mm256_extractf128_ps(a, 0), simde_mm256_extractf128_ps(b, 0), imm8)) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_dp_ps + #define _mm256_dp_ps(a, b, imm8) simde_mm256_dp_ps(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm256_extract_epi32 (simde__m256i a, const int index) + SIMDE_REQUIRE_RANGE(index, 0, 7) { + simde__m256i_private a_ = simde__m256i_to_private(a); + return a_.i32[index]; +} +#if defined(SIMDE_X86_AVX_NATIVE) && \ + (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,10,0)) + #define simde_mm256_extract_epi32(a, index) _mm256_extract_epi32(a, index) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_extract_epi32 + #define _mm256_extract_epi32(a, index) simde_mm256_extract_epi32(a, index) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int64_t +simde_mm256_extract_epi64 (simde__m256i a, const int index) + SIMDE_REQUIRE_RANGE(index, 0, 3) { + simde__m256i_private a_ = simde__m256i_to_private(a); + return a_.i64[index]; +} +#if defined(SIMDE_X86_AVX_NATIVE) && defined(SIMDE_ARCH_AMD64) + #if !defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0) + #define simde_mm256_extract_epi64(a, index) _mm256_extract_epi64(a, index) + #endif +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) + #undef _mm256_extract_epi64 + #define _mm256_extract_epi64(a, index) simde_mm256_extract_epi64(a, index) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_lddqu_si256 (simde__m256i const * mem_addr) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_loadu_si256(mem_addr); + #else + simde__m256i r; + simde_memcpy(&r, SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m256i), sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_lddqu_si256 + #define _mm256_lddqu_si256(a) simde_mm256_lddqu_si256(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_load_pd (const double mem_addr[HEDLEY_ARRAY_PARAM(4)]) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_load_pd(mem_addr); + #else + simde__m256d r; + simde_memcpy(&r, SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m256d), sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_load_pd + #define _mm256_load_pd(a) simde_mm256_load_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_load_ps (const float mem_addr[HEDLEY_ARRAY_PARAM(8)]) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_load_ps(mem_addr); + #else + simde__m256 r; + simde_memcpy(&r, SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m256), sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_load_ps + #define _mm256_load_ps(a) simde_mm256_load_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_load_si256 (simde__m256i const * mem_addr) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_load_si256(mem_addr); + #else + simde__m256i r; + simde_memcpy(&r, SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m256i), sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_load_si256 + #define _mm256_load_si256(a) simde_mm256_load_si256(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_loadu_pd (const double a[HEDLEY_ARRAY_PARAM(4)]) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_loadu_pd(a); + #else + simde__m256d r; + simde_memcpy(&r, a, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_loadu_pd + #define _mm256_loadu_pd(a) simde_mm256_loadu_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_loadu_ps (const float a[HEDLEY_ARRAY_PARAM(8)]) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_loadu_ps(a); + #else + simde__m256 r; + simde_memcpy(&r, a, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_loadu_ps + #define _mm256_loadu_ps(a) simde_mm256_loadu_ps(a) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) \ + && !defined(SIMDE_BUG_GCC_95483) && !defined(SIMDE_BUG_CLANG_REV_344862) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define simde_mm256_loadu_epi8(mem_addr) _mm256_loadu_epi8(mem_addr) +#else +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_loadu_epi8(void const * mem_addr) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_loadu_si256(SIMDE_ALIGN_CAST(__m256i const *, mem_addr)); + #else + simde__m256i r; + simde_memcpy(&r, mem_addr, sizeof(r)); + return r; + #endif +} +#endif +#define simde_x_mm256_loadu_epi8(mem_addr) simde_mm256_loadu_epi8(mem_addr) +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && (defined(SIMDE_BUG_GCC_95483) || defined(SIMDE_BUG_CLANG_REV_344862))) + #undef _mm256_loadu_epi8 + #define _mm256_loadu_epi8(a) simde_mm256_loadu_epi8(a) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) \ + && !defined(SIMDE_BUG_GCC_95483) && !defined(SIMDE_BUG_CLANG_REV_344862) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define simde_mm256_loadu_epi16(mem_addr) _mm256_loadu_epi16(mem_addr) +#else +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_loadu_epi16(void const * mem_addr) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_loadu_si256(SIMDE_ALIGN_CAST(__m256i const *, mem_addr)); + #else + simde__m256i r; + simde_memcpy(&r, mem_addr, sizeof(r)); + return r; + #endif +} +#endif +#define simde_x_mm256_loadu_epi16(mem_addr) simde_mm256_loadu_epi16(mem_addr) +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && (defined(SIMDE_BUG_GCC_95483) || defined(SIMDE_BUG_CLANG_REV_344862))) + #undef _mm256_loadu_epi16 + #define _mm256_loadu_epi16(a) simde_mm256_loadu_epi16(a) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && !defined(SIMDE_BUG_GCC_95483) \ + && !defined(SIMDE_BUG_CLANG_REV_344862) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define simde_mm256_loadu_epi32(mem_addr) _mm256_loadu_epi32(mem_addr) +#else +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_loadu_epi32(void const * mem_addr) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_loadu_si256(SIMDE_ALIGN_CAST(__m256i const *, mem_addr)); + #else + simde__m256i r; + simde_memcpy(&r, mem_addr, sizeof(r)); + return r; + #endif +} +#endif +#define simde_x_mm256_loadu_epi32(mem_addr) simde_mm256_loadu_epi32(mem_addr) +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && (defined(SIMDE_BUG_GCC_95483) || defined(SIMDE_BUG_CLANG_REV_344862))) + #undef _mm256_loadu_epi32 + #define _mm256_loadu_epi32(a) simde_mm256_loadu_epi32(a) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && !defined(SIMDE_BUG_GCC_95483) \ + && !defined(SIMDE_BUG_CLANG_REV_344862) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define simde_mm256_loadu_epi64(mem_addr) _mm256_loadu_epi64(mem_addr) +#else +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_loadu_epi64(void const * mem_addr) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_loadu_si256(SIMDE_ALIGN_CAST(__m256i const *, mem_addr)); + #else + simde__m256i r; + simde_memcpy(&r, mem_addr, sizeof(r)); + return r; + #endif +} +#endif +#define simde_x_mm256_loadu_epi64(mem_addr) simde_mm256_loadu_epi64(mem_addr) +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && (defined(SIMDE_BUG_GCC_95483) || defined(SIMDE_BUG_CLANG_REV_344862))) + #undef _mm256_loadu_epi64 + #define _mm256_loadu_epi64(a) simde_mm256_loadu_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_loadu_si256 (void const * mem_addr) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_loadu_si256(SIMDE_ALIGN_CAST(const __m256i*, mem_addr)); + #else + simde__m256i r; + simde_memcpy(&r, mem_addr, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_loadu_si256 + #define _mm256_loadu_si256(mem_addr) simde_mm256_loadu_si256(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_loadu2_m128 (const float hiaddr[HEDLEY_ARRAY_PARAM(4)], const float loaddr[HEDLEY_ARRAY_PARAM(4)]) { + #if defined(SIMDE_X86_AVX_NATIVE) && !defined(SIMDE_BUG_GCC_91341) && !defined(SIMDE_BUG_MCST_LCC_MISSING_AVX_LOAD_STORE_M128_FUNCS) + return _mm256_loadu2_m128(hiaddr, loaddr); + #else + return + simde_mm256_insertf128_ps(simde_mm256_castps128_ps256(simde_mm_loadu_ps(loaddr)), + simde_mm_loadu_ps(hiaddr), 1); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_loadu2_m128 + #define _mm256_loadu2_m128(hiaddr, loaddr) simde_mm256_loadu2_m128(hiaddr, loaddr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_loadu2_m128d (const double hiaddr[HEDLEY_ARRAY_PARAM(2)], const double loaddr[HEDLEY_ARRAY_PARAM(2)]) { + #if defined(SIMDE_X86_AVX_NATIVE) && !defined(SIMDE_BUG_GCC_91341) && !defined(SIMDE_BUG_MCST_LCC_MISSING_AVX_LOAD_STORE_M128_FUNCS) + return _mm256_loadu2_m128d(hiaddr, loaddr); + #else + return + simde_mm256_insertf128_pd(simde_mm256_castpd128_pd256(simde_mm_loadu_pd(loaddr)), + simde_mm_loadu_pd(hiaddr), 1); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_loadu2_m128d + #define _mm256_loadu2_m128d(hiaddr, loaddr) simde_mm256_loadu2_m128d(hiaddr, loaddr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_loadu2_m128i (const simde__m128i* hiaddr, const simde__m128i* loaddr) { + #if defined(SIMDE_X86_AVX_NATIVE) && !defined(SIMDE_BUG_GCC_91341) && !defined(SIMDE_BUG_MCST_LCC_MISSING_AVX_LOAD_STORE_M128_FUNCS) + return _mm256_loadu2_m128i(hiaddr, loaddr); + #else + return + simde_mm256_insertf128_si256(simde_mm256_castsi128_si256(simde_mm_loadu_si128(loaddr)), + simde_mm_loadu_si128(hiaddr), 1); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_loadu2_m128i + #define _mm256_loadu2_m128i(hiaddr, loaddr) simde_mm256_loadu2_m128i(hiaddr, loaddr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_maskload_pd (const simde_float64 mem_addr[HEDLEY_ARRAY_PARAM(2)], simde__m128i mask) { + #if defined(SIMDE_X86_AVX_NATIVE) + #if defined(__clang__) && !SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + return _mm_maskload_pd(mem_addr, HEDLEY_REINTERPRET_CAST(simde__m128d, mask)); + #else + return _mm_maskload_pd(mem_addr, mask); + #endif + #else + simde__m128d_private r_; + simde__m128i_private + mask_ = simde__m128i_to_private(mask), + mask_shr_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + mask_shr_.neon_i64 = vshrq_n_s64(mask_.neon_i64, 63); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return simde_mm_and_pd(simde_mm_load_pd(mem_addr), + simde__m128d_from_wasm_v128(wasm_i64x2_shr(mask_.wasm_v128, 63))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(mask_.i64) / sizeof(mask_.i64[0])) ; i++) { + mask_shr_.i64[i] = mask_.i64[i] >> 63; + } + #endif + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = mask_shr_.i64[i] ? mem_addr[i] : SIMDE_FLOAT64_C(0.0); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_maskload_pd + #define _mm_maskload_pd(mem_addr, mask) simde_mm_maskload_pd(HEDLEY_REINTERPRET_CAST(double const*, mem_addr), mask) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_maskload_pd (const simde_float64 mem_addr[HEDLEY_ARRAY_PARAM(4)], simde__m256i mask) { + #if defined(SIMDE_X86_AVX_NATIVE) + #if defined(__clang__) && !SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + return _mm256_maskload_pd(mem_addr, HEDLEY_REINTERPRET_CAST(simde__m256d, mask)); + #else + return _mm256_maskload_pd(mem_addr, mask); + #endif + #else + simde__m256d_private r_; + simde__m256i_private mask_ = simde__m256i_to_private(mask); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = (mask_.i64[i] >> 63) ? mem_addr[i] : SIMDE_FLOAT64_C(0.0); + } + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskload_pd + #define _mm256_maskload_pd(mem_addr, mask) simde_mm256_maskload_pd(HEDLEY_REINTERPRET_CAST(double const*, mem_addr), mask) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_maskload_ps (const simde_float32 mem_addr[HEDLEY_ARRAY_PARAM(4)], simde__m128i mask) { + #if defined(SIMDE_X86_AVX_NATIVE) + #if defined(__clang__) && !SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + return _mm_maskload_ps(mem_addr, HEDLEY_REINTERPRET_CAST(simde__m128, mask)); + #else + return _mm_maskload_ps(mem_addr, mask); + #endif + #else + simde__m128_private r_; + simde__m128i_private + mask_ = simde__m128i_to_private(mask), + mask_shr_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + mask_shr_.neon_i32 = vshrq_n_s32(mask_.neon_i32, 31); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return simde_mm_and_ps(simde_mm_load_ps(mem_addr), + simde__m128_from_wasm_v128(wasm_i32x4_shr(mask_.wasm_v128, 31))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(mask_.i32) / sizeof(mask_.i32[0])) ; i++) { + mask_shr_.i32[i] = mask_.i32[i] >> 31; + } + #endif + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = mask_shr_.i32[i] ? mem_addr[i] : SIMDE_FLOAT32_C(0.0); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_maskload_ps + #define _mm_maskload_ps(mem_addr, mask) simde_mm_maskload_ps(HEDLEY_REINTERPRET_CAST(float const*, mem_addr), mask) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_maskload_ps (const simde_float32 mem_addr[HEDLEY_ARRAY_PARAM(8)], simde__m256i mask) { + #if defined(SIMDE_X86_AVX_NATIVE) + #if defined(__clang__) && !SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + return _mm256_maskload_ps(mem_addr, HEDLEY_REINTERPRET_CAST(simde__m256, mask)); + #else + return _mm256_maskload_ps(mem_addr, mask); + #endif + #else + simde__m256_private r_; + simde__m256i_private mask_ = simde__m256i_to_private(mask); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = (mask_.i32[i] >> 31) ? mem_addr[i] : SIMDE_FLOAT32_C(0.0); + } + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskload_ps + #define _mm256_maskload_ps(mem_addr, mask) simde_mm256_maskload_ps(HEDLEY_REINTERPRET_CAST(float const*, mem_addr), mask) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_maskstore_pd (simde_float64 mem_addr[HEDLEY_ARRAY_PARAM(2)], simde__m128i mask, simde__m128d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + #if defined(__clang__) && !SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + _mm_maskstore_pd(mem_addr, HEDLEY_REINTERPRET_CAST(simde__m128d, mask), a); + #else + _mm_maskstore_pd(mem_addr, mask, a); + #endif + #else + simde__m128i_private mask_ = simde__m128i_to_private(mask); + simde__m128d_private a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + if ((HEDLEY_STATIC_CAST(unsigned long long, wasm_i64x2_extract_lane(mask_.wasm_v128, 0)) & 0x8000000000000000ull) != 0) + mem_addr[0] = wasm_f64x2_extract_lane(a_.wasm_v128, 0); + if ((HEDLEY_STATIC_CAST(unsigned long long, wasm_i64x2_extract_lane(mask_.wasm_v128, 1)) & 0x8000000000000000ull) != 0) + mem_addr[1] = wasm_f64x2_extract_lane(a_.wasm_v128, 1); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + if (mask_.u64[i] >> 63) + mem_addr[i] = a_.f64[i]; + } + #endif + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_maskstore_pd + #define _mm_maskstore_pd(mem_addr, mask, a) simde_mm_maskstore_pd(HEDLEY_REINTERPRET_CAST(double*, mem_addr), mask, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_maskstore_pd (simde_float64 mem_addr[HEDLEY_ARRAY_PARAM(4)], simde__m256i mask, simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + #if defined(__clang__) && !SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + _mm256_maskstore_pd(mem_addr, HEDLEY_REINTERPRET_CAST(simde__m256d, mask), a); + #else + _mm256_maskstore_pd(mem_addr, mask, a); + #endif + #else + simde__m256i_private mask_ = simde__m256i_to_private(mask); + simde__m256d_private a_ = simde__m256d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + if (mask_.u64[i] & (UINT64_C(1) << 63)) + mem_addr[i] = a_.f64[i]; + } + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskstore_pd + #define _mm256_maskstore_pd(mem_addr, mask, a) simde_mm256_maskstore_pd(HEDLEY_REINTERPRET_CAST(double*, mem_addr), mask, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_maskstore_ps (simde_float32 mem_addr[HEDLEY_ARRAY_PARAM(4)], simde__m128i mask, simde__m128 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + #if defined(__clang__) && !SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + _mm_maskstore_ps(mem_addr, HEDLEY_REINTERPRET_CAST(simde__m128, mask), a); + #else + _mm_maskstore_ps(mem_addr, mask, a); + #endif + #else + simde__m128i_private mask_ = simde__m128i_to_private(mask); + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + if ((HEDLEY_STATIC_CAST(unsigned long long, wasm_i32x4_extract_lane(mask_.wasm_v128, 0)) & 0x80000000ull) != 0) + mem_addr[0] = wasm_f32x4_extract_lane(a_.wasm_v128, 0); + if ((HEDLEY_STATIC_CAST(unsigned long long, wasm_i32x4_extract_lane(mask_.wasm_v128, 1)) & 0x80000000ull) != 0) + mem_addr[1] = wasm_f32x4_extract_lane(a_.wasm_v128, 1); + if ((HEDLEY_STATIC_CAST(unsigned long long, wasm_i32x4_extract_lane(mask_.wasm_v128, 2)) & 0x80000000ull) != 0) + mem_addr[2] = wasm_f32x4_extract_lane(a_.wasm_v128, 2); + if ((HEDLEY_STATIC_CAST(unsigned long long, wasm_i32x4_extract_lane(mask_.wasm_v128, 3)) & 0x80000000ull) != 0) + mem_addr[3] = wasm_f32x4_extract_lane(a_.wasm_v128, 3); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + if (mask_.u32[i] & (UINT32_C(1) << 31)) + mem_addr[i] = a_.f32[i]; + } + #endif + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_maskstore_ps + #define _mm_maskstore_ps(mem_addr, mask, a) simde_mm_maskstore_ps(HEDLEY_REINTERPRET_CAST(float*, mem_addr), mask, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_maskstore_ps (simde_float32 mem_addr[HEDLEY_ARRAY_PARAM(8)], simde__m256i mask, simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + #if defined(__clang__) && !SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + _mm256_maskstore_ps(mem_addr, HEDLEY_REINTERPRET_CAST(simde__m256, mask), a); + #else + _mm256_maskstore_ps(mem_addr, mask, a); + #endif + #else + simde__m256i_private mask_ = simde__m256i_to_private(mask); + simde__m256_private a_ = simde__m256_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + if (mask_.u32[i] & (UINT32_C(1) << 31)) + mem_addr[i] = a_.f32[i]; + } + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskstore_ps + #define _mm256_maskstore_ps(mem_addr, mask, a) simde_mm256_maskstore_ps(HEDLEY_REINTERPRET_CAST(float*, mem_addr), mask, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_min_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_min_ps(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_min_ps(a_.m128[0], b_.m128[0]); + r_.m128[1] = simde_mm_min_ps(a_.m128[1], b_.m128[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = (a_.f32[i] < b_.f32[i]) ? a_.f32[i] : b_.f32[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_min_ps + #define _mm256_min_ps(a, b) simde_mm256_min_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_min_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_min_pd(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_min_pd(a_.m128d[0], b_.m128d[0]); + r_.m128d[1] = simde_mm_min_pd(a_.m128d[1], b_.m128d[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = (a_.f64[i] < b_.f64[i]) ? a_.f64[i] : b_.f64[i]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_min_pd + #define _mm256_min_pd(a, b) simde_mm256_min_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_max_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_max_ps(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_max_ps(a_.m128[0], b_.m128[0]); + r_.m128[1] = simde_mm_max_ps(a_.m128[1], b_.m128[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = (a_.f32[i] > b_.f32[i]) ? a_.f32[i] : b_.f32[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_max_ps + #define _mm256_max_ps(a, b) simde_mm256_max_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_max_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_max_pd(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_max_pd(a_.m128d[0], b_.m128d[0]); + r_.m128d[1] = simde_mm_max_pd(a_.m128d[1], b_.m128d[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = (a_.f64[i] > b_.f64[i]) ? a_.f64[i] : b_.f64[i]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_max_pd + #define _mm256_max_pd(a, b) simde_mm256_max_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_movedup_pd (simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_movedup_pd(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 32, a_.f64, a_.f64, 0, 0, 2, 2); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i += 2) { + r_.f64[i] = r_.f64[i + 1] = a_.f64[i]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_movedup_pd + #define _mm256_movedup_pd(a) simde_mm256_movedup_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_movehdup_ps (simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_movehdup_ps(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 32, a_.f32, a_.f32, 1, 1, 3, 3, 5, 5, 7, 7); + #else + SIMDE_VECTORIZE + for (size_t i = 1 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i += 2) { + r_.f32[i - 1] = r_.f32[i] = a_.f32[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_movehdup_ps + #define _mm256_movehdup_ps(a) simde_mm256_movehdup_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_moveldup_ps (simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_moveldup_ps(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 32, a_.f32, a_.f32, 0, 0, 2, 2, 4, 4, 6, 6); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i += 2) { + r_.f32[i] = r_.f32[i + 1] = a_.f32[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_moveldup_ps + #define _mm256_moveldup_ps(a) simde_mm256_moveldup_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm256_movemask_ps (simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_movemask_ps(a); + #else + simde__m256_private a_ = simde__m256_to_private(a); + int r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + r |= (a_.u32[i] >> 31) << i; + } + + return r; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_movemask_ps + #define _mm256_movemask_ps(a) simde_mm256_movemask_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm256_movemask_pd (simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_movemask_pd(a); + #else + simde__m256d_private a_ = simde__m256d_to_private(a); + int r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + r |= (a_.u64[i] >> 63) << i; + } + + return r; + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_movemask_pd + #define _mm256_movemask_pd(a) simde_mm256_movemask_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mul_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_mul_ps(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_mul_ps(a_.m128[0], b_.m128[0]); + r_.m128[1] = simde_mm_mul_ps(a_.m128[1], b_.m128[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = a_.f32 * b_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[i] * b_.f32[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_mul_ps + #define _mm256_mul_ps(a, b) simde_mm256_mul_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mul_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_mul_pd(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_mul_pd(a_.m128d[0], b_.m128d[0]); + r_.m128d[1] = simde_mm_mul_pd(a_.m128d[1], b_.m128d[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = a_.f64 * b_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[i] * b_.f64[i]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_mul_pd + #define _mm256_mul_pd(a, b) simde_mm256_mul_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_or_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_or_ps(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_or_ps(a_.m128[0], b_.m128[0]); + r_.m128[1] = simde_mm_or_ps(a_.m128[1], b_.m128[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f | b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] | b_.u32[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_or_ps + #define _mm256_or_ps(a, b) simde_mm256_or_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_or_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_or_pd(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_or_pd(a_.m128d[0], b_.m128d[0]); + r_.m128d[1] = simde_mm_or_pd(a_.m128d[1], b_.m128d[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f | b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = a_.u64[i] | b_.u64[i]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_or_pd + #define _mm256_or_pd(a, b) simde_mm256_or_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_permute_ps (simde__m256 a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.m128_private[i >> 2].f32[(imm8 >> ((i << 1) & 7)) & 3]; + } + + return simde__m256_from_private(r_); +} +#if defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm256_permute_ps(a, imm8) _mm256_permute_ps(a, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_permute_ps + #define _mm256_permute_ps(a, imm8) simde_mm256_permute_ps(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_permute_pd (simde__m256d a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[((imm8 >> i) & 1) + (i & 2)]; + } + + return simde__m256d_from_private(r_); +} +#if defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm256_permute_pd(a, imm8) _mm256_permute_pd(a, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_permute_pd + #define _mm256_permute_pd(a, imm8) simde_mm256_permute_pd(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_permute_ps (simde__m128 a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[(imm8 >> ((i << 1) & 7)) & 3]; + } + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm_permute_ps(a, imm8) _mm_permute_ps(a, imm8) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) +# define simde_mm_permute_ps(a, imm8) simde__m128_from_wasm_v128(wasm_i32x4_shuffle(simde__m128_to_wasm_v128(a), simde__m128_to_wasm_v128(a), ((imm8) & 3), (((imm8) >> 2) & 3 ), (((imm8) >> 4) & 3), (((imm8) >> 6) & 3))) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_permute_ps + #define _mm_permute_ps(a, imm8) simde_mm_permute_ps(a, imm8) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_permute_pd (simde__m128d a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[((imm8 >> i) & 1) + (i & 2)]; + } + + return simde__m128d_from_private(r_); +} +#if defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm_permute_pd(a, imm8) _mm_permute_pd(a, imm8) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) +# define simde_mm_permute_pd(a, imm8) simde__m128d_from_wasm_v128(wasm_i64x2_shuffle(simde__m128d_to_wasm_v128(a), simde__m128d_to_wasm_v128(a), ((imm8) & 1), (((imm8) >> 1) & 1 ))) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_permute_pd + #define _mm_permute_pd(a, imm8) simde_mm_permute_pd(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_permutevar_ps (simde__m128 a, simde__m128i b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm_permutevar_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + simde__m128i_private b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_make( + (a_.f32[wasm_i32x4_extract_lane(b_.wasm_v128, 0) & 3]), + (a_.f32[wasm_i32x4_extract_lane(b_.wasm_v128, 1) & 3]), + (a_.f32[wasm_i32x4_extract_lane(b_.wasm_v128, 2) & 3]), + (a_.f32[wasm_i32x4_extract_lane(b_.wasm_v128, 3) & 3])); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[b_.i32[i] & 3]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_permutevar_ps + #define _mm_permutevar_ps(a, b) simde_mm_permutevar_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_permutevar_pd (simde__m128d a, simde__m128i b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm_permutevar_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + simde__m128i_private b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_make( + (a_.f64[(wasm_i64x2_extract_lane(b_.wasm_v128, 0) >> 1) & 1]), + (a_.f64[(wasm_i64x2_extract_lane(b_.wasm_v128, 1) >> 1) & 1])); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[(b_.i64[i] & 2) >> 1]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_permutevar_pd + #define _mm_permutevar_pd(a, b) simde_mm_permutevar_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_permutevar_ps (simde__m256 a, simde__m256i b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_permutevar_ps(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + simde__m256i_private b_ = simde__m256i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[(b_.i32[i] & 3) + (i & 4)]; + } + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutevar_ps + #define _mm256_permutevar_ps(a, b) simde_mm256_permutevar_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_permutevar_pd (simde__m256d a, simde__m256i b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_permutevar_pd(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + simde__m256i_private b_ = simde__m256i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[((b_.i64[i] & 2) >> 1) + (i & 2)]; + } + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutevar_pd + #define _mm256_permutevar_pd(a, b) simde_mm256_permutevar_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_permute2f128_ps (simde__m256 a, simde__m256 b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + r_.m128_private[0] = (imm8 & 0x08) ? simde__m128_to_private(simde_mm_setzero_ps()) : ((imm8 & 0x02) ? b_.m128_private[(imm8 ) & 1] : a_.m128_private[(imm8 ) & 1]); + r_.m128_private[1] = (imm8 & 0x80) ? simde__m128_to_private(simde_mm_setzero_ps()) : ((imm8 & 0x20) ? b_.m128_private[(imm8 >> 4) & 1] : a_.m128_private[(imm8 >> 4) & 1]); + + return simde__m256_from_private(r_); +} +#if defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm256_permute2f128_ps(a, b, imm8) _mm256_permute2f128_ps(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_permute2f128_ps + #define _mm256_permute2f128_ps(a, b, imm8) simde_mm256_permute2f128_ps(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_permute2f128_pd (simde__m256d a, simde__m256d b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + r_.m128d_private[0] = (imm8 & 0x08) ? simde__m128d_to_private(simde_mm_setzero_pd()) : ((imm8 & 0x02) ? b_.m128d_private[(imm8 ) & 1] : a_.m128d_private[(imm8 ) & 1]); + r_.m128d_private[1] = (imm8 & 0x80) ? simde__m128d_to_private(simde_mm_setzero_pd()) : ((imm8 & 0x20) ? b_.m128d_private[(imm8 >> 4) & 1] : a_.m128d_private[(imm8 >> 4) & 1]); + + return simde__m256d_from_private(r_); +} +#if defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm256_permute2f128_pd(a, b, imm8) _mm256_permute2f128_pd(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_permute2f128_pd + #define _mm256_permute2f128_pd(a, b, imm8) simde_mm256_permute2f128_pd(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_permute2f128_si256 (simde__m256i a, simde__m256i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + r_.m128i_private[0] = (imm8 & 0x08) ? simde__m128i_to_private(simde_mm_setzero_si128()) : ((imm8 & 0x02) ? b_.m128i_private[(imm8 ) & 1] : a_.m128i_private[(imm8 ) & 1]); + r_.m128i_private[1] = (imm8 & 0x80) ? simde__m128i_to_private(simde_mm_setzero_si128()) : ((imm8 & 0x20) ? b_.m128i_private[(imm8 >> 4) & 1] : a_.m128i_private[(imm8 >> 4) & 1]); + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX_NATIVE) +# define simde_mm256_permute2f128_si128(a, b, imm8) _mm256_permute2f128_si128(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_permute2f128_si256 + #define _mm256_permute2f128_si256(a, b, imm8) simde_mm256_permute2f128_si256(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_rcp_ps (simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_rcp_ps(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_rcp_ps(a_.m128[0]); + r_.m128[1] = simde_mm_rcp_ps(a_.m128[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = SIMDE_FLOAT32_C(1.0) / a_.f32[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_rcp_ps + #define _mm256_rcp_ps(a) simde_mm256_rcp_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_rsqrt_ps (simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_rsqrt_ps(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if defined(simde_math_sqrtf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = 1.0f / simde_math_sqrtf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_rsqrt_ps + #define _mm256_rsqrt_ps(a) simde_mm256_rsqrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_setr_epi8 ( + int8_t e31, int8_t e30, int8_t e29, int8_t e28, int8_t e27, int8_t e26, int8_t e25, int8_t e24, + int8_t e23, int8_t e22, int8_t e21, int8_t e20, int8_t e19, int8_t e18, int8_t e17, int8_t e16, + int8_t e15, int8_t e14, int8_t e13, int8_t e12, int8_t e11, int8_t e10, int8_t e9, int8_t e8, + int8_t e7, int8_t e6, int8_t e5, int8_t e4, int8_t e3, int8_t e2, int8_t e1, int8_t e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_setr_epi8( + e31, e30, e29, e28, e27, e26, e25, e24, + e23, e22, e21, e20, e19, e18, e17, e16, + e15, e14, e13, e12, e11, e10, e9, e8, + e7, e6, e5, e4, e3, e2, e1, e0); + #else + return simde_mm256_set_epi8( + e0, e1, e2, e3, e4, e5, e6, e7, + e8, e9, e10, e11, e12, e13, e14, e15, + e16, e17, e18, e19, e20, e21, e22, e23, + e24, e25, e26, e27, e28, e29, e30, e31); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_setr_epi8 + #define _mm256_setr_epi8(e31, e30, e29, e28, e27, e26, e25, e24, e23, e22, e21, e20, e19, e18, e17, e16, e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) \ + simde_mm256_setr_epi8(e31, e30, e29, e28, e27, e26, e25, e24, e23, e22, e21, e20, e19, e18, e17, e16, e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_setr_epi16 ( + int16_t e15, int16_t e14, int16_t e13, int16_t e12, int16_t e11, int16_t e10, int16_t e9, int16_t e8, + int16_t e7, int16_t e6, int16_t e5, int16_t e4, int16_t e3, int16_t e2, int16_t e1, int16_t e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_setr_epi16( + e15, e14, e13, e12, e11, e10, e9, e8, + e7, e6, e5, e4, e3, e2, e1, e0); + #else + return simde_mm256_set_epi16( + e0, e1, e2, e3, e4, e5, e6, e7, + e8, e9, e10, e11, e12, e13, e14, e15); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_setr_epi16 + #define _mm256_setr_epi16(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) \ + simde_mm256_setr_epi16(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_setr_epi32 ( + int32_t e7, int32_t e6, int32_t e5, int32_t e4, int32_t e3, int32_t e2, int32_t e1, int32_t e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_setr_epi32(e7, e6, e5, e4, e3, e2, e1, e0); + #else + return simde_mm256_set_epi32(e0, e1, e2, e3, e4, e5, e6, e7); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_setr_epi32 + #define _mm256_setr_epi32(e7, e6, e5, e4, e3, e2, e1, e0) \ + simde_mm256_setr_epi32(e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_setr_epi64x (int64_t e3, int64_t e2, int64_t e1, int64_t e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_setr_epi64x(e3, e2, e1, e0); + #else + return simde_mm256_set_epi64x(e0, e1, e2, e3); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_setr_epi64x + #define _mm256_setr_epi64x(e3, e2, e1, e0) \ + simde_mm256_setr_epi64x(e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_setr_ps ( + simde_float32 e7, simde_float32 e6, simde_float32 e5, simde_float32 e4, + simde_float32 e3, simde_float32 e2, simde_float32 e1, simde_float32 e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_setr_ps(e7, e6, e5, e4, e3, e2, e1, e0); + #else + return simde_mm256_set_ps(e0, e1, e2, e3, e4, e5, e6, e7); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_setr_ps + #define _mm256_setr_ps(e7, e6, e5, e4, e3, e2, e1, e0) \ + simde_mm256_setr_ps(e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_setr_pd (simde_float64 e3, simde_float64 e2, simde_float64 e1, simde_float64 e0) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_setr_pd(e3, e2, e1, e0); + #else + return simde_mm256_set_pd(e0, e1, e2, e3); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_setr_pd + #define _mm256_setr_pd(e3, e2, e1, e0) \ + simde_mm256_setr_pd(e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_setr_m128 (simde__m128 lo, simde__m128 hi) { + #if defined(SIMDE_X86_AVX_NATIVE) && \ + !defined(SIMDE_BUG_GCC_REV_247851) && \ + SIMDE_DETECT_CLANG_VERSION_CHECK(3,6,0) + return _mm256_setr_m128(lo, hi); + #else + return simde_mm256_set_m128(hi, lo); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_setr_m128 + #define _mm256_setr_m128(lo, hi) \ + simde_mm256_setr_m128(lo, hi) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_setr_m128d (simde__m128d lo, simde__m128d hi) { + #if defined(SIMDE_X86_AVX_NATIVE) && \ + !defined(SIMDE_BUG_GCC_REV_247851) && \ + SIMDE_DETECT_CLANG_VERSION_CHECK(3,6,0) + return _mm256_setr_m128d(lo, hi); + #else + return simde_mm256_set_m128d(hi, lo); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_setr_m128d + #define _mm256_setr_m128d(lo, hi) \ + simde_mm256_setr_m128d(lo, hi) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_setr_m128i (simde__m128i lo, simde__m128i hi) { + #if defined(SIMDE_X86_AVX_NATIVE) && \ + !defined(SIMDE_BUG_GCC_REV_247851) && \ + SIMDE_DETECT_CLANG_VERSION_CHECK(3,6,0) + return _mm256_setr_m128i(lo, hi); + #else + return simde_mm256_set_m128i(hi, lo); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_setr_m128i + #define _mm256_setr_m128i(lo, hi) \ + simde_mm256_setr_m128i(lo, hi) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_shuffle_ps (simde__m256 a, simde__m256 b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + r_.f32[0] = a_.m128_private[0].f32[(imm8 >> 0) & 3]; + r_.f32[1] = a_.m128_private[0].f32[(imm8 >> 2) & 3]; + r_.f32[2] = b_.m128_private[0].f32[(imm8 >> 4) & 3]; + r_.f32[3] = b_.m128_private[0].f32[(imm8 >> 6) & 3]; + r_.f32[4] = a_.m128_private[1].f32[(imm8 >> 0) & 3]; + r_.f32[5] = a_.m128_private[1].f32[(imm8 >> 2) & 3]; + r_.f32[6] = b_.m128_private[1].f32[(imm8 >> 4) & 3]; + r_.f32[7] = b_.m128_private[1].f32[(imm8 >> 6) & 3]; + + return simde__m256_from_private(r_); +} +#if defined(SIMDE_X86_AVX_NATIVE) + #define simde_mm256_shuffle_ps(a, b, imm8) _mm256_shuffle_ps(a, b, imm8) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) + #define simde_mm256_shuffle_ps(a, b, imm8) \ + simde_mm256_set_m128( \ + simde_mm_shuffle_ps(simde_mm256_extractf128_ps(a, 1), simde_mm256_extractf128_ps(b, 1), (imm8)), \ + simde_mm_shuffle_ps(simde_mm256_extractf128_ps(a, 0), simde_mm256_extractf128_ps(b, 0), (imm8))) +#elif defined(SIMDE_SHUFFLE_VECTOR_) + #define simde_mm256_shuffle_ps(a, b, imm8) \ + SIMDE_SHUFFLE_VECTOR_(32, 32, a, b, \ + (((imm8) >> 0) & 3) + 0, \ + (((imm8) >> 2) & 3) + 0, \ + (((imm8) >> 4) & 3) + 8, \ + (((imm8) >> 6) & 3) + 8, \ + (((imm8) >> 0) & 3) + 4, \ + (((imm8) >> 2) & 3) + 4, \ + (((imm8) >> 4) & 3) + 12, \ + (((imm8) >> 6) & 3) + 12) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_shuffle_ps + #define _mm256_shuffle_ps(a, b, imm8) simde_mm256_shuffle_ps(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_shuffle_pd (simde__m256d a, simde__m256d b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + r_.f64[0] = a_.f64[((imm8 ) & 1) ]; + r_.f64[1] = b_.f64[((imm8 >> 1) & 1) ]; + r_.f64[2] = a_.f64[((imm8 >> 2) & 1) | 2]; + r_.f64[3] = b_.f64[((imm8 >> 3) & 1) | 2]; + + return simde__m256d_from_private(r_); +} +#if defined(SIMDE_X86_AVX_NATIVE) + #define simde_mm256_shuffle_pd(a, b, imm8) _mm256_shuffle_pd(a, b, imm8) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) + #define simde_mm256_shuffle_pd(a, b, imm8) \ + simde_mm256_set_m128d( \ + simde_mm_shuffle_pd(simde_mm256_extractf128_pd(a, 1), simde_mm256_extractf128_pd(b, 1), (imm8 >> 2) & 3), \ + simde_mm_shuffle_pd(simde_mm256_extractf128_pd(a, 0), simde_mm256_extractf128_pd(b, 0), (imm8 >> 0) & 3)) +#elif defined(SIMDE_SHUFFLE_VECTOR_) + #define simde_mm256_shuffle_pd(a, b, imm8) \ + SIMDE_SHUFFLE_VECTOR_(64, 32, a, b, \ + (((imm8) >> 0) & 1) + 0, \ + (((imm8) >> 1) & 1) + 4, \ + (((imm8) >> 2) & 1) + 2, \ + (((imm8) >> 3) & 1) + 6) +#endif +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_shuffle_pd + #define _mm256_shuffle_pd(a, b, imm8) simde_mm256_shuffle_pd(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_sqrt_ps (simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_sqrt_ps(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_sqrt_ps(a_.m128[0]); + r_.m128[1] = simde_mm_sqrt_ps(a_.m128[1]); + #elif defined(simde_math_sqrtf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_sqrtf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_sqrt_ps + #define _mm256_sqrt_ps(a) simde_mm256_sqrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_sqrt_pd (simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_sqrt_pd(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_sqrt_pd(a_.m128d[0]); + r_.m128d[1] = simde_mm_sqrt_pd(a_.m128d[1]); + #elif defined(simde_math_sqrt) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_sqrt(a_.f64[i]); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_sqrt_pd + #define _mm256_sqrt_pd(a) simde_mm256_sqrt_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_store_ps (simde_float32 mem_addr[8], simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + _mm256_store_ps(mem_addr, a); + #else + simde_memcpy(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m256), &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_store_ps + #define _mm256_store_ps(mem_addr, a) simde_mm256_store_ps(HEDLEY_REINTERPRET_CAST(float*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_store_pd (simde_float64 mem_addr[4], simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + _mm256_store_pd(mem_addr, a); + #else + simde_memcpy(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m256d), &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_store_pd + #define _mm256_store_pd(mem_addr, a) simde_mm256_store_pd(HEDLEY_REINTERPRET_CAST(double*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_store_si256 (simde__m256i* mem_addr, simde__m256i a) { + #if defined(SIMDE_X86_AVX_NATIVE) + _mm256_store_si256(mem_addr, a); + #else + simde_memcpy(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m256i), &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_store_si256 + #define _mm256_store_si256(mem_addr, a) simde_mm256_store_si256(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_storeu_ps (simde_float32 mem_addr[8], simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + _mm256_storeu_ps(mem_addr, a); + #else + simde_memcpy(mem_addr, &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_storeu_ps + #define _mm256_storeu_ps(mem_addr, a) simde_mm256_storeu_ps(HEDLEY_REINTERPRET_CAST(float*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_storeu_pd (simde_float64 mem_addr[4], simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + _mm256_storeu_pd(mem_addr, a); + #else + simde_memcpy(mem_addr, &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_storeu_pd + #define _mm256_storeu_pd(mem_addr, a) simde_mm256_storeu_pd(HEDLEY_REINTERPRET_CAST(double*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_storeu_si256 (void* mem_addr, simde__m256i a) { + #if defined(SIMDE_X86_AVX_NATIVE) + _mm256_storeu_si256(SIMDE_ALIGN_CAST(__m256i*, mem_addr), a); + #else + simde_memcpy(mem_addr, &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_storeu_si256 + #define _mm256_storeu_si256(mem_addr, a) simde_mm256_storeu_si256(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_storeu2_m128 (simde_float32 hi_addr[4], simde_float32 lo_addr[4], simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) && !defined(SIMDE_BUG_GCC_91341) && !defined(SIMDE_BUG_MCST_LCC_MISSING_AVX_LOAD_STORE_M128_FUNCS) + _mm256_storeu2_m128(hi_addr, lo_addr, a); + #else + simde_mm_storeu_ps(lo_addr, simde_mm256_castps256_ps128(a)); + simde_mm_storeu_ps(hi_addr, simde_mm256_extractf128_ps(a, 1)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_storeu2_m128 + #define _mm256_storeu2_m128(hi_addr, lo_addr, a) simde_mm256_storeu2_m128(hi_addr, lo_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_storeu2_m128d (simde_float64 hi_addr[2], simde_float64 lo_addr[2], simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) && !defined(SIMDE_BUG_GCC_91341) && !defined(SIMDE_BUG_MCST_LCC_MISSING_AVX_LOAD_STORE_M128_FUNCS) + _mm256_storeu2_m128d(hi_addr, lo_addr, a); + #else + simde_mm_storeu_pd(lo_addr, simde_mm256_castpd256_pd128(a)); + simde_mm_storeu_pd(hi_addr, simde_mm256_extractf128_pd(a, 1)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_storeu2_m128d + #define _mm256_storeu2_m128d(hi_addr, lo_addr, a) simde_mm256_storeu2_m128d(hi_addr, lo_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_storeu2_m128i (simde__m128i* hi_addr, simde__m128i* lo_addr, simde__m256i a) { + #if defined(SIMDE_X86_AVX_NATIVE) && !defined(SIMDE_BUG_GCC_91341) && !defined(SIMDE_BUG_MCST_LCC_MISSING_AVX_LOAD_STORE_M128_FUNCS) + _mm256_storeu2_m128i(hi_addr, lo_addr, a); + #else + simde_mm_storeu_si128(lo_addr, simde_mm256_castsi256_si128(a)); + simde_mm_storeu_si128(hi_addr, simde_mm256_extractf128_si256(a, 1)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_storeu2_m128i + #define _mm256_storeu2_m128i(hi_addr, lo_addr, a) simde_mm256_storeu2_m128i(hi_addr, lo_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_stream_ps (simde_float32 mem_addr[8], simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + _mm256_stream_ps(mem_addr, a); + #elif HEDLEY_HAS_BUILTIN(__builtin_nontemporal_store) && defined(SIMDE_VECTOR_SUBSCRIPT) + __builtin_nontemporal_store(a, SIMDE_ALIGN_CAST(__typeof__(a)*, mem_addr)); + #else + simde_memcpy(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m256), &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_stream_ps + #define _mm256_stream_ps(mem_addr, a) simde_mm256_stream_ps(HEDLEY_REINTERPRET_CAST(float*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_stream_pd (simde_float64 mem_addr[4], simde__m256d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + _mm256_stream_pd(mem_addr, a); + #elif HEDLEY_HAS_BUILTIN(__builtin_nontemporal_store) && defined(SIMDE_VECTOR_SUBSCRIPT) + __builtin_nontemporal_store(a, SIMDE_ALIGN_CAST(__typeof__(a)*, mem_addr)); + #else + simde_memcpy(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m256d), &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_stream_pd + #define _mm256_stream_pd(mem_addr, a) simde_mm256_stream_pd(HEDLEY_REINTERPRET_CAST(double*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_stream_si256 (simde__m256i* mem_addr, simde__m256i a) { + #if defined(SIMDE_X86_AVX_NATIVE) + _mm256_stream_si256(mem_addr, a); + #elif HEDLEY_HAS_BUILTIN(__builtin_nontemporal_store) && defined(SIMDE_VECTOR_SUBSCRIPT) + __builtin_nontemporal_store(a, SIMDE_ALIGN_CAST(__typeof__(a)*, mem_addr)); + #else + simde_memcpy(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m256i), &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_stream_si256 + #define _mm256_stream_si256(mem_addr, a) simde_mm256_stream_si256(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_sub_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_sub_ps(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_sub_ps(a_.m128[0], b_.m128[0]); + r_.m128[1] = simde_mm_sub_ps(a_.m128[1], b_.m128[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = a_.f32 - b_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[i] - b_.f32[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_sub_ps + #define _mm256_sub_ps(a, b) simde_mm256_sub_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_hsub_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_hsub_ps(a, b); + #else + return simde_mm256_sub_ps(simde_x_mm256_deinterleaveeven_ps(a, b), simde_x_mm256_deinterleaveodd_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_hsub_ps + #define _mm256_hsub_ps(a, b) simde_mm256_hsub_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_sub_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_sub_pd(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_sub_pd(a_.m128d[0], b_.m128d[0]); + r_.m128d[1] = simde_mm_sub_pd(a_.m128d[1], b_.m128d[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = a_.f64 - b_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[i] - b_.f64[i]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_sub_pd + #define _mm256_sub_pd(a, b) simde_mm256_sub_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_hsub_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_hsub_pd(a, b); + #else + return simde_mm256_sub_pd(simde_x_mm256_deinterleaveeven_pd(a, b), simde_x_mm256_deinterleaveodd_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_hsub_pd + #define _mm256_hsub_pd(a, b) simde_mm256_hsub_pd(a, b) +#endif + +#if defined(SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_) + HEDLEY_DIAGNOSTIC_PUSH + SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_ +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_undefined_ps (void) { + simde__m256_private r_; + +#if \ + defined(SIMDE_X86_AVX_NATIVE) && \ + (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(5,0,0)) && \ + (!defined(__has_builtin) || HEDLEY_HAS_BUILTIN(__builtin_ia32_undef256)) + r_.n = _mm256_undefined_ps(); +#elif !defined(SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_) + r_ = simde__m256_to_private(simde_mm256_setzero_ps()); +#endif + + return simde__m256_from_private(r_); +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_undefined_ps + #define _mm256_undefined_ps() simde_mm256_undefined_ps() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_undefined_pd (void) { + simde__m256d_private r_; + +#if \ + defined(SIMDE_X86_AVX_NATIVE) && \ + (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(5,0,0)) && \ + (!defined(__has_builtin) || HEDLEY_HAS_BUILTIN(__builtin_ia32_undef256)) + r_.n = _mm256_undefined_pd(); +#elif !defined(SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_) + r_ = simde__m256d_to_private(simde_mm256_setzero_pd()); +#endif + + return simde__m256d_from_private(r_); +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_undefined_pd + #define _mm256_undefined_pd() simde_mm256_undefined_pd() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_undefined_si256 (void) { + simde__m256i_private r_; +#if \ + defined(SIMDE_X86_AVX_NATIVE) && \ + (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(5,0,0)) && \ + (!defined(__has_builtin) || HEDLEY_HAS_BUILTIN(__builtin_ia32_undef256)) + r_.n = _mm256_undefined_si256(); +#elif !defined(SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_) + r_ = simde__m256i_to_private(simde_mm256_setzero_si256()); +#endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_undefined_si256 + #define _mm256_undefined_si256() simde_mm256_undefined_si256() +#endif + +#if defined(SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_) + HEDLEY_DIAGNOSTIC_POP +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_xor_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_xor_ps(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128[0] = simde_mm_xor_ps(a_.m128[0], b_.m128[0]); + r_.m128[1] = simde_mm_xor_ps(a_.m128[1], b_.m128[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f ^ b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] ^ b_.u32[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_xor_ps + #define _mm256_xor_ps(a, b) simde_mm256_xor_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_xor_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_xor_pd(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128d[0] = simde_mm_xor_pd(a_.m128d[0], b_.m128d[0]); + r_.m128d[1] = simde_mm_xor_pd(a_.m128d[1], b_.m128d[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f ^ b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = a_.u64[i] ^ b_.u64[i]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_xor_pd + #define _mm256_xor_pd(a, b) simde_mm256_xor_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_x_mm256_xorsign_ps(simde__m256 dest, simde__m256 src) { + return simde_mm256_xor_ps(simde_mm256_and_ps(simde_mm256_set1_ps(-0.0f), src), dest); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_x_mm256_xorsign_pd(simde__m256d dest, simde__m256d src) { + return simde_mm256_xor_pd(simde_mm256_and_pd(simde_mm256_set1_pd(-0.0), src), dest); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_x_mm256_negate_ps(simde__m256 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return simde_mm256_xor_ps(a,_mm256_set1_ps(SIMDE_FLOAT32_C(-0.0))); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if defined(SIMDE_VECTOR_NEGATE) + r_.f32 = -a_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = -a_.f32[i]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_x_mm256_negate_pd(simde__m256d a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return simde_mm256_xor_pd(a, _mm256_set1_pd(SIMDE_FLOAT64_C(-0.0))); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if defined(SIMDE_VECTOR_NEGATE) + r_.f64 = -a_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = -a_.f64[i]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_unpackhi_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_unpackhi_ps(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 32, a_.f32, b_.f32, 2, 10, 3, 11, 6, 14, 7, 15); + #else + r_.f32[0] = a_.f32[2]; + r_.f32[1] = b_.f32[2]; + r_.f32[2] = a_.f32[3]; + r_.f32[3] = b_.f32[3]; + r_.f32[4] = a_.f32[6]; + r_.f32[5] = b_.f32[6]; + r_.f32[6] = a_.f32[7]; + r_.f32[7] = b_.f32[7]; + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_unpackhi_ps + #define _mm256_unpackhi_ps(a, b) simde_mm256_unpackhi_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_unpackhi_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_unpackhi_pd(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 32, a_.f64, b_.f64, 1, 5, 3, 7); + #else + r_.f64[0] = a_.f64[1]; + r_.f64[1] = b_.f64[1]; + r_.f64[2] = a_.f64[3]; + r_.f64[3] = b_.f64[3]; + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_unpackhi_pd + #define _mm256_unpackhi_pd(a, b) simde_mm256_unpackhi_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_unpacklo_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_unpacklo_ps(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 32, a_.f32, b_.f32, 0, 8, 1, 9, 4, 12, 5, 13); + #else + r_.f32[0] = a_.f32[0]; + r_.f32[1] = b_.f32[0]; + r_.f32[2] = a_.f32[1]; + r_.f32[3] = b_.f32[1]; + r_.f32[4] = a_.f32[4]; + r_.f32[5] = b_.f32[4]; + r_.f32[6] = a_.f32[5]; + r_.f32[7] = b_.f32[5]; + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_unpacklo_ps + #define _mm256_unpacklo_ps(a, b) simde_mm256_unpacklo_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_unpacklo_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_unpacklo_pd(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 32, a_.f64, b_.f64, 0, 4, 2, 6); + #else + r_.f64[0] = a_.f64[0]; + r_.f64[1] = b_.f64[0]; + r_.f64[2] = a_.f64[2]; + r_.f64[3] = b_.f64[2]; + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_unpacklo_pd + #define _mm256_unpacklo_pd(a, b) simde_mm256_unpacklo_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_zextps128_ps256 (simde__m128 a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_insertf128_ps(_mm256_setzero_ps(), a, 0); + #else + simde__m256_private r_; + + r_.m128_private[0] = simde__m128_to_private(a); + r_.m128_private[1] = simde__m128_to_private(simde_mm_setzero_ps()); + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_zextps128_ps256 + #define _mm256_zextps128_ps256(a) simde_mm256_zextps128_ps256(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_zextpd128_pd256 (simde__m128d a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_insertf128_pd(_mm256_setzero_pd(), a, 0); + #else + simde__m256d_private r_; + + r_.m128d_private[0] = simde__m128d_to_private(a); + r_.m128d_private[1] = simde__m128d_to_private(simde_mm_setzero_pd()); + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_zextpd128_pd256 + #define _mm256_zextpd128_pd256(a) simde_mm256_zextpd128_pd256(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_zextsi128_si256 (simde__m128i a) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_insertf128_si256(_mm256_setzero_si256(), a, 0); + #else + simde__m256i_private r_; + + r_.m128i_private[0] = simde__m128i_to_private(a); + r_.m128i_private[1] = simde__m128i_to_private(simde_mm_setzero_si128()); + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_zextsi128_si256 + #define _mm256_zextsi128_si256(a) simde_mm256_zextsi128_si256(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_testc_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm_testc_ps(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m = wasm_u32x4_shr(wasm_v128_or(wasm_v128_not(b_.wasm_v128), a_.wasm_v128), 31); + m = wasm_v128_and(m, simde_mm_movehl_ps(m, m)); + m = wasm_v128_and(m, simde_mm_shuffle_epi32(m, SIMDE_MM_SHUFFLE(3, 2, 0, 1))); + return wasm_i32x4_extract_lane(m, 0); + #else + uint_fast32_t r = 0; + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + r |= ~a_.u32[i] & b_.u32[i]; + } + + return HEDLEY_STATIC_CAST(int, ((~r >> 31) & 1)); + #endif + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_testc_ps + #define _mm_testc_ps(a, b) simde_mm_testc_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_testc_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm_testc_pd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m = wasm_u64x2_shr(wasm_v128_or(wasm_v128_not(b_.wasm_v128), a_.wasm_v128), 63); + return HEDLEY_STATIC_CAST(int, wasm_i64x2_extract_lane(m, 0) & wasm_i64x2_extract_lane(m, 1)); + #else + uint_fast64_t r = 0; + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + r |= ~a_.u64[i] & b_.u64[i]; + } + + return HEDLEY_STATIC_CAST(int, ((~r >> 63) & 1)); + #endif + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_testc_pd + #define _mm_testc_pd(a, b) simde_mm_testc_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm256_testc_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_testc_ps(a, b); + #else + uint_fast32_t r = 0; + simde__m256_private + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + r |= ~a_.u32[i] & b_.u32[i]; + } + + return HEDLEY_STATIC_CAST(int, ((~r >> 31) & 1)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_testc_ps + #define _mm256_testc_ps(a, b) simde_mm256_testc_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm256_testc_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_testc_pd(a, b); + #else + uint_fast64_t r = 0; + simde__m256d_private + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + r |= ~a_.u64[i] & b_.u64[i]; + } + + return HEDLEY_STATIC_CAST(int, ((~r >> 63) & 1)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_testc_pd + #define _mm256_testc_pd(a, b) simde_mm256_testc_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm256_testc_si256 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_testc_si256(a, b); + #else + int_fast32_t r = 0; + simde__m256i_private + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i32f) / sizeof(a_.i32f[0])) ; i++) { + r |= ~a_.i32f[i] & b_.i32f[i]; + } + + return HEDLEY_STATIC_CAST(int, !r); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_testc_si256 + #define _mm256_testc_si256(a, b) simde_mm256_testc_si256(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_testz_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm_testz_ps(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m = wasm_u32x4_shr(wasm_v128_not(wasm_v128_and(a_.wasm_v128, b_.wasm_v128)), 31); + m = wasm_v128_and(m, simde_mm_movehl_ps(m, m)); + m = wasm_v128_and(m, simde_mm_shuffle_epi32(m, SIMDE_MM_SHUFFLE(3, 2, 0, 1))); + return wasm_i32x4_extract_lane(m, 0); + #else + uint_fast32_t r = 0; + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + r |= a_.u32[i] & b_.u32[i]; + } + + return HEDLEY_STATIC_CAST(int, ((~r >> 31) & 1)); + #endif + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_testz_ps + #define _mm_testz_ps(a, b) simde_mm_testz_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_testz_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm_testz_pd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m = wasm_u64x2_shr(wasm_v128_not(wasm_v128_and(a_.wasm_v128, b_.wasm_v128)), 63); + return HEDLEY_STATIC_CAST(int, wasm_i64x2_extract_lane(m, 0) & wasm_i64x2_extract_lane(m, 1)); + #else + uint_fast64_t r = 0; + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + r |= a_.u64[i] & b_.u64[i]; + } + + return HEDLEY_STATIC_CAST(int, ((~r >> 63) & 1)); + #endif + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_testz_pd + #define _mm_testz_pd(a, b) simde_mm_testz_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm256_testz_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_testz_ps(a, b); + #else + uint_fast32_t r = 0; + simde__m256_private + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + r |= a_.u32[i] & b_.u32[i]; + } + + return HEDLEY_STATIC_CAST(int, ((~r >> 31) & 1)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_testz_ps + #define _mm256_testz_ps(a, b) simde_mm256_testz_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm256_testz_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_testz_pd(a, b); + #else + uint_fast64_t r = 0; + simde__m256d_private + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + r |= a_.u64[i] & b_.u64[i]; + } + + return HEDLEY_STATIC_CAST(int, ((~r >> 63) & 1)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_testz_pd + #define _mm256_testz_pd(a, b) simde_mm256_testz_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm256_testz_si256 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_testz_si256(a, b); + #else + int_fast32_t r = 0; + simde__m256i_private + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r = simde_mm_testz_si128(a_.m128i[0], b_.m128i[0]) && simde_mm_testz_si128(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i32f) / sizeof(a_.i32f[0])) ; i++) { + r |= a_.i32f[i] & b_.i32f[i]; + } + + r = !r; + #endif + + return HEDLEY_STATIC_CAST(int, r); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_testz_si256 + #define _mm256_testz_si256(a, b) simde_mm256_testz_si256(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_testnzc_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm_testnzc_ps(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m = wasm_u32x4_shr(wasm_v128_and(a_.wasm_v128, b_.wasm_v128), 31); + v128_t m2 = wasm_u32x4_shr(wasm_v128_andnot(b_.wasm_v128, a_.wasm_v128), 31); + m = wasm_v128_or(m, simde_mm_movehl_ps(m, m)); + m2 = wasm_v128_or(m2, simde_mm_movehl_ps(m2, m2)); + m = wasm_v128_or(m, simde_mm_shuffle_epi32(m, SIMDE_MM_SHUFFLE(3, 2, 0, 1))); + m2 = wasm_v128_or(m2, simde_mm_shuffle_epi32(m2, SIMDE_MM_SHUFFLE(3, 2, 0, 1))); + return wasm_i32x4_extract_lane(m, 0) & wasm_i32x4_extract_lane(m2, 0); + #else + uint32_t rz = 0, rc = 0; + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + rc |= ~a_.u32[i] & b_.u32[i]; + rz |= a_.u32[i] & b_.u32[i]; + } + + return + (rc >> ((sizeof(rc) * CHAR_BIT) - 1)) & + (rz >> ((sizeof(rz) * CHAR_BIT) - 1)); + #endif + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_testnzc_ps + #define _mm_testnzc_ps(a, b) simde_mm_testnzc_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_testnzc_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm_testnzc_pd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + #if defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m = wasm_u64x2_shr(wasm_v128_and(a_.wasm_v128, b_.wasm_v128), 63); + v128_t m2 = wasm_u64x2_shr(wasm_v128_andnot(b_.wasm_v128, a_.wasm_v128), 63); + return HEDLEY_STATIC_CAST(int, (wasm_i64x2_extract_lane(m, 0) | wasm_i64x2_extract_lane(m, 1)) + & (wasm_i64x2_extract_lane(m2, 0) | wasm_i64x2_extract_lane(m2, 1))); + #else + uint64_t rc = 0, rz = 0; + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + rc |= ~a_.u64[i] & b_.u64[i]; + rz |= a_.u64[i] & b_.u64[i]; + } + + return + (rc >> ((sizeof(rc) * CHAR_BIT) - 1)) & + (rz >> ((sizeof(rz) * CHAR_BIT) - 1)); + #endif + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm_testnzc_pd + #define _mm_testnzc_pd(a, b) simde_mm_testnzc_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm256_testnzc_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_testnzc_ps(a, b); + #else + uint32_t rc = 0, rz = 0; + simde__m256_private + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + rc |= ~a_.u32[i] & b_.u32[i]; + rz |= a_.u32[i] & b_.u32[i]; + } + + return + (rc >> ((sizeof(rc) * CHAR_BIT) - 1)) & + (rz >> ((sizeof(rz) * CHAR_BIT) - 1)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_testnzc_ps + #define _mm256_testnzc_ps(a, b) simde_mm256_testnzc_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm256_testnzc_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_testnzc_pd(a, b); + #else + uint64_t rc = 0, rz = 0; + simde__m256d_private + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + rc |= ~a_.u64[i] & b_.u64[i]; + rz |= a_.u64[i] & b_.u64[i]; + } + + return + (rc >> ((sizeof(rc) * CHAR_BIT) - 1)) & + (rz >> ((sizeof(rz) * CHAR_BIT) - 1)); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_testnzc_pd + #define _mm256_testnzc_pd(a, b) simde_mm256_testnzc_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm256_testnzc_si256 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX_NATIVE) + return _mm256_testnzc_si256(a, b); + #else + int32_t rc = 0, rz = 0; + simde__m256i_private + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + for (size_t i = 0 ; i < (sizeof(a_.i32f) / sizeof(a_.i32f[0])) ; i++) { + rc |= ~a_.i32f[i] & b_.i32f[i]; + rz |= a_.i32f[i] & b_.i32f[i]; + } + + return !!(rc & rz); + #endif +} +#if defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_testnzc_si256 + #define _mm256_testnzc_si256(a, b) simde_mm256_testnzc_si256(a, b) +#endif + +SIMDE_END_DECLS_ + +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx2.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx2.h new file mode 100644 index 00000000..c01c1e8f --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx2.h @@ -0,0 +1,5758 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2018-2020 Evan Nemerson + * 2019-2020 Michael R. Crusoe + * 2020 Himanshi Mathur + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX2_H) +#define SIMDE_X86_AVX2_H + +#include "avx.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_abs_epi8 (simde__m256i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_abs_epi8(a); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_abs_epi8(a_.m128i[0]); + r_.m128i[1] = simde_mm_abs_epi8(a_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] < INT32_C(0)) ? -a_.i8[i] : a_.i8[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_abs_epi8 + #define _mm256_abs_epi8(a) simde_mm256_abs_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_abs_epi16 (simde__m256i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_abs_epi16(a); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_abs_epi16(a_.m128i[0]); + r_.m128i[1] = simde_mm_abs_epi16(a_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] < INT32_C(0)) ? -a_.i16[i] : a_.i16[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_abs_epi16 + #define _mm256_abs_epi16(a) simde_mm256_abs_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_abs_epi32(simde__m256i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_abs_epi32(a); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_abs_epi32(a_.m128i[0]); + r_.m128i[1] = simde_mm_abs_epi32(a_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0; i < (sizeof(r_.i32) / sizeof(r_.i32[0])); i++) { + r_.i32[i] = (a_.i32[i] < INT32_C(0)) ? -a_.i32[i] : a_.i32[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_abs_epi32 + #define _mm256_abs_epi32(a) simde_mm256_abs_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_add_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_add_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_add_epi8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_add_epi8(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = a_.i8 + b_.i8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] + b_.i8[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_add_epi8 + #define _mm256_add_epi8(a, b) simde_mm256_add_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_add_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_add_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_add_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_add_epi16(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = a_.i16 + b_.i16; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] + b_.i16[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_add_epi16 + #define _mm256_add_epi16(a, b) simde_mm256_add_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_hadd_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_hadd_epi16(a, b); + #else + return simde_mm256_add_epi16(simde_x_mm256_deinterleaveeven_epi16(a, b), simde_x_mm256_deinterleaveodd_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_hadd_epi16 + #define _mm256_hadd_epi16(a, b) simde_mm256_hadd_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_add_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_add_epi32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_add_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_add_epi32(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 + b_.i32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] + b_.i32[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_add_epi32 + #define _mm256_add_epi32(a, b) simde_mm256_add_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_hadd_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_hadd_epi32(a, b); + #else + return simde_mm256_add_epi32(simde_x_mm256_deinterleaveeven_epi32(a, b), simde_x_mm256_deinterleaveodd_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_hadd_epi32 + #define _mm256_hadd_epi32(a, b) simde_mm256_hadd_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_add_epi64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_add_epi64(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_add_epi64(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_add_epi64(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_CLANG_BAD_VI64_OPS) + r_.i64 = a_.i64 + b_.i64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] + b_.i64[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_add_epi64 + #define _mm256_add_epi64(a, b) simde_mm256_add_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_alignr_epi8 (simde__m256i a, simde__m256i b, int count) + SIMDE_REQUIRE_CONSTANT_RANGE(count, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + if (HEDLEY_UNLIKELY(count > 31)) + return simde_mm256_setzero_si256(); + + for (size_t h = 0 ; h < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; h++) { + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private[h].i8) / sizeof(r_.m128i_private[h].i8[0])) ; i++) { + const int srcpos = count + HEDLEY_STATIC_CAST(int, i); + if (srcpos > 31) { + r_.m128i_private[h].i8[i] = 0; + } else if (srcpos > 15) { + r_.m128i_private[h].i8[i] = a_.m128i_private[h].i8[(srcpos) & 15]; + } else { + r_.m128i_private[h].i8[i] = b_.m128i_private[h].i8[srcpos]; + } + } + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) && !defined(SIMDE_BUG_PGI_30106) +# define simde_mm256_alignr_epi8(a, b, count) _mm256_alignr_epi8(a, b, count) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) +# define simde_mm256_alignr_epi8(a, b, count) \ + simde_mm256_set_m128i( \ + simde_mm_alignr_epi8(simde_mm256_extracti128_si256(a, 1), simde_mm256_extracti128_si256(b, 1), (count)), \ + simde_mm_alignr_epi8(simde_mm256_extracti128_si256(a, 0), simde_mm256_extracti128_si256(b, 0), (count))) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_alignr_epi8 + #define _mm256_alignr_epi8(a, b, count) simde_mm256_alignr_epi8(a, b, (count)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_and_si256 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_and_si256(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_and_si128(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_and_si128(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f & b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] & b_.i64[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_and_si256 + #define _mm256_and_si256(a, b) simde_mm256_and_si256(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_andnot_si256 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_andnot_si256(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_andnot_si128(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_andnot_si128(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = ~(a_.i32f[i]) & b_.i32f[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_andnot_si256 + #define _mm256_andnot_si256(a, b) simde_mm256_andnot_si256(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_adds_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_adds_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_adds_epi8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_adds_epi8(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = simde_math_adds_i8(a_.i8[i], b_.i8[i]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_adds_epi8 + #define _mm256_adds_epi8(a, b) simde_mm256_adds_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_adds_epi16(simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_adds_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_adds_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_adds_epi16(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = simde_math_adds_i16(a_.i16[i], b_.i16[i]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_adds_epi16 + #define _mm256_adds_epi16(a, b) simde_mm256_adds_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_hadds_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_hadds_epi16(a, b); + #else + return simde_mm256_adds_epi16(simde_x_mm256_deinterleaveeven_epi16(a, b), simde_x_mm256_deinterleaveodd_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_hadds_epi16 + #define _mm256_hadds_epi16(a, b) simde_mm256_hadds_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_adds_epu8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_adds_epu8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_adds_epu8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_adds_epu8(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = simde_math_adds_u8(a_.u8[i], b_.u8[i]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_adds_epu8 + #define _mm256_adds_epu8(a, b) simde_mm256_adds_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_adds_epu16(simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_adds_epu16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_adds_epu16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_adds_epu16(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = simde_math_adds_u16(a_.u16[i], b_.u16[i]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_adds_epu16 + #define _mm256_adds_epu16(a, b) simde_mm256_adds_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_avg_epu8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_avg_epu8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] + b_.u8[i] + 1) >> 1; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_avg_epu8 + #define _mm256_avg_epu8(a, b) simde_mm256_avg_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_avg_epu16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_avg_epu16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] + b_.u16[i] + 1) >> 1; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_avg_epu16 + #define _mm256_avg_epu16(a, b) simde_mm256_avg_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_blend_epi32(simde__m128i a, simde__m128i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ((imm8 >> i) & 1) ? b_.i32[i] : a_.i32[i]; + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm_blend_epi32(a, b, imm8) _mm_blend_epi32(a, b, imm8) +#elif SIMDE_NATURAL_FLOAT_VECTOR_SIZE_LE(128) +# define simde_mm_blend_epi32(a, b, imm8) \ + simde_mm_castps_si128(simde_mm_blend_ps(simde_mm_castsi128_ps(a), simde_mm_castsi128_ps(b), (imm8))) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_blend_epi32 + #define _mm_blend_epi32(a, b, imm8) simde_mm_blend_epi32(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_blend_epi16(simde__m256i a, simde__m256i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = ((imm8 >> i%8) & 1) ? b_.i16[i] : a_.i16[i]; + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) && defined(SIMDE_BUG_CLANG_REV_234560) +# define simde_mm256_blend_epi16(a, b, imm8) _mm256_castpd_si256(_mm256_blend_epi16(a, b, imm8)) +#elif defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_blend_epi16(a, b, imm8) _mm256_blend_epi16(a, b, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) +# define simde_mm256_blend_epi16(a, b, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_blend_epi16(simde_mm256_extracti128_si256(a, 1), simde_mm256_extracti128_si256(b, 1), (imm8)), \ + simde_mm_blend_epi16(simde_mm256_extracti128_si256(a, 0), simde_mm256_extracti128_si256(b, 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_blend_epi16 + #define _mm256_blend_epi16(a, b, imm8) simde_mm256_blend_epi16(a, b, imm8) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_blend_epi32(simde__m256i a, simde__m256i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ((imm8 >> i) & 1) ? b_.i32[i] : a_.i32[i]; + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_blend_epi32(a, b, imm8) _mm256_blend_epi32(a, b, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) +# define simde_mm256_blend_epi32(a, b, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_blend_epi32(simde_mm256_extracti128_si256(a, 1), simde_mm256_extracti128_si256(b, 1), (imm8) >> 4), \ + simde_mm_blend_epi32(simde_mm256_extracti128_si256(a, 0), simde_mm256_extracti128_si256(b, 0), (imm8) & 0x0F)) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_blend_epi32 + #define _mm256_blend_epi32(a, b, imm8) simde_mm256_blend_epi32(a, b, imm8) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_blendv_epi8(simde__m256i a, simde__m256i b, simde__m256i mask) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_blendv_epi8(a, b, mask); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b), + mask_ = simde__m256i_to_private(mask); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_blendv_epi8(a_.m128i[0], b_.m128i[0], mask_.m128i[0]); + r_.m128i[1] = simde_mm_blendv_epi8(a_.m128i[1], b_.m128i[1], mask_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + __typeof__(mask_.i8) tmp = mask_.i8 >> 7; + r_.i8 = (tmp & b_.i8) | (~tmp & a_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + int8_t tmp = mask_.i8[i] >> 7; + r_.i8[i] = (tmp & b_.i8[i]) | (~tmp & a_.i8[i]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_blendv_epi8(a, b, imm8) _mm256_blendv_epi8(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_blendv_epi8 + #define _mm256_blendv_epi8(a, b, mask) simde_mm256_blendv_epi8(a, b, mask) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_broadcastb_epi8 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm_broadcastb_epi8(a); + #else + simde__m128i_private r_; + simde__m128i_private a_= simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[0]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_broadcastb_epi8 + #define _mm_broadcastb_epi8(a) simde_mm_broadcastb_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_broadcastb_epi8 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_broadcastb_epi8(a); + #else + simde__m256i_private r_; + simde__m128i_private a_= simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[0]; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_broadcastb_epi8 + #define _mm256_broadcastb_epi8(a) simde_mm256_broadcastb_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_broadcastw_epi16 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm_broadcastw_epi16(a); + #else + simde__m128i_private r_; + simde__m128i_private a_= simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[0]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_broadcastw_epi16 + #define _mm_broadcastw_epi16(a) simde_mm_broadcastw_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_broadcastw_epi16 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_broadcastw_epi16(a); + #else + simde__m256i_private r_; + simde__m128i_private a_= simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[0]; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_broadcastw_epi16 + #define _mm256_broadcastw_epi16(a) simde_mm256_broadcastw_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_broadcastd_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm_broadcastd_epi32(a); + #else + simde__m128i_private r_; + simde__m128i_private a_= simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[0]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_broadcastd_epi32 + #define _mm_broadcastd_epi32(a) simde_mm_broadcastd_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_broadcastd_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_broadcastd_epi32(a); + #else + simde__m256i_private r_; + simde__m128i_private a_= simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[0]; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_broadcastd_epi32 + #define _mm256_broadcastd_epi32(a) simde_mm256_broadcastd_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_broadcastq_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm_broadcastq_epi64(a); + #else + simde__m128i_private r_; + simde__m128i_private a_= simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[0]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_broadcastq_epi64 + #define _mm_broadcastq_epi64(a) simde_mm_broadcastq_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_broadcastq_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_broadcastq_epi64(a); + #else + simde__m256i_private r_; + simde__m128i_private a_= simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[0]; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_broadcastq_epi64 + #define _mm256_broadcastq_epi64(a) simde_mm256_broadcastq_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_broadcastss_ps (simde__m128 a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm_broadcastss_ps(a); + #elif defined(SIMDE_X86_SSE_NATIVE) + return simde_mm_shuffle_ps(a, a, 0); + #else + simde__m128_private r_; + simde__m128_private a_= simde__m128_to_private(a); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.f32, a_.f32, 0, 0, 0, 0); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[0]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_broadcastss_ps + #define _mm_broadcastss_ps(a) simde_mm_broadcastss_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_broadcastss_ps (simde__m128 a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_broadcastss_ps(a); + #else + simde__m256_private r_; + simde__m128_private a_= simde__m128_to_private(a); + + #if defined(SIMDE_X86_AVX_NATIVE) + __m128 tmp = _mm_permute_ps(a_.n, 0); + r_.n = _mm256_insertf128_ps(_mm256_castps128_ps256(tmp), tmp, 1); + #elif HEDLEY_HAS_BUILTIN(__builtin_shufflevector) + r_.f32 = __builtin_shufflevector(a_.f32, a_.f32, 0, 0, 0, 0, 0, 0, 0, 0); + #elif SIMDE_NATURAL_FLOAT_VECTOR_SIZE_LE(128) + r_.m128[0] = r_.m128[1] = simde_mm_broadcastss_ps(simde__m128_from_private(a_)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[0]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_broadcastss_ps + #define _mm256_broadcastss_ps(a) simde_mm256_broadcastss_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_broadcastsd_pd (simde__m128d a) { + return simde_mm_movedup_pd(a); +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_broadcastsd_pd + #define _mm_broadcastsd_pd(a) simde_mm_broadcastsd_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_broadcastsd_pd (simde__m128d a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_broadcastsd_pd(a); + #else + simde__m256d_private r_; + simde__m128d_private a_= simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[0]; + } + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_broadcastsd_pd + #define _mm256_broadcastsd_pd(a) simde_mm256_broadcastsd_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_broadcastsi128_si256 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) && \ + (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(4,8,0)) + return _mm256_broadcastsi128_si256(a); + #else + simde__m256i_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i_private[0] = a_; + r_.m128i_private[1] = a_; + #else + r_.i64[0] = a_.i64[0]; + r_.i64[1] = a_.i64[1]; + r_.i64[2] = a_.i64[0]; + r_.i64[3] = a_.i64[1]; + #endif + + return simde__m256i_from_private(r_); + #endif +} +#define simde_mm_broadcastsi128_si256(a) simde_mm256_broadcastsi128_si256(a) +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_broadcastsi128_si256 + #define _mm256_broadcastsi128_si256(a) simde_mm256_broadcastsi128_si256(a) + #undef _mm_broadcastsi128_si256 + #define _mm_broadcastsi128_si256(a) simde_mm256_broadcastsi128_si256(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_bslli_epi128 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + const int ssize = HEDLEY_STATIC_CAST(int, (sizeof(r_.i8) / sizeof(r_.i8[0]))); + + SIMDE_VECTORIZE + for (int i = 0 ; i < ssize ; i++) { + const int e = i - imm8; + if(i >= (ssize/2)) { + if(e >= (ssize/2) && e < ssize) + r_.i8[i] = a_.i8[e]; + else + r_.i8[i] = 0; + } + else{ + if(e >= 0 && e < (ssize/2)) + r_.i8[i] = a_.i8[e]; + else + r_.i8[i] = 0; + } + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) && \ + (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(4,8,0)) && \ + SIMDE_DETECT_CLANG_VERSION_CHECK(3,7,0) + #define simde_mm256_bslli_epi128(a, imm8) _mm256_bslli_epi128(a, imm8) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_bslli_epi128 + #define _mm256_bslli_epi128(a, imm8) simde_mm256_bslli_epi128(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_bsrli_epi128 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + const int ssize = HEDLEY_STATIC_CAST(int, (sizeof(r_.i8) / sizeof(r_.i8[0]))); + + SIMDE_VECTORIZE + for (int i = 0 ; i < ssize ; i++) { + const int e = i + imm8; + if(i < (ssize/2)) { + if(e >= 0 && e < (ssize/2)) + r_.i8[i] = a_.i8[e]; + else + r_.i8[i] = 0; + } + else{ + if(e >= (ssize/2) && e < ssize) + r_.i8[i] = a_.i8[e]; + else + r_.i8[i] = 0; + } + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) && \ + (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(4,8,0)) && \ + SIMDE_DETECT_CLANG_VERSION_CHECK(3,7,0) + #define simde_mm256_bsrli_epi128(a, imm8) _mm256_bsrli_epi128(a, imm8) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_bsrli_epi128 + #define _mm256_bsrli_epi128(a, imm8) simde_mm256_bsrli_epi128(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cmpeq_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cmpeq_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_cmpeq_epi8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_cmpeq_epi8(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] == b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpeq_epi8 + #define _mm256_cmpeq_epi8(a, b) simde_mm256_cmpeq_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cmpeq_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cmpeq_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_cmpeq_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_cmpeq_epi16(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] == b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpeq_epi16 + #define _mm256_cmpeq_epi16(a, b) simde_mm256_cmpeq_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cmpeq_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cmpeq_epi32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_cmpeq_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_cmpeq_epi32(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] == b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpeq_epi32 + #define _mm256_cmpeq_epi32(a, b) simde_mm256_cmpeq_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cmpeq_epi64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cmpeq_epi64(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_cmpeq_epi64(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_cmpeq_epi64(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] == b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpeq_epi64 + #define _mm256_cmpeq_epi64(a, b) simde_mm256_cmpeq_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cmpgt_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cmpgt_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_cmpgt_epi8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_cmpgt_epi8(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), a_.i8 > b_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] > b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpgt_epi8 + #define _mm256_cmpgt_epi8(a, b) simde_mm256_cmpgt_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cmpgt_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cmpgt_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_cmpgt_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_cmpgt_epi16(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = a_.i16 > b_.i16; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] > b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpgt_epi16 + #define _mm256_cmpgt_epi16(a, b) simde_mm256_cmpgt_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cmpgt_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cmpgt_epi32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_cmpgt_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_cmpgt_epi32(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.i32 > b_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] > b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpgt_epi32 + #define _mm256_cmpgt_epi32(a, b) simde_mm256_cmpgt_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cmpgt_epi64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cmpgt_epi64(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_cmpgt_epi64(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_cmpgt_epi64(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 > b_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] > b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpgt_epi64 + #define _mm256_cmpgt_epi64(a, b) simde_mm256_cmpgt_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cvtepi8_epi16 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cvtepi8_epi16(a); + #else + simde__m256i_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i16, a_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i8[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtepi8_epi16 + #define _mm256_cvtepi8_epi16(a) simde_mm256_cvtepi8_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cvtepi8_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cvtepi8_epi32(a); + #else + simde__m256i_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i32, a_.m64_private[0].i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i8[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtepi8_epi32 + #define _mm256_cvtepi8_epi32(a) simde_mm256_cvtepi8_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cvtepi8_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cvtepi8_epi64(a); + #else + simde__m256i_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i8[i]; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtepi8_epi64 + #define _mm256_cvtepi8_epi64(a) simde_mm256_cvtepi8_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cvtepi16_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cvtepi16_epi32(a); + #else + simde__m256i_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i32, a_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i16[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtepi16_epi32 + #define _mm256_cvtepi16_epi32(a) simde_mm256_cvtepi16_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cvtepi16_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cvtepi16_epi64(a); + #else + simde__m256i_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i64, a_.m64_private[0].i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i16[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtepi16_epi64 + #define _mm256_cvtepi16_epi64(a) simde_mm256_cvtepi16_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cvtepi32_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cvtepi32_epi64(a); + #else + simde__m256i_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i64, a_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i32[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtepi32_epi64 + #define _mm256_cvtepi32_epi64(a) simde_mm256_cvtepi32_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cvtepu8_epi16 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cvtepu8_epi16(a); + #else + simde__m256i_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i16, a_.u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.u8[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtepu8_epi16 + #define _mm256_cvtepu8_epi16(a) simde_mm256_cvtepu8_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cvtepu8_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cvtepu8_epi32(a); + #else + simde__m256i_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i32, a_.m64_private[0].u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.u8[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtepu8_epi32 + #define _mm256_cvtepu8_epi32(a) simde_mm256_cvtepu8_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cvtepu8_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cvtepu8_epi64(a); + #else + simde__m256i_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.u8[i]; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtepu8_epi64 + #define _mm256_cvtepu8_epi64(a) simde_mm256_cvtepu8_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cvtepu16_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cvtepu16_epi32(a); + #else + simde__m256i_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i32, a_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.u16[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtepu16_epi32 + #define _mm256_cvtepu16_epi32(a) simde_mm256_cvtepu16_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cvtepu16_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cvtepu16_epi64(a); + #else + simde__m256i_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i64, a_.m64_private[0].u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.u16[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtepu16_epi64 + #define _mm256_cvtepu16_epi64(a) simde_mm256_cvtepu16_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cvtepu32_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_cvtepu32_epi64(a); + #else + simde__m256i_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i64, a_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.u32[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtepu32_epi64 + #define _mm256_cvtepu32_epi64(a) simde_mm256_cvtepu32_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm256_extract_epi8 (simde__m256i a, const int index) + SIMDE_REQUIRE_RANGE(index, 0, 31){ + simde__m256i_private a_ = simde__m256i_to_private(a); + return a_.i8[index]; +} +#if defined(SIMDE_X86_AVX2_NATIVE) && \ + (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,10,0)) + #define simde_mm256_extract_epi8(a, index) _mm256_extract_epi8(a, index) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_extract_epi8 + #define _mm256_extract_epi8(a, index) simde_mm256_extract_epi8(a, index) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm256_extract_epi16 (simde__m256i a, const int index) + SIMDE_REQUIRE_RANGE(index, 0, 15) { + simde__m256i_private a_ = simde__m256i_to_private(a); + return a_.i16[index]; +} +#if defined(SIMDE_X86_AVX2_NATIVE) && \ + (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,10,0)) + #define simde_mm256_extract_epi16(a, index) _mm256_extract_epi16(a, index) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_extract_epi16 + #define _mm256_extract_epi16(a, index) simde_mm256_extract_epi16(a, index) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm256_extracti128_si256 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m256i_private a_ = simde__m256i_to_private(a); + return a_.m128i[imm8]; +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_extracti128_si256(a, imm8) _mm256_extracti128_si256(a, imm8) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_extracti128_si256 + #define _mm256_extracti128_si256(a, imm8) simde_mm256_extracti128_si256(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_i32gather_epi32(const int32_t* base_addr, simde__m128i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex), + r_; + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i32) / sizeof(vindex_.i32[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int32_t dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.i32[i] = dst; + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_i32gather_epi32(base_addr, vindex, scale) _mm_i32gather_epi32(SIMDE_CHECKED_REINTERPRET_CAST(int const*, int32_t const*, base_addr), vindex, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_i32gather_epi32 + #define _mm_i32gather_epi32(base_addr, vindex, scale) simde_mm_i32gather_epi32(SIMDE_CHECKED_REINTERPRET_CAST(int32_t const*, int const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_i32gather_epi32(simde__m128i src, const int32_t* base_addr, simde__m128i vindex, simde__m128i mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex), + src_ = simde__m128i_to_private(src), + mask_ = simde__m128i_to_private(mask), + r_; + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i32) / sizeof(vindex_.i32[0])) ; i++) { + if ((mask_.i32[i] >> 31) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int32_t dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.i32[i] = dst; + } + else { + r_.i32[i] = src_.i32[i]; + } + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_mask_i32gather_epi32(src, base_addr, vindex, mask, scale) _mm_mask_i32gather_epi32(src, SIMDE_CHECKED_REINTERPRET_CAST(int const*, int32_t const*, base_addr), vindex, mask, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_i32gather_epi32 + #define _mm_mask_i32gather_epi32(src, base_addr, vindex, mask, scale) simde_mm_mask_i32gather_epi32(src, SIMDE_CHECKED_REINTERPRET_CAST(int32_t const*, int const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_i32gather_epi32(const int32_t* base_addr, simde__m256i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m256i_private + vindex_ = simde__m256i_to_private(vindex), + r_; + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i32) / sizeof(vindex_.i32[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int32_t dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.i32[i] = dst; + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_i32gather_epi32(base_addr, vindex, scale) _mm256_i32gather_epi32(SIMDE_CHECKED_REINTERPRET_CAST(int const*, int32_t const*, base_addr), vindex, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_i32gather_epi32 + #define _mm256_i32gather_epi32(base_addr, vindex, scale) simde_mm256_i32gather_epi32(SIMDE_CHECKED_REINTERPRET_CAST(int32_t const*, int const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_i32gather_epi32(simde__m256i src, const int32_t* base_addr, simde__m256i vindex, simde__m256i mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m256i_private + vindex_ = simde__m256i_to_private(vindex), + src_ = simde__m256i_to_private(src), + mask_ = simde__m256i_to_private(mask), + r_; + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i32) / sizeof(vindex_.i32[0])) ; i++) { + if ((mask_.i32[i] >> 31) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int32_t dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.i32[i] = dst; + } + else { + r_.i32[i] = src_.i32[i]; + } + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_mask_i32gather_epi32(src, base_addr, vindex, mask, scale) _mm256_mask_i32gather_epi32(src, SIMDE_CHECKED_REINTERPRET_CAST(int const*, int32_t const*, base_addr), vindex, mask, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_i32gather_epi32 + #define _mm256_mask_i32gather_epi32(src, base_addr, vindex, mask, scale) simde_mm256_mask_i32gather_epi32(src, SIMDE_CHECKED_REINTERPRET_CAST(int32_t const*, int const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_i64gather_epi32(const int32_t* base_addr, simde__m128i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex), + r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int32_t dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.i32[i] = dst; + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_i64gather_epi32(base_addr, vindex, scale) _mm_i64gather_epi32(SIMDE_CHECKED_REINTERPRET_CAST(int const*, int32_t const*, base_addr), vindex, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_i64gather_epi32 + #define _mm_i64gather_epi32(base_addr, vindex, scale) simde_mm_i64gather_epi32(SIMDE_CHECKED_REINTERPRET_CAST(int32_t const*, int const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_i64gather_epi32(simde__m128i src, const int32_t* base_addr, simde__m128i vindex, simde__m128i mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex), + src_ = simde__m128i_to_private(src), + mask_ = simde__m128i_to_private(mask), + r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + if ((mask_.i32[i] >> 31) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int32_t dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.i32[i] = dst; + } + else { + r_.i32[i] = src_.i32[i]; + } + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_mask_i64gather_epi32(src, base_addr, vindex, mask, scale) _mm_mask_i64gather_epi32(src, SIMDE_CHECKED_REINTERPRET_CAST(int const*, int32_t const*, base_addr), vindex, mask, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_i64gather_epi32 + #define _mm_mask_i64gather_epi32(src, base_addr, vindex, mask, scale) simde_mm_mask_i64gather_epi32(src, SIMDE_CHECKED_REINTERPRET_CAST(int32_t const*, int const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm256_i64gather_epi32(const int32_t* base_addr, simde__m256i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m256i_private + vindex_ = simde__m256i_to_private(vindex); + simde__m128i_private + r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int32_t dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.i32[i] = dst; + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_i64gather_epi32(base_addr, vindex, scale) _mm256_i64gather_epi32(SIMDE_CHECKED_REINTERPRET_CAST(int const*, int32_t const*, base_addr), vindex, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_i64gather_epi32 + #define _mm256_i64gather_epi32(base_addr, vindex, scale) simde_mm256_i64gather_epi32(SIMDE_CHECKED_REINTERPRET_CAST(int32_t const*, int const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm256_mask_i64gather_epi32(simde__m128i src, const int32_t* base_addr, simde__m256i vindex, simde__m128i mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m256i_private + vindex_ = simde__m256i_to_private(vindex); + simde__m128i_private + src_ = simde__m128i_to_private(src), + mask_ = simde__m128i_to_private(mask), + r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + if ((mask_.i32[i] >> 31) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int32_t dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.i32[i] = dst; + } + else { + r_.i32[i] = src_.i32[i]; + } + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_mask_i64gather_epi32(src, base_addr, vindex, mask, scale) _mm256_mask_i64gather_epi32(src, SIMDE_CHECKED_REINTERPRET_CAST(int const*, int32_t const*, base_addr), vindex, mask, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_i64gather_epi32 + #define _mm256_mask_i64gather_epi32(src, base_addr, vindex, mask, scale) simde_mm256_mask_i64gather_epi32(src, SIMDE_CHECKED_REINTERPRET_CAST(int32_t const*, int const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_i32gather_epi64(const int64_t* base_addr, simde__m128i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex), + r_; + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int64_t dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.i64[i] = dst; + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #if SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + #define simde_mm_i32gather_epi64(base_addr, vindex, scale) _mm_i32gather_epi64(HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, scale) + #else + #define simde_mm_i32gather_epi64(base_addr, vindex, scale) _mm_i32gather_epi64(HEDLEY_REINTERPRET_CAST(long long const*, base_addr), vindex, scale) + #endif +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_i32gather_epi64 + #define _mm_i32gather_epi64(base_addr, vindex, scale) simde_mm_i32gather_epi64(HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_i32gather_epi64(simde__m128i src, const int64_t* base_addr, simde__m128i vindex, simde__m128i mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex), + src_ = simde__m128i_to_private(src), + mask_ = simde__m128i_to_private(mask), + r_; + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + if ((mask_.i64[i] >> 63) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int64_t dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.i64[i] = dst; + } + else { + r_.i64[i] = src_.i64[i]; + } + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #if SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + #define simde_mm_mask_i32gather_epi64(src, base_addr, vindex, mask, scale) _mm_mask_i32gather_epi64(src, HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, mask, scale) + #else + #define simde_mm_mask_i32gather_epi64(src, base_addr, vindex, mask, scale) _mm_mask_i32gather_epi64(src, HEDLEY_REINTERPRET_CAST(long long const*, base_addr), vindex, mask, scale) + #endif +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_i32gather_epi64 + #define _mm_mask_i32gather_epi64(src, base_addr, vindex, mask, scale) simde_mm_mask_i32gather_epi64(src, HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_i32gather_epi64(const int64_t* base_addr, simde__m128i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex); + simde__m256i_private + r_; + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i32) / sizeof(vindex_.i32[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int64_t dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.i64[i] = dst; + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #if SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + #define simde_mm256_i32gather_epi64(base_addr, vindex, scale) _mm256_i32gather_epi64(HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, scale) + #else + #define simde_mm256_i32gather_epi64(base_addr, vindex, scale) _mm256_i32gather_epi64(HEDLEY_REINTERPRET_CAST(long long const*, base_addr), vindex, scale) + #endif +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_i32gather_epi64 + #define _mm256_i32gather_epi64(base_addr, vindex, scale) simde_mm256_i32gather_epi64(HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_i32gather_epi64(simde__m256i src, const int64_t* base_addr, simde__m128i vindex, simde__m256i mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m256i_private + src_ = simde__m256i_to_private(src), + mask_ = simde__m256i_to_private(mask), + r_; + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i32) / sizeof(vindex_.i32[0])) ; i++) { + if ((mask_.i64[i] >> 63) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int64_t dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.i64[i] = dst; + } + else { + r_.i64[i] = src_.i64[i]; + } + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #if SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + #define simde_mm256_mask_i32gather_epi64(src, base_addr, vindex, mask, scale) _mm256_mask_i32gather_epi64(src, HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, mask, scale) + #else + #define simde_mm256_mask_i32gather_epi64(src, base_addr, vindex, mask, scale) _mm256_mask_i32gather_epi64(src, HEDLEY_REINTERPRET_CAST(long long const*, base_addr), vindex, mask, scale) + #endif +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_i32gather_epi64 + #define _mm256_mask_i32gather_epi64(src, base_addr, vindex, mask, scale) simde_mm256_mask_i32gather_epi64(src, HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_i64gather_epi64(const int64_t* base_addr, simde__m128i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex), + r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int64_t dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.i64[i] = dst; + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #if SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + #define simde_mm_i64gather_epi64(base_addr, vindex, scale) _mm_i64gather_epi64(HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, scale) + #else + #define simde_mm_i64gather_epi64(base_addr, vindex, scale) _mm_i64gather_epi64(HEDLEY_REINTERPRET_CAST(long long const*, base_addr), vindex, scale) + #endif +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_i64gather_epi64 + #define _mm_i64gather_epi64(base_addr, vindex, scale) simde_mm_i64gather_epi64(HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_i64gather_epi64(simde__m128i src, const int64_t* base_addr, simde__m128i vindex, simde__m128i mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex), + src_ = simde__m128i_to_private(src), + mask_ = simde__m128i_to_private(mask), + r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + if ((mask_.i64[i] >> 63) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int64_t dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.i64[i] = dst; + } + else { + r_.i64[i] = src_.i64[i]; + } + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #if SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + #define simde_mm_mask_i64gather_epi64(src, base_addr, vindex, mask, scale) _mm_mask_i64gather_epi64(src, HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, mask, scale) + #else + #define simde_mm_mask_i64gather_epi64(src, base_addr, vindex, mask, scale) _mm_mask_i64gather_epi64(src, HEDLEY_REINTERPRET_CAST(long long const*, base_addr), vindex, mask, scale) + #endif +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_i64gather_epi64 + #define _mm_mask_i64gather_epi64(src, base_addr, vindex, mask, scale) simde_mm_mask_i64gather_epi64(src, HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_i64gather_epi64(const int64_t* base_addr, simde__m256i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m256i_private + vindex_ = simde__m256i_to_private(vindex), + r_ = simde__m256i_to_private(simde_mm256_setzero_si256()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int64_t dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.i64[i] = dst; + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #if SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + #define simde_mm256_i64gather_epi64(base_addr, vindex, scale) _mm256_i64gather_epi64(HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, scale) + #else + #define simde_mm256_i64gather_epi64(base_addr, vindex, scale) _mm256_i64gather_epi64(HEDLEY_REINTERPRET_CAST(long long const*, base_addr), vindex, scale) + #endif +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_i64gather_epi64 + #define _mm256_i64gather_epi64(base_addr, vindex, scale) simde_mm256_i64gather_epi64(HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_i64gather_epi64(simde__m256i src, const int64_t* base_addr, simde__m256i vindex, simde__m256i mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m256i_private + vindex_ = simde__m256i_to_private(vindex), + src_ = simde__m256i_to_private(src), + mask_ = simde__m256i_to_private(mask), + r_ = simde__m256i_to_private(simde_mm256_setzero_si256()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + if ((mask_.i64[i] >> 63) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int64_t dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.i64[i] = dst; + } + else { + r_.i64[i] = src_.i64[i]; + } + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #if SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + #define simde_mm256_mask_i64gather_epi64(src, base_addr, vindex, mask, scale) _mm256_mask_i64gather_epi64(src, HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, mask, scale) + #else + #define simde_mm256_mask_i64gather_epi64(src, base_addr, vindex, mask, scale) _mm256_mask_i64gather_epi64(src, HEDLEY_REINTERPRET_CAST(long long const*, base_addr), vindex, mask, scale) + #endif +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_i64gather_epi64 + #define _mm256_mask_i64gather_epi64(src, base_addr, vindex, mask, scale) simde_mm256_mask_i64gather_epi64(src, HEDLEY_REINTERPRET_CAST(int64_t const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_i32gather_ps(const simde_float32* base_addr, simde__m128i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex); + simde__m128_private + r_; + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i32) / sizeof(vindex_.i32[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float32 dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.f32[i] = dst; + } + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_i32gather_ps(base_addr, vindex, scale) _mm_i32gather_ps(SIMDE_CHECKED_REINTERPRET_CAST(float const*, simde_float32 const*, base_addr), vindex, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_i32gather_ps + #define _mm_i32gather_ps(base_addr, vindex, scale) simde_mm_i32gather_ps(SIMDE_CHECKED_REINTERPRET_CAST(simde_float32 const*, float const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mask_i32gather_ps(simde__m128 src, const simde_float32* base_addr, simde__m128i vindex, simde__m128 mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex); + simde__m128_private + src_ = simde__m128_to_private(src), + mask_ = simde__m128_to_private(mask), + r_; + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i32) / sizeof(vindex_.i32[0])) ; i++) { + if ((mask_.i32[i] >> 31) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float32 dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.f32[i] = dst; + } + else { + r_.f32[i] = src_.f32[i]; + } + } + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_mask_i32gather_ps(src, base_addr, vindex, mask, scale) _mm_mask_i32gather_ps(src, SIMDE_CHECKED_REINTERPRET_CAST(float const*, simde_float32 const*, base_addr), vindex, mask, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_i32gather_ps + #define _mm_mask_i32gather_ps(src, base_addr, vindex, mask, scale) simde_mm_mask_i32gather_ps(src, SIMDE_CHECKED_REINTERPRET_CAST(simde_float32 const*, float const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_i32gather_ps(const simde_float32* base_addr, simde__m256i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m256i_private + vindex_ = simde__m256i_to_private(vindex); + simde__m256_private + r_; + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i32) / sizeof(vindex_.i32[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float32 dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.f32[i] = dst; + } + + return simde__m256_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_i32gather_ps(base_addr, vindex, scale) _mm256_i32gather_ps(SIMDE_CHECKED_REINTERPRET_CAST(float const*, simde_float32 const*, (base_addr)), (vindex), (scale)) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_i32gather_ps + #define _mm256_i32gather_ps(base_addr, vindex, scale) simde_mm256_i32gather_ps(SIMDE_CHECKED_REINTERPRET_CAST(simde_float32 const*, float const*, (base_addr)), (vindex), (scale)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask_i32gather_ps(simde__m256 src, const simde_float32* base_addr, simde__m256i vindex, simde__m256 mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m256i_private + vindex_ = simde__m256i_to_private(vindex); + simde__m256_private + src_ = simde__m256_to_private(src), + mask_ = simde__m256_to_private(mask), + r_; + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i32) / sizeof(vindex_.i32[0])) ; i++) { + if ((mask_.i32[i] >> 31) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float32 dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.f32[i] = dst; + } + else { + r_.f32[i] = src_.f32[i]; + } + } + + return simde__m256_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_mask_i32gather_ps(src, base_addr, vindex, mask, scale) _mm256_mask_i32gather_ps(src, SIMDE_CHECKED_REINTERPRET_CAST(float const*, simde_float32 const*, base_addr), vindex, mask, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_i32gather_ps + #define _mm256_mask_i32gather_ps(src, base_addr, vindex, mask, scale) simde_mm256_mask_i32gather_ps(src, SIMDE_CHECKED_REINTERPRET_CAST(simde_float32 const*, float const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_i64gather_ps(const simde_float32* base_addr, simde__m128i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex); + simde__m128_private + r_ = simde__m128_to_private(simde_mm_setzero_ps()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float32 dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.f32[i] = dst; + } + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_i64gather_ps(base_addr, vindex, scale) _mm_i64gather_ps(SIMDE_CHECKED_REINTERPRET_CAST(float const*, simde_float32 const*, base_addr), vindex, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_i64gather_ps + #define _mm_i64gather_ps(base_addr, vindex, scale) simde_mm_i64gather_ps(SIMDE_CHECKED_REINTERPRET_CAST(simde_float32 const*, float const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mask_i64gather_ps(simde__m128 src, const simde_float32* base_addr, simde__m128i vindex, simde__m128 mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex); + simde__m128_private + src_ = simde__m128_to_private(src), + mask_ = simde__m128_to_private(mask), + r_ = simde__m128_to_private(simde_mm_setzero_ps()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + if ((mask_.i32[i] >> 31) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float32 dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.f32[i] = dst; + } + else { + r_.f32[i] = src_.f32[i]; + } + } + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_mask_i64gather_ps(src, base_addr, vindex, mask, scale) _mm_mask_i64gather_ps(src, SIMDE_CHECKED_REINTERPRET_CAST(float const*, float32_t const*, base_addr), vindex, mask, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_i64gather_ps + #define _mm_mask_i64gather_ps(src, base_addr, vindex, mask, scale) simde_mm_mask_i64gather_ps(src, SIMDE_CHECKED_REINTERPRET_CAST(simde_float32 const*, float const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm256_i64gather_ps(const simde_float32* base_addr, simde__m256i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m256i_private + vindex_ = simde__m256i_to_private(vindex); + simde__m128_private + r_ = simde__m128_to_private(simde_mm_setzero_ps()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float32 dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.f32[i] = dst; + } + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_i64gather_ps(base_addr, vindex, scale) _mm256_i64gather_ps(SIMDE_CHECKED_REINTERPRET_CAST(float const*, simde_float32 const*, base_addr), vindex, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_i64gather_ps + #define _mm256_i64gather_ps(base_addr, vindex, scale) simde_mm256_i64gather_ps(SIMDE_CHECKED_REINTERPRET_CAST(simde_float32 const*, float const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm256_mask_i64gather_ps(simde__m128 src, const simde_float32* base_addr, simde__m256i vindex, simde__m128 mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m256i_private + vindex_ = simde__m256i_to_private(vindex); + simde__m128_private + src_ = simde__m128_to_private(src), + mask_ = simde__m128_to_private(mask), + r_ = simde__m128_to_private(simde_mm_setzero_ps()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + if ((mask_.i32[i] >> 31) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float32 dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.f32[i] = dst; + } + else { + r_.f32[i] = src_.f32[i]; + } + } + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_mask_i64gather_ps(src, base_addr, vindex, mask, scale) _mm256_mask_i64gather_ps(src, SIMDE_CHECKED_REINTERPRET_CAST(float const*, simde_float32 const*, base_addr), vindex, mask, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_i64gather_ps + #define _mm256_mask_i64gather_ps(src, base_addr, vindex, mask, scale) simde_mm256_mask_i64gather_ps(src, SIMDE_CHECKED_REINTERPRET_CAST(simde_float32 const*, float const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_i32gather_pd(const simde_float64* base_addr, simde__m128i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex); + simde__m128d_private + r_; + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float64 dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.f64[i] = dst; + } + + return simde__m128d_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_i32gather_pd(base_addr, vindex, scale) _mm_i32gather_pd(HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_i32gather_pd + #define _mm_i32gather_pd(base_addr, vindex, scale) simde_mm_i32gather_pd(HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mask_i32gather_pd(simde__m128d src, const simde_float64* base_addr, simde__m128i vindex, simde__m128d mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex); + simde__m128d_private + src_ = simde__m128d_to_private(src), + mask_ = simde__m128d_to_private(mask), + r_; + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + if ((mask_.i64[i] >> 63) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float64 dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.f64[i] = dst; + } + else { + r_.f64[i] = src_.f64[i]; + } + } + + return simde__m128d_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_mask_i32gather_pd(src, base_addr, vindex, mask, scale) _mm_mask_i32gather_pd(src, HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, mask, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_i32gather_pd + #define _mm_mask_i32gather_pd(src, base_addr, vindex, mask, scale) simde_mm_mask_i32gather_pd(src, HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_i32gather_pd(const simde_float64* base_addr, simde__m128i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex); + simde__m256d_private + r_; + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i32) / sizeof(vindex_.i32[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float64 dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.f64[i] = dst; + } + + return simde__m256d_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_i32gather_pd(base_addr, vindex, scale) _mm256_i32gather_pd(HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_i32gather_pd + #define _mm256_i32gather_pd(base_addr, vindex, scale) simde_mm256_i32gather_pd(HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mask_i32gather_pd(simde__m256d src, const simde_float64* base_addr, simde__m128i vindex, simde__m256d mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m256d_private + src_ = simde__m256d_to_private(src), + mask_ = simde__m256d_to_private(mask), + r_; + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i32) / sizeof(vindex_.i32[0])) ; i++) { + if ((mask_.i64[i] >> 63) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float64 dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.f64[i] = dst; + } + else { + r_.f64[i] = src_.f64[i]; + } + } + + return simde__m256d_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_mask_i32gather_pd(src, base_addr, vindex, mask, scale) _mm256_mask_i32gather_pd(src, HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, mask, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_i32gather_pd + #define _mm256_mask_i32gather_pd(src, base_addr, vindex, mask, scale) simde_mm256_mask_i32gather_pd(src, HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_i64gather_pd(const simde_float64* base_addr, simde__m128i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex); + simde__m128d_private + r_ = simde__m128d_to_private(simde_mm_setzero_pd()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float64 dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.f64[i] = dst; + } + + return simde__m128d_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_i64gather_pd(base_addr, vindex, scale) _mm_i64gather_pd(HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_i64gather_pd + #define _mm_i64gather_pd(base_addr, vindex, scale) simde_mm_i64gather_pd(HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mask_i64gather_pd(simde__m128d src, const simde_float64* base_addr, simde__m128i vindex, simde__m128d mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m128i_private + vindex_ = simde__m128i_to_private(vindex); + simde__m128d_private + src_ = simde__m128d_to_private(src), + mask_ = simde__m128d_to_private(mask), + r_ = simde__m128d_to_private(simde_mm_setzero_pd()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + if ((mask_.i64[i] >> 63) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float64 dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.f64[i] = dst; + } + else { + r_.f64[i] = src_.f64[i]; + } + } + + return simde__m128d_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_mask_i64gather_pd(src, base_addr, vindex, mask, scale) _mm_mask_i64gather_pd(src, HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, mask, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_i64gather_pd + #define _mm_mask_i64gather_pd(src, base_addr, vindex, mask, scale) simde_mm_mask_i64gather_pd(src, HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_i64gather_pd(const simde_float64* base_addr, simde__m256i vindex, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m256i_private + vindex_ = simde__m256i_to_private(vindex); + simde__m256d_private + r_ = simde__m256d_to_private(simde_mm256_setzero_pd()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float64 dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.f64[i] = dst; + } + + return simde__m256d_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_i64gather_pd(base_addr, vindex, scale) _mm256_i64gather_pd(HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_i64gather_pd + #define _mm256_i64gather_pd(base_addr, vindex, scale) simde_mm256_i64gather_pd(HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mask_i64gather_pd(simde__m256d src, const simde_float64* base_addr, simde__m256i vindex, simde__m256d mask, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m256i_private + vindex_ = simde__m256i_to_private(vindex); + simde__m256d_private + src_ = simde__m256d_to_private(src), + mask_ = simde__m256d_to_private(mask), + r_ = simde__m256d_to_private(simde_mm256_setzero_pd()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + if ((mask_.i64[i] >> 63) & 1) { + const uint8_t* src1 = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float64 dst; + simde_memcpy(&dst, src1, sizeof(dst)); + r_.f64[i] = dst; + } + else { + r_.f64[i] = src_.f64[i]; + } + } + + return simde__m256d_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_mask_i64gather_pd(src, base_addr, vindex, mask, scale) _mm256_mask_i64gather_pd(src, HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, mask, scale) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_i64gather_pd + #define _mm256_mask_i64gather_pd(src, base_addr, vindex, mask, scale) simde_mm256_mask_i64gather_pd(src, HEDLEY_REINTERPRET_CAST(simde_float64 const*, base_addr), vindex, mask, scale) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_inserti128_si256(simde__m256i a, simde__m128i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m256i_private a_ = simde__m256i_to_private(a); + simde__m128i_private b_ = simde__m128i_to_private(b); + + a_.m128i_private[ imm8 & 1 ] = b_; + + return simde__m256i_from_private(a_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_inserti128_si256(a, b, imm8) _mm256_inserti128_si256(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_inserti128_si256 + #define _mm256_inserti128_si256(a, b, imm8) simde_mm256_inserti128_si256(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_madd_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_madd_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_madd_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_madd_epi16(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_CONVERT_VECTOR_) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) + SIMDE_ALIGN_TO_32 int32_t product SIMDE_VECTOR(64); + SIMDE_ALIGN_TO_32 int32_t a32x16 SIMDE_VECTOR(64); + SIMDE_ALIGN_TO_32 int32_t b32x16 SIMDE_VECTOR(64); + SIMDE_ALIGN_TO_32 int32_t even SIMDE_VECTOR(32); + SIMDE_ALIGN_TO_32 int32_t odd SIMDE_VECTOR(32); + + SIMDE_CONVERT_VECTOR_(a32x16, a_.i16); + SIMDE_CONVERT_VECTOR_(b32x16, b_.i16); + product = a32x16 * b32x16; + + even = __builtin_shufflevector(product, product, 0, 2, 4, 6, 8, 10, 12, 14); + odd = __builtin_shufflevector(product, product, 1, 3, 5, 7, 9, 11, 13, 15); + + r_.i32 = even + odd; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_) / sizeof(r_.i16[0])) ; i += 2) { + r_.i32[i / 2] = (a_.i16[i] * b_.i16[i]) + (a_.i16[i + 1] * b_.i16[i + 1]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_madd_epi16 + #define _mm256_madd_epi16(a, b) simde_mm256_madd_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maddubs_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_maddubs_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_maddubs_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_maddubs_epi16(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + const int idx = HEDLEY_STATIC_CAST(int, i) << 1; + int32_t ts = + (HEDLEY_STATIC_CAST(int16_t, a_.u8[ idx ]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[ idx ])) + + (HEDLEY_STATIC_CAST(int16_t, a_.u8[idx + 1]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[idx + 1])); + r_.i16[i] = (ts > INT16_MIN) ? ((ts < INT16_MAX) ? HEDLEY_STATIC_CAST(int16_t, ts) : INT16_MAX) : INT16_MIN; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_maddubs_epi16 + #define _mm256_maddubs_epi16(a, b) simde_mm256_maddubs_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskload_epi32 (const int32_t mem_addr[HEDLEY_ARRAY_PARAM(4)], simde__m128i mask) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm_maskload_epi32(mem_addr, mask); + #else + simde__m128i_private + r_, + mask_ = simde__m128i_to_private(mask), + mask_shr_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + mask_shr_.neon_i32 = vshrq_n_s32(mask_.neon_i32, 31); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + mask_shr_.i32[i] = mask_.i32[i] >> 31; + } + #endif + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = mask_shr_.i32[i] ? mem_addr[i] : INT32_C(0); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_maskload_epi32 + #define _mm_maskload_epi32(mem_addr, mask) simde_mm_maskload_epi32(HEDLEY_REINTERPRET_CAST(int32_t const*, mem_addr), mask) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskload_epi32 (const int32_t mem_addr[HEDLEY_ARRAY_PARAM(4)], simde__m256i mask) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_maskload_epi32(mem_addr, mask); + #else + simde__m256i_private + mask_ = simde__m256i_to_private(mask), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (mask_.i32[i] >> 31) ? mem_addr[i] : INT32_C(0); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskload_epi32 + #define _mm256_maskload_epi32(mem_addr, mask) simde_mm256_maskload_epi32(HEDLEY_REINTERPRET_CAST(int32_t const*, mem_addr), mask) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskload_epi64 (const int64_t mem_addr[HEDLEY_ARRAY_PARAM(2)], simde__m128i mask) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm_maskload_epi64(HEDLEY_REINTERPRET_CAST(const long long *, mem_addr), mask); + #else + simde__m128i_private + r_, + mask_ = simde__m128i_to_private(mask), + mask_shr_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + mask_shr_.neon_i64 = vshrq_n_s64(mask_.neon_i64, 63); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(mask_.i64) / sizeof(mask_.i64[0])) ; i++) { + mask_shr_.i64[i] = mask_.i64[i] >> 63; + } + #endif + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = mask_shr_.i64[i] ? mem_addr[i] : INT64_C(0); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_maskload_epi64 + #define _mm_maskload_epi64(mem_addr, mask) simde_mm_maskload_epi64(HEDLEY_REINTERPRET_CAST(int64_t const*, mem_addr), mask) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskload_epi64 (const int64_t mem_addr[HEDLEY_ARRAY_PARAM(4)], simde__m256i mask) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_maskload_epi64(HEDLEY_REINTERPRET_CAST(const long long *, mem_addr), mask); + #else + simde__m256i_private + mask_ = simde__m256i_to_private(mask), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (mask_.i64[i] >> 63) ? mem_addr[i] : INT64_C(0); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskload_epi64 + #define _mm256_maskload_epi64(mem_addr, mask) simde_mm256_maskload_epi64(HEDLEY_REINTERPRET_CAST(int64_t const*, mem_addr), mask) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_maskstore_epi32 (int32_t mem_addr[HEDLEY_ARRAY_PARAM(4)], simde__m128i mask, simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + _mm_maskstore_epi32(mem_addr, mask, a); + #else + simde__m128i_private mask_ = simde__m128i_to_private(mask); + simde__m128i_private a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + if (mask_.u32[i] & (UINT32_C(1) << 31)) + mem_addr[i] = a_.i32[i]; + } + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_maskstore_epi32 + #define _mm_maskstore_epi32(mem_addr, mask, a) simde_mm_maskstore_epi32(HEDLEY_REINTERPRET_CAST(int32_t *, mem_addr), mask, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_maskstore_epi32 (int32_t mem_addr[HEDLEY_ARRAY_PARAM(8)], simde__m256i mask, simde__m256i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + _mm256_maskstore_epi32(mem_addr, mask, a); + #else + simde__m256i_private mask_ = simde__m256i_to_private(mask); + simde__m256i_private a_ = simde__m256i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + if (mask_.u32[i] & (UINT32_C(1) << 31)) + mem_addr[i] = a_.i32[i]; + } + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskstore_epi32 + #define _mm256_maskstore_epi32(mem_addr, mask, a) simde_mm256_maskstore_epi32(HEDLEY_REINTERPRET_CAST(int32_t *, mem_addr), mask, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_maskstore_epi64 (int64_t mem_addr[HEDLEY_ARRAY_PARAM(2)], simde__m128i mask, simde__m128i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + _mm_maskstore_epi64(HEDLEY_REINTERPRET_CAST(long long *, mem_addr), mask, a); + #else + simde__m128i_private mask_ = simde__m128i_to_private(mask); + simde__m128i_private a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + if (mask_.u64[i] >> 63) + mem_addr[i] = a_.i64[i]; + } + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_maskstore_epi64 + #define _mm_maskstore_epi64(mem_addr, mask, a) simde_mm_maskstore_epi64(HEDLEY_REINTERPRET_CAST(int64_t *, mem_addr), mask, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_maskstore_epi64 (int64_t mem_addr[HEDLEY_ARRAY_PARAM(4)], simde__m256i mask, simde__m256i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + _mm256_maskstore_epi64(HEDLEY_REINTERPRET_CAST(long long *, mem_addr), mask, a); + #else + simde__m256i_private mask_ = simde__m256i_to_private(mask); + simde__m256i_private a_ = simde__m256i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + if (mask_.u64[i] & (UINT64_C(1) << 63)) + mem_addr[i] = a_.i64[i]; + } + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskstore_epi64 + #define _mm256_maskstore_epi64(mem_addr, mask, a) simde_mm256_maskstore_epi64(HEDLEY_REINTERPRET_CAST(int64_t *, mem_addr), mask, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_max_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) && !defined(__PGI) + return _mm256_max_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_max_epi8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_max_epi8(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] > b_.i8[i] ? a_.i8[i] : b_.i8[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_max_epi8 + #define _mm256_max_epi8(a, b) simde_mm256_max_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_max_epu8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_max_epu8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_max_epu8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_max_epu8(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] > b_.u8[i]) ? a_.u8[i] : b_.u8[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_max_epu8 + #define _mm256_max_epu8(a, b) simde_mm256_max_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_max_epu16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_max_epu16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_max_epu16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_max_epu16(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] > b_.u16[i]) ? a_.u16[i] : b_.u16[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_max_epu16 + #define _mm256_max_epu16(a, b) simde_mm256_max_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_max_epu32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_max_epu32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_max_epu32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_max_epu32(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] > b_.u32[i]) ? a_.u32[i] : b_.u32[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_max_epu32 + #define _mm256_max_epu32(a, b) simde_mm256_max_epu32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_max_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_max_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_max_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_max_epi16(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] > b_.i16[i]) ? a_.i16[i] : b_.i16[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_max_epi16 + #define _mm256_max_epi16(a, b) simde_mm256_max_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_max_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_max_epi32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_max_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_max_epi32(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] > b_.i32[i] ? a_.i32[i] : b_.i32[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_max_epi32 + #define _mm256_max_epi32(a, b) simde_mm256_max_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_min_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) && !defined(__PGI) + return _mm256_min_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_min_epi8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_min_epi8(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] < b_.i8[i] ? a_.i8[i] : b_.i8[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_min_epi8 + #define _mm256_min_epi8(a, b) simde_mm256_min_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_min_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_min_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_min_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_min_epi16(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] < b_.i16[i]) ? a_.i16[i] : b_.i16[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_min_epi16 + #define _mm256_min_epi16(a, b) simde_mm256_min_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_min_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_min_epi32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_min_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_min_epi32(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] < b_.i32[i] ? a_.i32[i] : b_.i32[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_min_epi32 + #define _mm256_min_epi32(a, b) simde_mm256_min_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_min_epu8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_min_epu8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_min_epu8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_min_epu8(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] < b_.u8[i]) ? a_.u8[i] : b_.u8[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_min_epu8 + #define _mm256_min_epu8(a, b) simde_mm256_min_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_min_epu16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_min_epu16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_min_epu16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_min_epu16(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] < b_.u16[i]) ? a_.u16[i] : b_.u16[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_min_epu16 + #define _mm256_min_epu16(a, b) simde_mm256_min_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_min_epu32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_min_epu32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_min_epu32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_min_epu32(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] < b_.u32[i]) ? a_.u32[i] : b_.u32[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_min_epu32 + #define _mm256_min_epu32(a, b) simde_mm256_min_epu32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm256_movemask_epi8 (simde__m256i a) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_movemask_epi8(a); + #else + simde__m256i_private a_ = simde__m256i_to_private(a); + uint32_t r = 0; + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(a_.m128i) / sizeof(a_.m128i[0])) ; i++) { + r |= HEDLEY_STATIC_CAST(uint32_t,simde_mm_movemask_epi8(a_.m128i[i])) << (16 * i); + } + #else + r = 0; + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0])) ; i++) { + r |= HEDLEY_STATIC_CAST(uint32_t, (a_.u8[31 - i] >> 7)) << (31 - i); + } + #endif + + return HEDLEY_STATIC_CAST(int32_t, r); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_movemask_epi8 + #define _mm256_movemask_epi8(a) simde_mm256_movemask_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mpsadbw_epu8 (simde__m256i a, simde__m256i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + const int a_offset1 = imm8 & 4; + const int b_offset1 = (imm8 & 3) << 2; + const int a_offset2 = (imm8 >> 3) & 4; + const int b_offset2 = ((imm8 >> 3) & 3) << 2; + + #if defined(simde_math_abs) + const int halfway_point = HEDLEY_STATIC_CAST(int, (sizeof(r_.u16) / sizeof(r_.u16[0])) ) / 2; + for (int i = 0 ; i < halfway_point ; i++) { + r_.u16[i] = + HEDLEY_STATIC_CAST(uint16_t, simde_math_abs(HEDLEY_STATIC_CAST(int, a_.u8[a_offset1 + i + 0] - b_.u8[b_offset1 + 0]))) + + HEDLEY_STATIC_CAST(uint16_t, simde_math_abs(HEDLEY_STATIC_CAST(int, a_.u8[a_offset1 + i + 1] - b_.u8[b_offset1 + 1]))) + + HEDLEY_STATIC_CAST(uint16_t, simde_math_abs(HEDLEY_STATIC_CAST(int, a_.u8[a_offset1 + i + 2] - b_.u8[b_offset1 + 2]))) + + HEDLEY_STATIC_CAST(uint16_t, simde_math_abs(HEDLEY_STATIC_CAST(int, a_.u8[a_offset1 + i + 3] - b_.u8[b_offset1 + 3]))); + r_.u16[halfway_point + i] = + HEDLEY_STATIC_CAST(uint16_t, simde_math_abs(HEDLEY_STATIC_CAST(int, a_.u8[2 * halfway_point + a_offset2 + i + 0] - b_.u8[2 * halfway_point + b_offset2 + 0]))) + + HEDLEY_STATIC_CAST(uint16_t, simde_math_abs(HEDLEY_STATIC_CAST(int, a_.u8[2 * halfway_point + a_offset2 + i + 1] - b_.u8[2 * halfway_point + b_offset2 + 1]))) + + HEDLEY_STATIC_CAST(uint16_t, simde_math_abs(HEDLEY_STATIC_CAST(int, a_.u8[2 * halfway_point + a_offset2 + i + 2] - b_.u8[2 * halfway_point + b_offset2 + 2]))) + + HEDLEY_STATIC_CAST(uint16_t, simde_math_abs(HEDLEY_STATIC_CAST(int, a_.u8[2 * halfway_point + a_offset2 + i + 3] - b_.u8[2 * halfway_point + b_offset2 + 3]))); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) && SIMDE_DETECT_CLANG_VERSION_CHECK(3,9,0) + #define simde_mm256_mpsadbw_epu8(a, b, imm8) _mm256_mpsadbw_epu8(a, b, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + #define simde_mm256_mpsadbw_epu8(a, b, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_mpsadbw_epu8(simde_mm256_extracti128_si256(a, 1), simde_mm256_extracti128_si256(b, 1), (imm8 >> 3)), \ + simde_mm_mpsadbw_epu8(simde_mm256_extracti128_si256(a, 0), simde_mm256_extracti128_si256(b, 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_mpsadbw_epu8 + #define _mm256_mpsadbw_epu8(a, b, imm8) simde_mm256_mpsadbw_epu8(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mul_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_mul_epi32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_mul_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_mul_epi32(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = + HEDLEY_STATIC_CAST(int64_t, a_.i32[i * 2]) * + HEDLEY_STATIC_CAST(int64_t, b_.i32[i * 2]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) +# define _mm256_mul_epi32(a, b) simde_mm256_mul_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mul_epu32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_mul_epu32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_mul_epu32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_mul_epu32(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = HEDLEY_STATIC_CAST(uint64_t, a_.u32[i * 2]) * HEDLEY_STATIC_CAST(uint64_t, b_.u32[i * 2]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) +# define _mm256_mul_epu32(a, b) simde_mm256_mul_epu32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mulhi_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_mulhi_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, (HEDLEY_STATIC_CAST(uint32_t, HEDLEY_STATIC_CAST(int32_t, a_.i16[i]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[i])) >> 16)); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) +# define _mm256_mulhi_epi16(a, b) simde_mm256_mulhi_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mulhi_epu16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_mulhi_epu16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, HEDLEY_STATIC_CAST(uint32_t, a_.u16[i]) * HEDLEY_STATIC_CAST(uint32_t, b_.u16[i]) >> 16); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) +# define _mm256_mulhi_epu16(a, b) simde_mm256_mulhi_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mulhrs_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_mulhrs_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, (((HEDLEY_STATIC_CAST(int32_t, a_.i16[i]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[i])) + 0x4000) >> 15)); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) +# define _mm256_mulhrs_epi16(a, b) simde_mm256_mulhrs_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mullo_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_mullo_epi16(a, b); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, a_.i16[i] * b_.i16[i]); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_mullo_epi16 + #define _mm256_mullo_epi16(a, b) simde_mm256_mullo_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mullo_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_mullo_epi32(a, b); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = HEDLEY_STATIC_CAST(int32_t, a_.i32[i] * b_.i32[i]); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_mullo_epi32 + #define _mm256_mullo_epi32(a, b) simde_mm256_mullo_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_mullo_epu32 (simde__m256i a, simde__m256i b) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = a_.u32 * b_.u32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] * b_.u32[i]; + } + #endif + + return simde__m256i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_or_si256 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_or_si256(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_or_si128(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_or_si128(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f | b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] | b_.i32f[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_or_si256 + #define _mm256_or_si256(a, b) simde_mm256_or_si256(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_packs_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_packs_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_packs_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_packs_epi16(a_.m128i[1], b_.m128i[1]); + #else + const size_t halfway_point = (sizeof(r_.i8) / sizeof(r_.i8[0]))/2; + const size_t quarter_point = (sizeof(r_.i8) / sizeof(r_.i8[0]))/4; + SIMDE_VECTORIZE + for (size_t i = 0 ; i < quarter_point ; i++) { + r_.i8[i] = (a_.i16[i] > INT8_MAX) ? INT8_MAX : ((a_.i16[i] < INT8_MIN) ? INT8_MIN : HEDLEY_STATIC_CAST(int8_t, a_.i16[i])); + r_.i8[i + quarter_point] = (b_.i16[i] > INT8_MAX) ? INT8_MAX : ((b_.i16[i] < INT8_MIN) ? INT8_MIN : HEDLEY_STATIC_CAST(int8_t, b_.i16[i])); + r_.i8[halfway_point + i] = (a_.i16[quarter_point + i] > INT8_MAX) ? INT8_MAX : ((a_.i16[quarter_point + i] < INT8_MIN) ? INT8_MIN : HEDLEY_STATIC_CAST(int8_t, a_.i16[quarter_point + i])); + r_.i8[halfway_point + i + quarter_point] = (b_.i16[quarter_point + i] > INT8_MAX) ? INT8_MAX : ((b_.i16[quarter_point + i] < INT8_MIN) ? INT8_MIN : HEDLEY_STATIC_CAST(int8_t, b_.i16[quarter_point + i])); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_packs_epi16 + #define _mm256_packs_epi16(a, b) simde_mm256_packs_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_packs_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_packs_epi32(a, b); + #else + simde__m256i_private + r_, + v_[] = { + simde__m256i_to_private(a), + simde__m256i_to_private(b) + }; + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_packs_epi32(v_[0].m128i[0], v_[1].m128i[0]); + r_.m128i[1] = simde_mm_packs_epi32(v_[0].m128i[1], v_[1].m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + const int32_t v = v_[(i >> 2) & 1].i32[(i & 11) - ((i & 8) >> 1)]; + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, (v > INT16_MAX) ? INT16_MAX : ((v < INT16_MIN) ? INT16_MIN : v)); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_packs_epi32 + #define _mm256_packs_epi32(a, b) simde_mm256_packs_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_packus_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_packus_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_packus_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_packus_epi16(a_.m128i[1], b_.m128i[1]); + #else + const size_t halfway_point = (sizeof(r_.i8) / sizeof(r_.i8[0])) / 2; + const size_t quarter_point = (sizeof(r_.i8) / sizeof(r_.i8[0])) / 4; + SIMDE_VECTORIZE + for (size_t i = 0 ; i < quarter_point ; i++) { + r_.u8[i] = (a_.i16[i] > UINT8_MAX) ? UINT8_MAX : ((a_.i16[i] < 0) ? UINT8_C(0) : HEDLEY_STATIC_CAST(uint8_t, a_.i16[i])); + r_.u8[i + quarter_point] = (b_.i16[i] > UINT8_MAX) ? UINT8_MAX : ((b_.i16[i] < 0) ? UINT8_C(0) : HEDLEY_STATIC_CAST(uint8_t, b_.i16[i])); + r_.u8[halfway_point + i] = (a_.i16[quarter_point + i] > UINT8_MAX) ? UINT8_MAX : ((a_.i16[quarter_point + i] < 0) ? UINT8_C(0) : HEDLEY_STATIC_CAST(uint8_t, a_.i16[quarter_point + i])); + r_.u8[halfway_point + i + quarter_point] = (b_.i16[quarter_point + i] > UINT8_MAX) ? UINT8_MAX : ((b_.i16[quarter_point + i] < 0) ? UINT8_C(0) : HEDLEY_STATIC_CAST(uint8_t, b_.i16[quarter_point + i])); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_packus_epi16 + #define _mm256_packus_epi16(a, b) simde_mm256_packus_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_packus_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_packus_epi32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_packus_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_packus_epi32(a_.m128i[1], b_.m128i[1]); + #else + const size_t halfway_point = (sizeof(r_.i16) / sizeof(r_.i16[0])) / 2; + const size_t quarter_point = (sizeof(r_.i16) / sizeof(r_.i16[0])) / 4; + SIMDE_VECTORIZE + for (size_t i = 0 ; i < quarter_point ; i++) { + r_.u16[i] = (a_.i32[i] > UINT16_MAX) ? UINT16_MAX : ((a_.i32[i] < 0) ? UINT16_C(0) : HEDLEY_STATIC_CAST(uint16_t, a_.i32[i])); + r_.u16[i + quarter_point] = (b_.i32[i] > UINT16_MAX) ? UINT16_MAX : ((b_.i32[i] < 0) ? UINT16_C(0) : HEDLEY_STATIC_CAST(uint16_t, b_.i32[i])); + r_.u16[halfway_point + i] = (a_.i32[quarter_point + i] > UINT16_MAX) ? UINT16_MAX : ((a_.i32[quarter_point + i] < 0) ? UINT16_C(0) : HEDLEY_STATIC_CAST(uint16_t, a_.i32[quarter_point + i])); + r_.u16[halfway_point + i + quarter_point] = (b_.i32[quarter_point + i] > UINT16_MAX) ? UINT16_MAX : ((b_.i32[quarter_point + i] < 0) ? UINT16_C(0) : HEDLEY_STATIC_CAST(uint16_t, b_.i32[quarter_point + i])); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_packus_epi32 + #define _mm256_packus_epi32(a, b) simde_mm256_packus_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_permute2x128_si256 (simde__m256i a, simde__m256i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + r_.m128i_private[0] = (imm8 & 0x08) ? simde__m128i_to_private(simde_mm_setzero_si128()) : ((imm8 & 0x02) ? b_.m128i_private[(imm8 ) & 1] : a_.m128i_private[(imm8 ) & 1]); + r_.m128i_private[1] = (imm8 & 0x80) ? simde__m128i_to_private(simde_mm_setzero_si128()) : ((imm8 & 0x20) ? b_.m128i_private[(imm8 >> 4) & 1] : a_.m128i_private[(imm8 >> 4) & 1]); + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_permute2x128_si256(a, b, imm8) _mm256_permute2x128_si256(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_permute2x128_si256 + #define _mm256_permute2x128_si256(a, b, imm8) simde_mm256_permute2x128_si256(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_permute4x64_epi64 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + r_.i64[0] = (imm8 & 0x02) ? a_.i64[((imm8 ) & 1)+2] : a_.i64[(imm8 ) & 1]; + r_.i64[1] = (imm8 & 0x08) ? a_.i64[((imm8 >> 2 ) & 1)+2] : a_.i64[(imm8 >> 2 ) & 1]; + r_.i64[2] = (imm8 & 0x20) ? a_.i64[((imm8 >> 4 ) & 1)+2] : a_.i64[(imm8 >> 4 ) & 1]; + r_.i64[3] = (imm8 & 0x80) ? a_.i64[((imm8 >> 6 ) & 1)+2] : a_.i64[(imm8 >> 6 ) & 1]; + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_permute4x64_epi64(a, imm8) _mm256_permute4x64_epi64(a, imm8) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_permute4x64_epi64 + #define _mm256_permute4x64_epi64(a, imm8) simde_mm256_permute4x64_epi64(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_permute4x64_pd (simde__m256d a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + r_.f64[0] = (imm8 & 0x02) ? a_.f64[((imm8 ) & 1)+2] : a_.f64[(imm8 ) & 1]; + r_.f64[1] = (imm8 & 0x08) ? a_.f64[((imm8 >> 2 ) & 1)+2] : a_.f64[(imm8 >> 2 ) & 1]; + r_.f64[2] = (imm8 & 0x20) ? a_.f64[((imm8 >> 4 ) & 1)+2] : a_.f64[(imm8 >> 4 ) & 1]; + r_.f64[3] = (imm8 & 0x80) ? a_.f64[((imm8 >> 6 ) & 1)+2] : a_.f64[(imm8 >> 6 ) & 1]; + + return simde__m256d_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_permute4x64_pd(a, imm8) _mm256_permute4x64_pd(a, imm8) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_permute4x64_pd + #define _mm256_permute4x64_pd(a, imm8) simde_mm256_permute4x64_pd(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_permutevar8x32_epi32 (simde__m256i a, simde__m256i idx) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_permutevar8x32_epi32(a, idx); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + idx_ = simde__m256i_to_private(idx); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[idx_.i32[i] & 7]; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutevar8x32_epi32 + #define _mm256_permutevar8x32_epi32(a, idx) simde_mm256_permutevar8x32_epi32(a, idx) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_permutevar8x32_ps (simde__m256 a, simde__m256i idx) { + #if defined(SIMDE_X86_AVX2_NATIVE) + #if defined(__clang__) && !SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + return _mm256_permutevar8x32_ps(a, HEDLEY_REINTERPRET_CAST(simde__m256, idx)); + #else + return _mm256_permutevar8x32_ps(a, idx); + #endif + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + simde__m256i_private + idx_ = simde__m256i_to_private(idx); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[idx_.i32[i] & 7]; + } + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutevar8x32_ps + #define _mm256_permutevar8x32_ps(a, idx) simde_mm256_permutevar8x32_ps(a, idx) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sad_epu8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_sad_epu8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_sad_epu8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_sad_epu8(a_.m128i[1], b_.m128i[1]); + #else + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + uint16_t tmp = 0; + SIMDE_VECTORIZE_REDUCTION(+:tmp) + for (size_t j = 0 ; j < ((sizeof(r_.u8) / sizeof(r_.u8[0])) / 4) ; j++) { + const size_t e = j + (i * 8); + tmp += (a_.u8[e] > b_.u8[e]) ? (a_.u8[e] - b_.u8[e]) : (b_.u8[e] - a_.u8[e]); + } + r_.i64[i] = tmp; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sad_epu8 + #define _mm256_sad_epu8(a, b) simde_mm256_sad_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_shuffle_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_shuffle_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_shuffle_epi8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_shuffle_epi8(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < ((sizeof(r_.u8) / sizeof(r_.u8[0])) / 2) ; i++) { + r_.u8[ i ] = (b_.u8[ i ] & 0x80) ? 0 : a_.u8[(b_.u8[ i ] & 0x0f) ]; + r_.u8[i + 16] = (b_.u8[i + 16] & 0x80) ? 0 : a_.u8[(b_.u8[i + 16] & 0x0f) + 16]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_shuffle_epi8 + #define _mm256_shuffle_epi8(a, b) simde_mm256_shuffle_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_shuffle_epi32 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + for (size_t i = 0 ; i < ((sizeof(r_.i32) / sizeof(r_.i32[0])) / 2) ; i++) { + r_.i32[i] = a_.i32[(imm8 >> (i * 2)) & 3]; + } + for (size_t i = 0 ; i < ((sizeof(r_.i32) / sizeof(r_.i32[0])) / 2) ; i++) { + r_.i32[i + 4] = a_.i32[((imm8 >> (i * 2)) & 3) + 4]; + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_shuffle_epi32(a, imm8) _mm256_shuffle_epi32(a, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) && !defined(__PGI) +# define simde_mm256_shuffle_epi32(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_shuffle_epi32(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_shuffle_epi32(simde_mm256_extracti128_si256(a, 0), (imm8))) +#elif defined(SIMDE_SHUFFLE_VECTOR_) +# define simde_mm256_shuffle_epi32(a, imm8) (__extension__ ({ \ + const simde__m256i_private simde_tmp_a_ = simde__m256i_to_private(a); \ + simde__m256i_from_private((simde__m256i_private) { .i32 = \ + SIMDE_SHUFFLE_VECTOR_(32, 32, \ + (simde_tmp_a_).i32, \ + (simde_tmp_a_).i32, \ + ((imm8) ) & 3, \ + ((imm8) >> 2) & 3, \ + ((imm8) >> 4) & 3, \ + ((imm8) >> 6) & 3, \ + (((imm8) ) & 3) + 4, \ + (((imm8) >> 2) & 3) + 4, \ + (((imm8) >> 4) & 3) + 4, \ + (((imm8) >> 6) & 3) + 4) }); })) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_shuffle_epi32 + #define _mm256_shuffle_epi32(a, imm8) simde_mm256_shuffle_epi32(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_shufflehi_epi16(a, imm8) _mm256_shufflehi_epi16(a, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) +# define simde_mm256_shufflehi_epi16(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_shufflehi_epi16(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_shufflehi_epi16(simde_mm256_extracti128_si256(a, 0), (imm8))) +#elif defined(SIMDE_SHUFFLE_VECTOR_) +# define simde_mm256_shufflehi_epi16(a, imm8) (__extension__ ({ \ + const simde__m256i_private simde_tmp_a_ = simde__m256i_to_private(a); \ + simde__m256i_from_private((simde__m256i_private) { .i16 = \ + SIMDE_SHUFFLE_VECTOR_(16, 32, \ + (simde_tmp_a_).i16, \ + (simde_tmp_a_).i16, \ + 0, 1, 2, 3, \ + (((imm8) ) & 3) + 4, \ + (((imm8) >> 2) & 3) + 4, \ + (((imm8) >> 4) & 3) + 4, \ + (((imm8) >> 6) & 3) + 4, \ + 8, 9, 10, 11, \ + ((((imm8) ) & 3) + 8 + 4), \ + ((((imm8) >> 2) & 3) + 8 + 4), \ + ((((imm8) >> 4) & 3) + 8 + 4), \ + ((((imm8) >> 6) & 3) + 8 + 4) \ + ) }); })) +#else +# define simde_mm256_shufflehi_epi16(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_shufflehi_epi16(simde_mm256_extracti128_si256(a, 1), imm8), \ + simde_mm_shufflehi_epi16(simde_mm256_extracti128_si256(a, 0), imm8)) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_shufflehi_epi16 + #define _mm256_shufflehi_epi16(a, imm8) simde_mm256_shufflehi_epi16(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_shufflelo_epi16(a, imm8) _mm256_shufflelo_epi16(a, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) +# define simde_mm256_shufflelo_epi16(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_shufflelo_epi16(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_shufflelo_epi16(simde_mm256_extracti128_si256(a, 0), (imm8))) +#elif defined(SIMDE_SHUFFLE_VECTOR_) +# define simde_mm256_shufflelo_epi16(a, imm8) (__extension__ ({ \ + const simde__m256i_private simde_tmp_a_ = simde__m256i_to_private(a); \ + simde__m256i_from_private((simde__m256i_private) { .i16 = \ + SIMDE_SHUFFLE_VECTOR_(16, 32, \ + (simde_tmp_a_).i16, \ + (simde_tmp_a_).i16, \ + (((imm8) ) & 3), \ + (((imm8) >> 2) & 3), \ + (((imm8) >> 4) & 3), \ + (((imm8) >> 6) & 3), \ + 4, 5, 6, 7, \ + ((((imm8) ) & 3) + 8), \ + ((((imm8) >> 2) & 3) + 8), \ + ((((imm8) >> 4) & 3) + 8), \ + ((((imm8) >> 6) & 3) + 8), \ + 12, 13, 14, 15) }); })) +#else +# define simde_mm256_shufflelo_epi16(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_shufflelo_epi16(simde_mm256_extracti128_si256(a, 1), imm8), \ + simde_mm_shufflelo_epi16(simde_mm256_extracti128_si256(a, 0), imm8)) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_shufflelo_epi16 + #define _mm256_shufflelo_epi16(a, imm8) simde_mm256_shufflelo_epi16(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sign_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_sign_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (b_.i8[i] == INT8_C(0)) ? INT8_C(0) : (b_.i8[i] < INT8_C(0)) ? -a_.i8[i] : a_.i8[i]; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sign_epi8 + #define _mm256_sign_epi8(a, b) simde_mm256_sign_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sign_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_sign_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (b_.i16[i] == INT16_C(0)) ? INT16_C(0) : (b_.i16[i] < INT16_C(0)) ? -a_.i16[i] : a_.i16[i]; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sign_epi16 + #define _mm256_sign_epi16(a, b) simde_mm256_sign_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sign_epi32(simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_sign_epi32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0; i < (sizeof(r_.i32) / sizeof(r_.i32[0])); i++) { + r_.i32[i] = (b_.i32[i] == INT32_C(0)) ? INT32_C(0) : (b_.i32[i] < INT32_C(0)) ? -a_.i32[i] : a_.i32[i]; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sign_epi32 + #define _mm256_sign_epi32(a, b) simde_mm256_sign_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sll_epi16 (simde__m256i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_sll_epi16(a, count); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_sll_epi16(a_.m128i[0], count); + r_.m128i[1] = simde_mm_sll_epi16(a_.m128i[1], count); + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + + uint64_t shift = HEDLEY_STATIC_CAST(uint64_t, count_.i64[0]); + if (shift > 15) + return simde_mm256_setzero_si256(); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i16 = a_.i16 << HEDLEY_STATIC_CAST(int16_t, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, a_.i16[i] << (shift)); + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sll_epi16 + #define _mm256_sll_epi16(a, count) simde_mm256_sll_epi16(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sll_epi32 (simde__m256i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_sll_epi32(a, count); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_sll_epi32(a_.m128i[0], count); + r_.m128i[1] = simde_mm_sll_epi32(a_.m128i[1], count); + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + + uint64_t shift = HEDLEY_STATIC_CAST(uint64_t, count_.i64[0]); + if (shift > 31) + return simde_mm256_setzero_si256(); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i32 = a_.i32 << HEDLEY_STATIC_CAST(int32_t, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = HEDLEY_STATIC_CAST(int32_t, a_.i32[i] << (shift)); + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sll_epi32 + #define _mm256_sll_epi32(a, count) simde_mm256_sll_epi32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sll_epi64 (simde__m256i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_sll_epi64(a, count); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_sll_epi64(a_.m128i[0], count); + r_.m128i[1] = simde_mm_sll_epi64(a_.m128i[1], count); + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + + uint64_t shift = HEDLEY_STATIC_CAST(uint64_t, count_.i64[0]); + if (shift > 63) + return simde_mm256_setzero_si256(); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i64 = a_.i64 << HEDLEY_STATIC_CAST(int64_t, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = HEDLEY_STATIC_CAST(int64_t, a_.i64[i] << (shift)); + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sll_epi64 + #define _mm256_sll_epi64(a, count) simde_mm256_sll_epi64(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_slli_epi16 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + /* Note: There is no consistency in how compilers handle values outside of + the expected range, hence the discrepancy between what we allow and what + Intel specifies. Some compilers will return 0, others seem to just mask + off everything outside of the range. */ + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) sv = vec_splats(HEDLEY_STATIC_CAST(unsigned short, imm8)); + for (size_t i = 0 ; i < (sizeof(a_.altivec_i16) / sizeof(a_.altivec_i16[0])) ; i++) { + r_.altivec_i16[i] = vec_sl(a_.altivec_i16[i], sv); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i16 = a_.i16 << HEDLEY_STATIC_CAST(int16_t, imm8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, a_.i16[i] << (imm8 & 0xff)); + } + #endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_slli_epi16(a, imm8) _mm256_slli_epi16(a, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) +# define simde_mm256_slli_epi16(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_slli_epi16(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_slli_epi16(simde_mm256_extracti128_si256(a, 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_slli_epi16 + #define _mm256_slli_epi16(a, imm8) simde_mm256_slli_epi16(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_slli_epi32 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) sv = vec_splats(HEDLEY_STATIC_CAST(unsigned int, imm8)); + for (size_t i = 0 ; i < (sizeof(a_.altivec_i32) / sizeof(a_.altivec_i32[0])) ; i++) { + r_.altivec_i32[i] = vec_sl(a_.altivec_i32[i], sv); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i32 = a_.i32 << HEDLEY_STATIC_CAST(int32_t, imm8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] << (imm8 & 0xff); + } + #endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_slli_epi32(a, imm8) _mm256_slli_epi32(a, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) +# define simde_mm256_slli_epi32(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_slli_epi32(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_slli_epi32(simde_mm256_extracti128_si256(a, 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_slli_epi32 + #define _mm256_slli_epi32(a, imm8) simde_mm256_slli_epi32(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_slli_epi64 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + +#if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i64 = a_.i64 << HEDLEY_STATIC_CAST(int64_t, imm8); +#else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] << (imm8 & 0xff); + } +#endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_slli_epi64(a, imm8) _mm256_slli_epi64(a, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) +# define simde_mm256_slli_epi64(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_slli_epi64(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_slli_epi64(simde_mm256_extracti128_si256(a, 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_slli_epi64 + #define _mm256_slli_epi64(a, imm8) simde_mm256_slli_epi64(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_slli_si256 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + for (size_t h = 0 ; h < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; h++) { + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private[h].i8) / sizeof(r_.m128i_private[h].i8[0])) ; i++) { + const int e = HEDLEY_STATIC_CAST(int, i) - imm8; + r_.m128i_private[h].i8[i] = (e >= 0) ? a_.m128i_private[h].i8[e] : 0; + } + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_slli_si256(a, imm8) _mm256_slli_si256(a, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) && !defined(__PGI) +# define simde_mm256_slli_si256(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_slli_si128(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_slli_si128(simde_mm256_extracti128_si256(a, 0), (imm8))) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) +# define simde_mm256_slli_si256(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_bslli_si128(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_bslli_si128(simde_mm256_extracti128_si256(a, 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_slli_si256 + #define _mm256_slli_si256(a, imm8) simde_mm256_slli_si256(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sllv_epi32 (simde__m128i a, simde__m128i b) { + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vshlq_u32(a_.neon_u32, vreinterpretq_s32_u32(b_.neon_u32)); + r_.neon_u32 = vandq_u32(r_.neon_u32, vcltq_u32(b_.neon_u32, vdupq_n_u32(32))); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), (b_.u32 < UINT32_C(32))) & (a_.u32 << b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (b_.u32[i] < 32) ? (a_.u32[i] << b_.u32[i]) : 0; + } + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_sllv_epi32(a, b) _mm_sllv_epi32(a, b) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_sllv_epi32 + #define _mm_sllv_epi32(a, b) simde_mm_sllv_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sllv_epi32 (simde__m256i a, simde__m256i b) { + simde__m256i_private + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b), + r_; + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_sllv_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_sllv_epi32(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), (b_.u32 < 32)) & (a_.u32 << b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (b_.u32[i] < 32) ? (a_.u32[i] << b_.u32[i]) : 0; + } + #endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_sllv_epi32(a, b) _mm256_sllv_epi32(a, b) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sllv_epi32 + #define _mm256_sllv_epi32(a, b) simde_mm256_sllv_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sllv_epi64 (simde__m128i a, simde__m128i b) { + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vshlq_u64(a_.neon_u64, vreinterpretq_s64_u64(b_.neon_u64)); + r_.neon_u64 = vandq_u64(r_.neon_u64, vcltq_u64(b_.neon_u64, vdupq_n_u64(64))); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), (b_.u64 < 64)) & (a_.u64 << b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (b_.u64[i] < 64) ? (a_.u64[i] << b_.u64[i]) : 0; + } + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_sllv_epi64(a, b) _mm_sllv_epi64(a, b) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_sllv_epi64 + #define _mm_sllv_epi64(a, b) simde_mm_sllv_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sllv_epi64 (simde__m256i a, simde__m256i b) { + simde__m256i_private + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b), + r_; + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_sllv_epi64(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_sllv_epi64(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), (b_.u64 < 64)) & (a_.u64 << b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (b_.u64[i] < 64) ? (a_.u64[i] << b_.u64[i]) : 0; + } + #endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_sllv_epi64(a, b) _mm256_sllv_epi64(a, b) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sllv_epi64 + #define _mm256_sllv_epi64(a, b) simde_mm256_sllv_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sra_epi16 (simde__m256i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_sra_epi16(a, count); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_sra_epi16(a_.m128i[0], count); + r_.m128i[1] = simde_mm_sra_epi16(a_.m128i[1], count); + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + + uint64_t shift = HEDLEY_STATIC_CAST(uint64_t, count_.i64[0]); + + if (shift > 15) shift = 15; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i16 = a_.i16 >> HEDLEY_STATIC_CAST(int16_t, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] >> shift; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sra_epi16 + #define _mm256_sra_epi16(a, count) simde_mm256_sra_epi16(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sra_epi32 (simde__m256i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_sra_epi32(a, count); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_sra_epi32(a_.m128i[0], count); + r_.m128i[1] = simde_mm_sra_epi32(a_.m128i[1], count); + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + uint64_t shift = HEDLEY_STATIC_CAST(uint64_t, count_.i64[0]); + + if (shift > 31) shift = 31; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i32 = a_.i32 >> HEDLEY_STATIC_CAST(int16_t, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] >> shift; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sra_epi32 + #define _mm256_sra_epi32(a, count) simde_mm256_sra_epi32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_srai_epi16 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + unsigned int shift = HEDLEY_STATIC_CAST(unsigned int, imm8); + + if (shift > 15) shift = 15; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i16 = a_.i16 >> HEDLEY_STATIC_CAST(int16_t, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] >> shift; + } + #endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_srai_epi16(a, imm8) _mm256_srai_epi16(a, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) +# define simde_mm256_srai_epi16(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_srai_epi16(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_srai_epi16(simde_mm256_extracti128_si256(a, 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_srai_epi16 + #define _mm256_srai_epi16(a, imm8) simde_mm256_srai_epi16(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_srai_epi32 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + unsigned int shift = HEDLEY_STATIC_CAST(unsigned int, imm8); + + if (shift > 31) shift = 31; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i32 = a_.i32 >> HEDLEY_STATIC_CAST(int16_t, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] >> shift; + } + #endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_srai_epi32(a, imm8) _mm256_srai_epi32(a, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) +# define simde_mm256_srai_epi32(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_srai_epi32(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_srai_epi32(simde_mm256_extracti128_si256(a, 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_srai_epi32 + #define _mm256_srai_epi32(a, imm8) simde_mm256_srai_epi32(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_srav_epi32 (simde__m128i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm_srav_epi32(a, count); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + count_ = simde__m128i_to_private(count); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int32x4_t cnt = vreinterpretq_s32_u32(vminq_u32(count_.neon_u32, vdupq_n_u32(31))); + r_.neon_i32 = vshlq_s32(a_.neon_i32, vnegq_s32(cnt)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + uint32_t shift = HEDLEY_STATIC_CAST(uint32_t, count_.i32[i]); + r_.i32[i] = a_.i32[i] >> HEDLEY_STATIC_CAST(int, shift > 31 ? 31 : shift); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_srav_epi32 + #define _mm_srav_epi32(a, count) simde_mm_srav_epi32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_srav_epi32 (simde__m256i a, simde__m256i count) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_srav_epi32(a, count); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + count_ = simde__m256i_to_private(count); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_srav_epi32(a_.m128i[0], count_.m128i[0]); + r_.m128i[1] = simde_mm_srav_epi32(a_.m128i[1], count_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + uint32_t shift = HEDLEY_STATIC_CAST(uint32_t, count_.i32[i]); + if (shift > 31) shift = 31; + r_.i32[i] = a_.i32[i] >> shift; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_srav_epi32 + #define _mm256_srav_epi32(a, count) simde_mm256_srav_epi32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_srl_epi16 (simde__m256i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_srl_epi16(a, count); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_srl_epi16(a_.m128i[0], count); + r_.m128i[1] = simde_mm_srl_epi16(a_.m128i[1], count); + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + + uint64_t shift = HEDLEY_STATIC_CAST(uint64_t , (count_.i64[0] > 16 ? 16 : count_.i64[0])); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u16 = a_.u16 >> SIMDE_CAST_VECTOR_SHIFT_COUNT(16, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.u16[i] = a_.u16[i] >> (shift); + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_srl_epi16 + #define _mm256_srl_epi16(a, count) simde_mm256_srl_epi16(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_srl_epi32 (simde__m256i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_srl_epi32(a, count); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_srl_epi32(a_.m128i[0], count); + r_.m128i[1] = simde_mm_srl_epi32(a_.m128i[1], count); + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + + uint64_t shift = HEDLEY_STATIC_CAST(uint64_t , (count_.i64[0] > 32 ? 32 : count_.i64[0])); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = a_.u32 >> SIMDE_CAST_VECTOR_SHIFT_COUNT(32, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.u32[i] = a_.u32[i] >> (shift); + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_srl_epi32 + #define _mm256_srl_epi32(a, count) simde_mm256_srl_epi32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_srl_epi64 (simde__m256i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_srl_epi64(a, count); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_srl_epi64(a_.m128i[0], count); + r_.m128i[1] = simde_mm_srl_epi64(a_.m128i[1], count); + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + + uint64_t shift = HEDLEY_STATIC_CAST(uint64_t , (count_.i64[0] > 64 ? 64 : count_.i64[0])); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u64 = a_.u64 >> SIMDE_CAST_VECTOR_SHIFT_COUNT(64, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.u64[i] = a_.u64[i] >> (shift); + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_srl_epi64 + #define _mm256_srl_epi64(a, count) simde_mm256_srl_epi64(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_srli_epi16 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + if (imm8 > 15) + return simde_mm256_setzero_si256(); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) sv = vec_splats(HEDLEY_STATIC_CAST(unsigned short, imm8)); + for (size_t i = 0 ; i < (sizeof(a_.altivec_u16) / sizeof(a_.altivec_u16[0])) ; i++) { + r_.altivec_u16[i] = vec_sr(a_.altivec_u16[i], sv); + } + #else + if (HEDLEY_STATIC_CAST(unsigned int, imm8) > 15) { + simde_memset(&r_, 0, sizeof(r_)); + } else { + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u16 = a_.u16 >> SIMDE_CAST_VECTOR_SHIFT_COUNT(16, imm8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = a_.u16[i] >> imm8; + } + #endif + } + #endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_srli_epi16(a, imm8) _mm256_srli_epi16(a, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) +# define simde_mm256_srli_epi16(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_srli_epi16(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_srli_epi16(simde_mm256_extracti128_si256(a, 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_srli_epi16 + #define _mm256_srli_epi16(a, imm8) simde_mm256_srli_epi16(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_srli_epi32 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) sv = vec_splats(HEDLEY_STATIC_CAST(unsigned int, imm8)); + for (size_t i = 0 ; i < (sizeof(a_.altivec_u32) / sizeof(a_.altivec_u32[0])) ; i++) { + r_.altivec_u32[i] = vec_sr(a_.altivec_u32[i], sv); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = a_.u32 >> SIMDE_CAST_VECTOR_SHIFT_COUNT(16, imm8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] >> imm8; + } + #endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_srli_epi32(a, imm8) _mm256_srli_epi32(a, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) +# define simde_mm256_srli_epi32(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_srli_epi32(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_srli_epi32(simde_mm256_extracti128_si256(a, 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_srli_epi32 + #define _mm256_srli_epi32(a, imm8) simde_mm256_srli_epi32(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_srli_epi64 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + +#if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u64 = a_.u64 >> SIMDE_CAST_VECTOR_SHIFT_COUNT(32, imm8); +#else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = a_.u64[i] >> imm8; + } +#endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_srli_epi64(a, imm8) _mm256_srli_epi64(a, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) +# define simde_mm256_srli_epi64(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_srli_epi64(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_srli_epi64(simde_mm256_extracti128_si256(a, 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_srli_epi64 + #define _mm256_srli_epi64(a, imm8) simde_mm256_srli_epi64(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_srli_si256 (simde__m256i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + for (size_t h = 0 ; h < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; h++) { + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private[h].i8) / sizeof(r_.m128i_private[h].i8[0])) ; i++) { + const int e = imm8 + HEDLEY_STATIC_CAST(int, i); + r_.m128i_private[h].i8[i] = (e < 16) ? a_.m128i_private[h].i8[e] : 0; + } + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) +# define simde_mm256_srli_si256(a, imm8) _mm256_srli_si256(a, imm8) +#elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) && !defined(__PGI) +# define simde_mm256_srli_si256(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_srli_si128(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_srli_si128(simde_mm256_extracti128_si256(a, 0), (imm8))) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) +# define simde_mm256_srli_si256(a, imm8) \ + simde_mm256_set_m128i( \ + simde_mm_bsrli_si128(simde_mm256_extracti128_si256(a, 1), (imm8)), \ + simde_mm_bsrli_si128(simde_mm256_extracti128_si256(a, 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_srli_si256 + #define _mm256_srli_si256(a, imm8) simde_mm256_srli_si256(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_srlv_epi32 (simde__m128i a, simde__m128i b) { + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + r_; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), (b_.u32 < 32)) & (a_.u32 >> b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (b_.u32[i] < 32) ? (a_.u32[i] >> b_.u32[i]) : 0; + } + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_srlv_epi32(a, b) _mm_srlv_epi32(a, b) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_srlv_epi32 + #define _mm_srlv_epi32(a, b) simde_mm_srlv_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_srlv_epi32 (simde__m256i a, simde__m256i b) { + simde__m256i_private + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b), + r_; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), (b_.u32 < 32)) & (a_.u32 >> b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (b_.u32[i] < 32) ? (a_.u32[i] >> b_.u32[i]) : 0; + } + #endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_srlv_epi32(a, b) _mm256_srlv_epi32(a, b) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_srlv_epi32 + #define _mm256_srlv_epi32(a, b) simde_mm256_srlv_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_srlv_epi64 (simde__m128i a, simde__m128i b) { + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + r_; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), (b_.u64 < 64)) & (a_.u64 >> b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (b_.u64[i] < 64) ? (a_.u64[i] >> b_.u64[i]) : 0; + } + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm_srlv_epi64(a, b) _mm_srlv_epi64(a, b) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm_srlv_epi64 + #define _mm_srlv_epi64(a, b) simde_mm_srlv_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_srlv_epi64 (simde__m256i a, simde__m256i b) { + simde__m256i_private + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b), + r_; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), (b_.u64 < 64)) & (a_.u64 >> b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (b_.u64[i] < 64) ? (a_.u64[i] >> b_.u64[i]) : 0; + } + #endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX2_NATIVE) + #define simde_mm256_srlv_epi64(a, b) _mm256_srlv_epi64(a, b) +#endif +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_srlv_epi64 + #define _mm256_srlv_epi64(a, b) simde_mm256_srlv_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_stream_load_si256 (const simde__m256i* mem_addr) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_stream_load_si256(HEDLEY_CONST_CAST(simde__m256i*, mem_addr)); + #elif HEDLEY_HAS_BUILTIN(__builtin_nontemporal_store) && defined(SIMDE_VECTOR_SUBSCRIPT) + return __builtin_nontemporal_load(mem_addr); + #else + simde__m256i r; + simde_memcpy(&r, SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m256i), sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) +# define _mm256_stream_load_si256(mem_addr) simde_mm256_stream_load_si256(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sub_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_sub_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_sub_epi8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_sub_epi8(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = a_.i8 - b_.i8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] - b_.i8[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sub_epi8 + #define _mm256_sub_epi8(a, b) simde_mm256_sub_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sub_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_sub_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_sub_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_sub_epi16(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = a_.i16 - b_.i16; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] - b_.i16[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sub_epi16 + #define _mm256_sub_epi16(a, b) simde_mm256_sub_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_hsub_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_hsub_epi16(a, b); + #else + return simde_mm256_sub_epi16(simde_x_mm256_deinterleaveeven_epi16(a, b), simde_x_mm256_deinterleaveodd_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_hsub_epi16 + #define _mm256_hsub_epi16(a, b) simde_mm256_hsub_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sub_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_sub_epi32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_sub_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_sub_epi32(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 - b_.i32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] - b_.i32[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sub_epi32 + #define _mm256_sub_epi32(a, b) simde_mm256_sub_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_hsub_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_hsub_epi32(a, b); + #else + return simde_mm256_sub_epi32(simde_x_mm256_deinterleaveeven_epi32(a, b), simde_x_mm256_deinterleaveodd_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_hsub_epi32 + #define _mm256_hsub_epi32(a, b) simde_mm256_hsub_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_sub_epi64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_sub_epi64(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_sub_epi64(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_sub_epi64(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 - b_.i64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] - b_.i64[i]; + } + #endif + + return simde__m256i_from_private(r_); +#endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_sub_epi64 + #define _mm256_sub_epi64(a, b) simde_mm256_sub_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_sub_epu32 (simde__m256i a, simde__m256i b) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = a_.u32 - b_.u32; + #elif SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_x_mm_sub_epu32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_x_mm_sub_epu32(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] - b_.u32[i]; + } + #endif + + return simde__m256i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_subs_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_subs_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_subs_epi8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_subs_epi8(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = simde_math_subs_i8(a_.i8[i], b_.i8[i]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_subs_epi8 + #define _mm256_subs_epi8(a, b) simde_mm256_subs_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_subs_epi16(simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_subs_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_subs_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_subs_epi16(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = simde_math_subs_i16(a_.i16[i], b_.i16[i]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_subs_epi16 + #define _mm256_subs_epi16(a, b) simde_mm256_subs_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_hsubs_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_hsubs_epi16(a, b); + #else + return simde_mm256_subs_epi16(simde_x_mm256_deinterleaveeven_epi16(a, b), simde_x_mm256_deinterleaveodd_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_hsubs_epi16 + #define _mm256_hsubs_epi16(a, b) simde_mm256_hsubs_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_subs_epu8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_subs_epu8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_subs_epu8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_subs_epu8(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = simde_math_subs_u8(a_.u8[i], b_.u8[i]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_subs_epu8 + #define _mm256_subs_epu8(a, b) simde_mm256_subs_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_subs_epu16(simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_subs_epu16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_subs_epu16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_subs_epu16(a_.m128i[1], b_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = simde_math_subs_u16(a_.u16[i], b_.u16[i]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_subs_epu16 + #define _mm256_subs_epu16(a, b) simde_mm256_subs_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_x_mm256_test_all_ones (simde__m256i a) { + simde__m256i_private a_ = simde__m256i_to_private(a); + int r; + int_fast32_t r_ = ~HEDLEY_STATIC_CAST(int_fast32_t, 0); + + SIMDE_VECTORIZE_REDUCTION(&:r_) + for (size_t i = 0 ; i < (sizeof(a_.i32f) / sizeof(a_.i32f[0])) ; i++) { + r_ &= a_.i32f[i]; + } + + r = (r_ == ~HEDLEY_STATIC_CAST(int_fast32_t, 0)); + + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_unpacklo_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_unpacklo_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_unpacklo_epi8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_unpacklo_epi8(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i8 = SIMDE_SHUFFLE_VECTOR_(8, 32, a_.i8, b_.i8, + 0, 32, 1, 33, 2, 34, 3, 35, + 4, 36, 5, 37, 6, 38, 7, 39, + 16, 48, 17, 49, 18, 50, 19, 51, + 20, 52, 21, 53, 22, 54, 23, 55); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0]) / 2) ; i++) { + r_.i8[2 * i] = a_.i8[i + ~(~i | 7)]; + r_.i8[2 * i + 1] = b_.i8[i + ~(~i | 7)]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_unpacklo_epi8 + #define _mm256_unpacklo_epi8(a, b) simde_mm256_unpacklo_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_unpacklo_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_unpacklo_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_unpacklo_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_unpacklo_epi16(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i16 =SIMDE_SHUFFLE_VECTOR_(16, 32, a_.i16, b_.i16, + 0, 16, 1, 17, 2, 18, 3, 19, 8, 24, 9, 25, 10, 26, 11, 27); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0]) / 2) ; i++) { + r_.i16[2 * i] = a_.i16[i + ~(~i | 3)]; + r_.i16[2 * i + 1] = b_.i16[i + ~(~i | 3)]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_unpacklo_epi16 + #define _mm256_unpacklo_epi16(a, b) simde_mm256_unpacklo_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_unpacklo_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_unpacklo_epi32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_unpacklo_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_unpacklo_epi32(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i32 = SIMDE_SHUFFLE_VECTOR_(32, 32, a_.i32, b_.i32, + 0, 8, 1, 9, 4, 12, 5, 13); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0]) / 2) ; i++) { + r_.i32[2 * i] = a_.i32[i + ~(~i | 1)]; + r_.i32[2 * i + 1] = b_.i32[i + ~(~i | 1)]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_unpacklo_epi32 + #define _mm256_unpacklo_epi32(a, b) simde_mm256_unpacklo_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_unpacklo_epi64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_unpacklo_epi64(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_unpacklo_epi64(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_unpacklo_epi64(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i64 = SIMDE_SHUFFLE_VECTOR_(64, 32, a_.i64, b_.i64, 0, 4, 2, 6); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0]) / 2) ; i++) { + r_.i64[2 * i] = a_.i64[2 * i]; + r_.i64[2 * i + 1] = b_.i64[2 * i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_unpacklo_epi64 + #define _mm256_unpacklo_epi64(a, b) simde_mm256_unpacklo_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_unpackhi_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_unpackhi_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_unpackhi_epi8(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_unpackhi_epi8(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i8 = SIMDE_SHUFFLE_VECTOR_(8, 32, a_.i8, b_.i8, + 8, 40, 9, 41, 10, 42, 11, 43, + 12, 44, 13, 45, 14, 46, 15, 47, + 24, 56, 25, 57, 26, 58, 27, 59, + 28, 60, 29, 61, 30, 62, 31, 63); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0]) / 2) ; i++) { + r_.i8[2 * i] = a_.i8[i + 8 + ~(~i | 7)]; + r_.i8[2 * i + 1] = b_.i8[i + 8 + ~(~i | 7)]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_unpackhi_epi8 + #define _mm256_unpackhi_epi8(a, b) simde_mm256_unpackhi_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_unpackhi_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_unpackhi_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_unpackhi_epi16(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_unpackhi_epi16(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i16 = SIMDE_SHUFFLE_VECTOR_(16, 32, a_.i16, b_.i16, + 4, 20, 5, 21, 6, 22, 7, 23, + 12, 28, 13, 29, 14, 30, 15, 31); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0]) / 2) ; i++) { + r_.i16[2 * i] = a_.i16[i + 4 + ~(~i | 3)]; + r_.i16[2 * i + 1] = b_.i16[i + 4 + ~(~i | 3)]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_unpackhi_epi16 + #define _mm256_unpackhi_epi16(a, b) simde_mm256_unpackhi_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_unpackhi_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_unpackhi_epi32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_unpackhi_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_unpackhi_epi32(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i32 = SIMDE_SHUFFLE_VECTOR_(32, 32, a_.i32, b_.i32, + 2, 10, 3, 11, 6, 14, 7, 15); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0]) / 2) ; i++) { + r_.i32[2 * i] = a_.i32[i + 2 + ~(~i | 1)]; + r_.i32[2 * i + 1] = b_.i32[i + 2 + ~(~i | 1)]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_unpackhi_epi32 + #define _mm256_unpackhi_epi32(a, b) simde_mm256_unpackhi_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_unpackhi_epi64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_unpackhi_epi64(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_unpackhi_epi64(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_unpackhi_epi64(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i64 = SIMDE_SHUFFLE_VECTOR_(64, 32, a_.i64, b_.i64, 1, 5, 3, 7); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0]) / 2) ; i++) { + r_.i64[2 * i] = a_.i64[2 * i + 1]; + r_.i64[2 * i + 1] = b_.i64[2 * i + 1]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_unpackhi_epi64 + #define _mm256_unpackhi_epi64(a, b) simde_mm256_unpackhi_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_xor_si256 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_xor_si256(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_INT_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_xor_si128(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_xor_si128(a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f ^ b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] ^ b_.i64[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX2_ENABLE_NATIVE_ALIASES) + #undef _mm256_xor_si256 + #define _mm256_xor_si256(a, b) simde_mm256_xor_si256(a, b) +#endif + +SIMDE_END_DECLS_ + +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX2_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512.h new file mode 100644 index 00000000..103b4666 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512.h @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_H) +#define SIMDE_X86_AVX512_H + +#include "avx512/types.h" + +#include "avx512/2intersect.h" +#include "avx512/4dpwssd.h" +#include "avx512/4dpwssds.h" +#include "avx512/abs.h" +#include "avx512/add.h" +#include "avx512/adds.h" +#include "avx512/and.h" +#include "avx512/andnot.h" +#include "avx512/avg.h" +#include "avx512/bitshuffle.h" +#include "avx512/blend.h" +#include "avx512/broadcast.h" +#include "avx512/cast.h" +#include "avx512/cmp.h" +#include "avx512/cmpeq.h" +#include "avx512/cmpge.h" +#include "avx512/cmpgt.h" +#include "avx512/cmple.h" +#include "avx512/cmplt.h" +#include "avx512/cmpneq.h" +#include "avx512/compress.h" +#include "avx512/conflict.h" +#include "avx512/copysign.h" +#include "avx512/cvt.h" +#include "avx512/cvtt.h" +#include "avx512/cvts.h" +#include "avx512/cvtus.h" +#include "avx512/dbsad.h" +#include "avx512/div.h" +#include "avx512/dpbf16.h" +#include "avx512/dpbusd.h" +#include "avx512/dpbusds.h" +#include "avx512/dpwssd.h" +#include "avx512/dpwssds.h" +#include "avx512/expand.h" +#include "avx512/extract.h" +#include "avx512/fixupimm.h" +#include "avx512/fixupimm_round.h" +#include "avx512/flushsubnormal.h" +#include "avx512/fmadd.h" +#include "avx512/fmsub.h" +#include "avx512/fnmadd.h" +#include "avx512/fnmsub.h" +#include "avx512/fpclass.h" +#include "avx512/gather.h" +#include "avx512/insert.h" +#include "avx512/kand.h" +#include "avx512/kshift.h" +#include "avx512/knot.h" +#include "avx512/kxor.h" +#include "avx512/load.h" +#include "avx512/loadu.h" +#include "avx512/lzcnt.h" +#include "avx512/madd.h" +#include "avx512/maddubs.h" +#include "avx512/max.h" +#include "avx512/min.h" +#include "avx512/mov.h" +#include "avx512/mov_mask.h" +#include "avx512/movm.h" +#include "avx512/mul.h" +#include "avx512/mulhi.h" +#include "avx512/mulhrs.h" +#include "avx512/mullo.h" +#include "avx512/multishift.h" +#include "avx512/negate.h" +#include "avx512/or.h" +#include "avx512/packs.h" +#include "avx512/packus.h" +#include "avx512/permutex.h" +#include "avx512/permutexvar.h" +#include "avx512/permutex2var.h" +#include "avx512/popcnt.h" +#include "avx512/range.h" +#include "avx512/range_round.h" +#include "avx512/rcp.h" +#include "avx512/reduce.h" +#include "avx512/rol.h" +#include "avx512/rolv.h" +#include "avx512/ror.h" +#include "avx512/rorv.h" +#include "avx512/round.h" +#include "avx512/roundscale.h" +#include "avx512/roundscale_round.h" +#include "avx512/sad.h" +#include "avx512/scalef.h" +#include "avx512/set.h" +#include "avx512/set1.h" +#include "avx512/set4.h" +#include "avx512/setr.h" +#include "avx512/setr4.h" +#include "avx512/setzero.h" +#include "avx512/setone.h" +#include "avx512/shldv.h" +#include "avx512/shuffle.h" +#include "avx512/sll.h" +#include "avx512/slli.h" +#include "avx512/sllv.h" +#include "avx512/sqrt.h" +#include "avx512/sra.h" +#include "avx512/srai.h" +#include "avx512/srav.h" +#include "avx512/srl.h" +#include "avx512/srli.h" +#include "avx512/srlv.h" +#include "avx512/store.h" +#include "avx512/storeu.h" +#include "avx512/sub.h" +#include "avx512/subs.h" +#include "avx512/ternarylogic.h" +#include "avx512/test.h" +#include "avx512/testn.h" +#include "avx512/unpacklo.h" +#include "avx512/unpackhi.h" +#include "avx512/xor.h" +#include "avx512/xorsign.h" + +#endif diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/2intersect.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/2intersect.h new file mode 100644 index 00000000..81b0ee1f --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/2intersect.h @@ -0,0 +1,249 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Ashleigh Newman-Jones + */ + +#if !defined(SIMDE_X86_AVX512_2INTERSECT_H) +#define SIMDE_X86_AVX512_2INTERSECT_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_2intersect_epi32(simde__m128i a, simde__m128i b, simde__mmask8 *k1, simde__mmask8 *k2) { + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + simde__mmask8 + k1_ = 0, + k2_ = 0; + + for (size_t i = 0 ; i < sizeof(a_.i32) / sizeof(a_.i32[0]) ; i++) { + #if defined(SIMDE_ENABLE_OPENMP) + #pragma omp simd reduction(|:k1_) reduction(|:k2_) + #else + SIMDE_VECTORIZE + #endif + for (size_t j = 0 ; j < sizeof(b_.i32) / sizeof(b_.i32[0]) ; j++) { + const int32_t m = a_.i32[i] == b_.i32[j]; + k1_ |= m << i; + k2_ |= m << j; + } + } + + *k1 = k1_; + *k2 = k2_; +} +#if defined(SIMDE_X86_AVX512VP2INTERSECT_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_2intersect_epi32(a, b, k1, k2) _mm_2intersect_epi32(a, b, k1, k2) +#endif +#if defined(SIMDE_X86_AVX512VP2INTERSECT_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_2intersect_epi32 + #define _mm_2intersect_epi32(a, b, k1, k2) simde_mm_2intersect_epi32(a, b, k1, k2) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_2intersect_epi64(simde__m128i a, simde__m128i b, simde__mmask8 *k1, simde__mmask8 *k2) { + #if defined(SIMDE_X86_AVX512VP2INTERSECT_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + _mm_2intersect_epi64(a, b, k1, k2); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + simde__mmask8 + k1_ = 0, + k2_ = 0; + + for (size_t i = 0 ; i < sizeof(a_.i64) / sizeof(a_.i64[0]) ; i++) { + #if defined(SIMDE_ENABLE_OPENMP) + #pragma omp simd reduction(|:k1_) reduction(|:k2_) + #else + SIMDE_VECTORIZE + #endif + for (size_t j = 0 ; j < sizeof(b_.i64) / sizeof(b_.i64[0]) ; j++) { + const int32_t m = a_.i64[i] == b_.i64[j]; + k1_ |= m << i; + k2_ |= m << j; + } + } + + *k1 = k1_; + *k2 = k2_; + #endif +} +#if defined(SIMDE_X86_AVX512VP2INTERSECT_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_2intersect_epi64 + #define _mm_2intersect_epi64(a, b, k1, k2) simde_mm_2intersect_epi64(a, b, k1, k2) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_2intersect_epi32(simde__m256i a, simde__m256i b, simde__mmask8 *k1, simde__mmask8 *k2) { + #if defined(SIMDE_X86_AVX512VP2INTERSECT_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + _mm256_2intersect_epi32(a, b, k1, k2); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + simde__mmask8 + k1_ = 0, + k2_ = 0; + + for (size_t i = 0 ; i < sizeof(a_.i32) / sizeof(a_.i32[0]) ; i++) { + #if defined(SIMDE_ENABLE_OPENMP) + #pragma omp simd reduction(|:k1_) reduction(|:k2_) + #else + SIMDE_VECTORIZE + #endif + for (size_t j = 0 ; j < sizeof(b_.i32) / sizeof(b_.i32[0]) ; j++) { + const int32_t m = a_.i32[i] == b_.i32[j]; + k1_ |= m << i; + k2_ |= m << j; + } + } + + *k1 = k1_; + *k2 = k2_; + #endif +} +#if defined(SIMDE_X86_AVX512VP2INTERSECT_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_2intersect_epi32 + #define _mm256_2intersect_epi32(a, b, k1, k2) simde_mm256_2intersect_epi32(a, b, k1, k2) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_2intersect_epi64(simde__m256i a, simde__m256i b, simde__mmask8 *k1, simde__mmask8 *k2) { + #if defined(SIMDE_X86_AVX512VP2INTERSECT_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + _mm256_2intersect_epi64(a, b, k1, k2); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + simde__mmask8 + k1_ = 0, + k2_ = 0; + + for (size_t i = 0 ; i < sizeof(a_.i64) / sizeof(a_.i64[0]) ; i++) { + #if defined(SIMDE_ENABLE_OPENMP) + #pragma omp simd reduction(|:k1_) reduction(|:k2_) + #else + SIMDE_VECTORIZE + #endif + for (size_t j = 0 ; j < sizeof(b_.i64) / sizeof(b_.i64[0]) ; j++) { + const int32_t m = a_.i64[i] == b_.i64[j]; + k1_ |= m << i; + k2_ |= m << j; + } + } + + *k1 = k1_; + *k2 = k2_; + #endif +} +#if defined(SIMDE_X86_AVX512VP2INTERSECT_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_2intersect_epi64 + #define _mm256_2intersect_epi64(a, b, k1, k2) simde_mm256_2intersect_epi64(a, b, k1, k2) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_2intersect_epi32(simde__m512i a, simde__m512i b, simde__mmask16 *k1, simde__mmask16 *k2) { + #if defined(SIMDE_X86_AVX512VP2INTERSECT_NATIVE) + _mm512_2intersect_epi32(a, b, k1, k2); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + simde__mmask16 + k1_ = 0, + k2_ = 0; + + for (size_t i = 0 ; i < sizeof(a_.i32) / sizeof(a_.i32[0]) ; i++) { + #if defined(SIMDE_ENABLE_OPENMP) + #pragma omp simd reduction(|:k1_) reduction(|:k2_) + #else + SIMDE_VECTORIZE + #endif + for (size_t j = 0 ; j < sizeof(b_.i32) / sizeof(b_.i32[0]) ; j++) { + const int32_t m = a_.i32[i] == b_.i32[j]; + k1_ |= m << i; + k2_ |= m << j; + } + } + + *k1 = k1_; + *k2 = k2_; + #endif +} +#if defined(SIMDE_X86_AVX512VP2INTERSECT_ENABLE_NATIVE_ALIASES) + #undef _mm512_2intersect_epi32 + #define _mm512_2intersect_epi32(a, b, k1, k2) simde_mm512_2intersect_epi32(a, b, k1, k2) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_2intersect_epi64(simde__m512i a, simde__m512i b, simde__mmask8 *k1, simde__mmask8 *k2) { + #if defined(SIMDE_X86_AVX512VP2INTERSECT_NATIVE) + _mm512_2intersect_epi64(a, b, k1, k2); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + simde__mmask8 + k1_ = 0, + k2_ = 0; + + for (size_t i = 0 ; i < sizeof(a_.i64) / sizeof(a_.i64[0]) ; i++) { + #if defined(SIMDE_ENABLE_OPENMP) + #pragma omp simd reduction(|:k1_) reduction(|:k2_) + #else + SIMDE_VECTORIZE + #endif + for (size_t j = 0 ; j < sizeof(b_.i64) / sizeof(b_.i64[0]) ; j++) { + const int32_t m = a_.i64[i] == b_.i64[j]; + k1_ |= m << i; + k2_ |= m << j; + } + } + + *k1 = k1_; + *k2 = k2_; + #endif +} +#if defined(SIMDE_X86_AVX512VP2INTERSECT_ENABLE_NATIVE_ALIASES) + #undef _mm512_2intersect_epi64 + #define _mm512_2intersect_epi64(a, b, k1, k2) simde_mm512_2intersect_epi64(a, b, k1, k2) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_2INTERSECT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/4dpwssd.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/4dpwssd.h new file mode 100644 index 00000000..2139099f --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/4dpwssd.h @@ -0,0 +1,67 @@ +#if !defined(SIMDE_X86_AVX512_4DPWSSD_H) +#define SIMDE_X86_AVX512_4DPWSSD_H + +#include "types.h" +#include "dpwssd.h" +#include "set1.h" +#include "mov.h" +#include "add.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_4dpwssd_epi32 (simde__m512i src, simde__m512i a0, simde__m512i a1, simde__m512i a2, simde__m512i a3, simde__m128i* b) { + #if defined(SIMDE_X86_AVX5124VNNIW_NATIVE) + return _mm512_4dpwssd_epi32(src, a0, a1, a2, a3, b); + #else + simde__m128i_private bv = simde__m128i_to_private(simde_mm_loadu_epi32(b)); + simde__m512i r; + + r = simde_mm512_dpwssd_epi32(src, a0, simde_mm512_set1_epi32(bv.i32[0])); + r = simde_mm512_add_epi32(simde_mm512_dpwssd_epi32(src, a1, simde_mm512_set1_epi32(bv.i32[1])), r); + r = simde_mm512_add_epi32(simde_mm512_dpwssd_epi32(src, a2, simde_mm512_set1_epi32(bv.i32[2])), r); + r = simde_mm512_add_epi32(simde_mm512_dpwssd_epi32(src, a3, simde_mm512_set1_epi32(bv.i32[3])), r); + + return r; + #endif +} +#if defined(SIMDE_X86_AVX5124VNNIW_ENABLE_NATIVE_ALIASES) + #undef simde_mm512_4dpwssd_epi32 + #define _mm512_4dpwssd_epi32(src, a0, a1, a2, a3, b) simde_mm512_4dpwssd_epi32(src, a0, a1, a2, a3, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_4dpwssd_epi32 (simde__m512i src, simde__mmask16 k, simde__m512i a0, simde__m512i a1, simde__m512i a2, simde__m512i a3, simde__m128i* b) { + #if defined(SIMDE_X86_AVX5124VNNIW_NATIVE) + return _mm512_mask_4dpwssd_epi32(src, k, a0, a1, a2, a3, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_4dpwssd_epi32(src, a0, a1, a2, a3, b)); + #endif +} +#if defined(SIMDE_X86_AVX5124VNNIW_ENABLE_NATIVE_ALIASES) + #undef simde_mm512_mask_4dpwssd_epi32 + #define _mm512_mask_4dpwssd_epi32(src, k, a0, a1, a2, a3, b) simde_mm512_mask_4dpwssd_epi32(src, k, a0, a1, a2, a3, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_4dpwssd_epi32 (simde__mmask16 k, simde__m512i src, simde__m512i a0, simde__m512i a1, simde__m512i a2, simde__m512i a3, simde__m128i* b) { + #if defined(SIMDE_X86_AVX5124VNNIW_NATIVE) + return _mm512_mask_4dpwssd_epi32(k, src, a0, a1, a2, a3, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_4dpwssd_epi32(src, a0, a1, a2, a3, b)); + #endif +} +#if defined(SIMDE_X86_AVX5124VNNIW_ENABLE_NATIVE_ALIASES) + #undef simde_mm512_maskz_4dpwssd_epi32 + #define _mm512_maskz_4dpwssd_epi32(k, src, a0, a1, a2, a3, b) simde_mm512_maskz_4dpwssd_epi32(k, src, a0, a1, a2, a3, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_4DPWSSD_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/4dpwssds.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/4dpwssds.h new file mode 100644 index 00000000..ef8cf978 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/4dpwssds.h @@ -0,0 +1,67 @@ +#if !defined(SIMDE_X86_AVX512_4DPWSSDS_H) +#define SIMDE_X86_AVX512_4DPWSSDS_H + +#include "types.h" +#include "dpwssds.h" +#include "set1.h" +#include "mov.h" +#include "adds.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_4dpwssds_epi32 (simde__m512i src, simde__m512i a0, simde__m512i a1, simde__m512i a2, simde__m512i a3, simde__m128i* b) { + #if defined(SIMDE_X86_AVX5124VNNIW_NATIVE) + return _mm512_4dpwssds_epi32(src, a0, a1, a2, a3, b); + #else + simde__m128i_private bv = simde__m128i_to_private(simde_mm_loadu_epi32(b)); + simde__m512i r; + + r = simde_mm512_dpwssds_epi32(src, a0, simde_mm512_set1_epi32(bv.i32[0])); + r = simde_x_mm512_adds_epi32(simde_mm512_dpwssds_epi32(src, a1, simde_mm512_set1_epi32(bv.i32[1])), r); + r = simde_x_mm512_adds_epi32(simde_mm512_dpwssds_epi32(src, a2, simde_mm512_set1_epi32(bv.i32[2])), r); + r = simde_x_mm512_adds_epi32(simde_mm512_dpwssds_epi32(src, a3, simde_mm512_set1_epi32(bv.i32[3])), r); + + return r; + #endif +} +#if defined(SIMDE_X86_AVX5124VNNIW_ENABLE_NATIVE_ALIASES) + #undef simde_mm512_4dpwssds_epi32 + #define _mm512_4dpwssds_epi32(src, a0, a1, a2, a3, b) simde_mm512_4dpwssds_epi32(src, a0, a1, a2, a3, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_4dpwssds_epi32 (simde__m512i src, simde__mmask16 k, simde__m512i a0, simde__m512i a1, simde__m512i a2, simde__m512i a3, simde__m128i* b) { + #if defined(SIMDE_X86_AVX5124VNNIW_NATIVE) + return _mm512_mask_4dpwssds_epi32(src, k, a0, a1, a2, a3, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_4dpwssds_epi32(src, a0, a1, a2, a3, b)); + #endif +} +#if defined(SIMDE_X86_AVX5124VNNIW_ENABLE_NATIVE_ALIASES) + #undef simde_mm512_mask_4dpwssds_epi32 + #define _mm512_mask_4dpwssds_epi32(src, k, a0, a1, a2, a3, b) simde_mm512_mask_4dpwssds_epi32(src, k, a0, a1, a2, a3, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_4dpwssds_epi32 (simde__mmask16 k, simde__m512i src, simde__m512i a0, simde__m512i a1, simde__m512i a2, simde__m512i a3, simde__m128i* b) { + #if defined(SIMDE_X86_AVX5124VNNIW_NATIVE) + return _mm512_mask_4dpwssds_epi32(k, src, a0, a1, a2, a3, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_4dpwssds_epi32(src, a0, a1, a2, a3, b)); + #endif +} +#if defined(SIMDE_X86_AVX5124VNNIW_ENABLE_NATIVE_ALIASES) + #undef simde_mm512_maskz_4dpwssds_epi32 + #define _mm512_maskz_4dpwssds_epi32(k, src, a0, a1, a2, a3, b) simde_mm512_maskz_4dpwssds_epi32(k, src, a0, a1, a2, a3, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_4DPWSSDS_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/abs.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/abs.h new file mode 100644 index 00000000..5ff00148 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/abs.h @@ -0,0 +1,580 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_ABS_H) +#define SIMDE_X86_AVX512_ABS_H + +#include "types.h" +#include "mov.h" +#include "../avx2.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_abs_epi8(simde__m128i src, simde__mmask16 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_abs_epi8(src, k, a); + #else + return simde_mm_mask_mov_epi8(src, k, simde_mm_abs_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_abs_epi8 + #define _mm_mask_abs_epi8(src, k, a) simde_mm_mask_abs_epi8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_abs_epi8(simde__mmask16 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_maskz_abs_epi8(k, a); + #else + return simde_mm_maskz_mov_epi8(k, simde_mm_abs_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_abs_epi8 + #define _mm_maskz_abs_epi8(k, a) simde_mm_maskz_abs_epi8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_abs_epi16(simde__m128i src, simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_abs_epi16(src, k, a); + #else + return simde_mm_mask_mov_epi16(src, k, simde_mm_abs_epi16(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_abs_epi16 + #define _mm_mask_abs_epi16(src, k, a) simde_mm_mask_abs_epi16(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_abs_epi16(simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_maskz_abs_epi16(k, a); + #else + return simde_mm_maskz_mov_epi16(k, simde_mm_abs_epi16(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_abs_epi16 + #define _mm_maskz_abs_epi16(k, a) simde_mm_maskz_abs_epi16(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_abs_epi32(simde__m128i src, simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_abs_epi32(src, k, a); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_abs_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_abs_epi32 + #define _mm_mask_abs_epi32(src, k, a) simde_mm_mask_abs_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_abs_epi32(simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_abs_epi32(k, a); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_abs_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_abs_epi32 + #define _mm_maskz_abs_epi32(k, a) simde_mm_maskz_abs_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_abs_epi64(simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_abs_epi64(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + const __m128i m = _mm_srai_epi32(_mm_shuffle_epi32(a, 0xF5), 31); + return _mm_sub_epi64(_mm_xor_si128(a, m), m); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i64 = vabsq_s64(a_.neon_i64); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const int64x2_t m = vshrq_n_s64(a_.neon_i64, 63); + r_.neon_i64 = vsubq_s64(veorq_s64(a_.neon_i64, m), m); + #elif (defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) && !defined(HEDLEY_IBM_VERSION)) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i64 = vec_abs(a_.altivec_i64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_abs(a_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + __typeof__(r_.i64) z = { 0, }; + __typeof__(r_.i64) m = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 < z); + r_.i64 = (-a_.i64 & m) | (a_.i64 & ~m); + #else + SIMDE_VECTORIZE + for (size_t i = 0; i < (sizeof(r_.i64) / sizeof(r_.i64[0])); i++) { + r_.i64[i] = (a_.i64[i] < INT64_C(0)) ? -a_.i64[i] : a_.i64[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_abs_epi64 + #define _mm_abs_epi64(a) simde_mm_abs_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_abs_epi64(simde__m128i src, simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_abs_epi64(src, k, a); + #else + return simde_mm_mask_mov_epi64(src, k, simde_mm_abs_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_abs_epi64 + #define _mm_mask_abs_epi64(src, k, a) simde_mm_mask_abs_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_abs_epi64(simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_abs_epi64(k, a); + #else + return simde_mm_maskz_mov_epi64(k, simde_mm_abs_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_abs_epi64 + #define _mm_maskz_abs_epi64(k, a) simde_mm_maskz_abs_epi64(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_abs_epi64(simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_abs_epi64(a); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_abs_epi64(a_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0; i < (sizeof(r_.i64) / sizeof(r_.i64[0])); i++) { + r_.i64[i] = (a_.i64[i] < INT64_C(0)) ? -a_.i64[i] : a_.i64[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_abs_epi64 + #define _mm256_abs_epi64(a) simde_mm256_abs_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_abs_epi64(simde__m256i src, simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_abs_epi64(src, k, a); + #else + return simde_mm256_mask_mov_epi64(src, k, simde_mm256_abs_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_abs_epi64 + #define _mm256_mask_abs_epi64(src, k, a) simde_mm256_mask_abs_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_abs_epi64(simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_abs_epi64(k, a); + #else + return simde_mm256_maskz_mov_epi64(k, simde_mm256_abs_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_abs_epi64 + #define _mm256_maskz_abs_epi64(k, a) simde_mm256_maskz_abs_epi64(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_abs_epi8 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_abs_epi8(a); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_abs_epi8(a_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] < INT32_C(0)) ? -a_.i8[i] : a_.i8[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_abs_epi8 + #define _mm512_abs_epi8(a) simde_mm512_abs_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_abs_epi8 (simde__m512i src, simde__mmask64 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_abs_epi8(src, k, a); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_abs_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_abs_epi8 + #define _mm512_mask_abs_epi8(src, k, a) simde_mm512_mask_abs_epi8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_abs_epi8 (simde__mmask64 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_abs_epi8(k, a); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_abs_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_abs_epi8 + #define _mm512_maskz_abs_epi8(k, a) simde_mm512_maskz_abs_epi8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_abs_epi16 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_abs_epi16(a); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_abs_epi16(a_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] < INT32_C(0)) ? -a_.i16[i] : a_.i16[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_abs_epi16 + #define _mm512_abs_epi16(a) simde_mm512_abs_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_abs_epi16 (simde__m512i src, simde__mmask32 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_abs_epi16(src, k, a); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_abs_epi16(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_abs_epi16 + #define _mm512_mask_abs_epi16(src, k, a) simde_mm512_mask_abs_epi16(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_abs_epi16 (simde__mmask32 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_abs_epi16(k, a); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_abs_epi16(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_abs_epi16 + #define _mm512_maskz_abs_epi16(k, a) simde_mm512_maskz_abs_epi16(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_abs_epi32(simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_abs_epi32(a); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_abs_epi32(a_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0; i < (sizeof(r_.i32) / sizeof(r_.i32[0])); i++) { + r_.i32[i] = (a_.i32[i] < INT64_C(0)) ? -a_.i32[i] : a_.i32[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_abs_epi32 + #define _mm512_abs_epi32(a) simde_mm512_abs_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_abs_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_abs_epi32(src, k, a); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_abs_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_abs_epi32 + #define _mm512_mask_abs_epi32(src, k, a) simde_mm512_mask_abs_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_abs_epi32(simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_abs_epi32(k, a); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_abs_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_abs_epi32 + #define _mm512_maskz_abs_epi32(k, a) simde_mm512_maskz_abs_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_abs_epi64(simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_abs_epi64(a); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_abs_epi64(a_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0; i < (sizeof(r_.i64) / sizeof(r_.i64[0])); i++) { + r_.i64[i] = (a_.i64[i] < INT64_C(0)) ? -a_.i64[i] : a_.i64[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_abs_epi64 + #define _mm512_abs_epi64(a) simde_mm512_abs_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_abs_epi64(simde__m512i src, simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_abs_epi64(src, k, a); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_abs_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_abs_epi64 + #define _mm512_mask_abs_epi64(src, k, a) simde_mm512_mask_abs_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_abs_epi64(simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_abs_epi64(k, a); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_abs_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_abs_epi64 + #define _mm512_maskz_abs_epi64(k, a) simde_mm512_maskz_abs_epi64(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_abs_ps(simde__m512 v2) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) + return _mm512_abs_ps(v2); + #else + simde__m512_private + r_, + v2_ = simde__m512_to_private(v2); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128_private) / sizeof(r_.m128_private[0])) ; i++) { + r_.m128_private[i].neon_f32 = vabsq_f32(v2_.m128_private[i].neon_f32); + } + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128_private) / sizeof(r_.m128_private[0])) ; i++) { + r_.m128_private[i].altivec_f32 = vec_abs(v2_.m128_private[i].altivec_f32); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0; i < (sizeof(r_.f32) / sizeof(r_.f32[0])); i++) { + r_.f32[i] = (v2_.f32[i] < INT64_C(0)) ? -v2_.f32[i] : v2_.f32[i]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_abs_ps + #define _mm512_abs_ps(v2) simde_mm512_abs_ps(v2) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_abs_ps(simde__m512 src, simde__mmask16 k, simde__m512 v2) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) + return _mm512_mask_abs_ps(src, k, v2); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_abs_ps(v2)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_abs_ps + #define _mm512_mask_abs_ps(src, k, v2) simde_mm512_mask_abs_ps(src, k, v2) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_abs_pd(simde__m512d v2) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_87467) + return _mm512_abs_pd(v2); + #elif defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) + /* gcc bug: https://gcc.gnu.org/legacy-ml/gcc-patches/2018-01/msg01962.html */ + return _mm512_abs_pd(_mm512_castpd_ps(v2)); + #else + simde__m512d_private + r_, + v2_ = simde__m512d_to_private(v2); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128d_private) / sizeof(r_.m128d_private[0])) ; i++) { + r_.m128d_private[i].neon_f64 = vabsq_f64(v2_.m128d_private[i].neon_f64); + } + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128d_private) / sizeof(r_.m128d_private[0])) ; i++) { + r_.m128d_private[i].altivec_f64 = vec_abs(v2_.m128d_private[i].altivec_f64); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0; i < (sizeof(r_.f64) / sizeof(r_.f64[0])); i++) { + r_.f64[i] = (v2_.f64[i] < INT64_C(0)) ? -v2_.f64[i] : v2_.f64[i]; + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_abs_pd + #define _mm512_abs_pd(v2) simde_mm512_abs_pd(v2) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_abs_pd(simde__m512d src, simde__mmask8 k, simde__m512d v2) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_87467) + return _mm512_mask_abs_pd(src, k, v2); + #elif defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) + /* gcc bug: https://gcc.gnu.org/legacy-ml/gcc-patches/2018-01/msg01962.html */ + return _mm512_mask_abs_pd(src, k, _mm512_castpd_ps(v2)); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_abs_pd(v2)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_abs_pd + #define _mm512_mask_abs_pd(src, k, v2) simde_mm512_mask_abs_pd(src, k, v2) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_ABS_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/add.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/add.h new file mode 100644 index 00000000..d192b2f5 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/add.h @@ -0,0 +1,625 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_ADD_H) +#define SIMDE_X86_AVX512_ADD_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_add_epi8(simde__m128i src, simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_add_epi8(src, k, a, b); + #else + return simde_mm_mask_mov_epi8(src, k, simde_mm_add_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_add_epi8 + #define _mm_mask_add_epi8(src, k, a, b) simde_mm_mask_add_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_add_epi8(simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_maskz_add_epi8(k, a, b); + #else + return simde_mm_maskz_mov_epi8(k, simde_mm_add_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_add_epi8 + #define _mm_maskz_add_epi8(k, a, b) simde_mm_maskz_add_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_add_epi16(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_add_epi16(src, k, a, b); + #else + return simde_mm_mask_mov_epi16(src, k, simde_mm_add_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_add_epi16 + #define _mm_mask_add_epi16(src, k, a, b) simde_mm_mask_add_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_add_epi16(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_maskz_add_epi16(k, a, b); + #else + return simde_mm_maskz_mov_epi16(k, simde_mm_add_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_add_epi16 + #define _mm_maskz_add_epi16(k, a, b) simde_mm_maskz_add_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_add_epi32(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_add_epi32(src, k, a, b); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_add_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_add_epi32 + #define _mm_mask_add_epi32(src, k, a, b) simde_mm_mask_add_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_add_epi32(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_add_epi32(k, a, b); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_add_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_add_epi32 + #define _mm_maskz_add_epi32(k, a, b) simde_mm_maskz_add_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_add_epi64(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_add_epi64(src, k, a, b); + #else + return simde_mm_mask_mov_epi64(src, k, simde_mm_add_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_add_epi64 + #define _mm_mask_add_epi64(src, k, a, b) simde_mm_mask_add_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_add_epi64(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_add_epi64(k, a, b); + #else + return simde_mm_maskz_mov_epi64(k, simde_mm_add_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_add_epi64 + #define _mm_maskz_add_epi64(k, a, b) simde_mm_maskz_add_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mask_add_ss(simde__m128 src, simde__mmask8 k, simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(8,1,0)) + return _mm_mask_add_ss(src, k, a, b); + #elif 1 + simde__m128_private + src_ = simde__m128_to_private(src), + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b), + r_ = simde__m128_to_private(a); + + r_.f32[0] = (k & 1) ? (a_.f32[0] + b_.f32[0]) : src_.f32[0]; + + return simde__m128_from_private(r_); + #else + return simde_mm_move_ss(a, simde_mm_mask_mov_ps(src, k, simde_mm_add_ps(a, b))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_add_ss + #define _mm_mask_add_ss(src, k, a, b) simde_mm_mask_add_ss(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_maskz_add_ss(simde__mmask8 k, simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(8,1,0)) + return _mm_maskz_add_ss(k, a, b); + #elif 1 + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b), + r_ = simde__m128_to_private(a); + + r_.f32[0] = (k & 1) ? (a_.f32[0] + b_.f32[0]) : 0.0f; + + return simde__m128_from_private(r_); + #else + return simde_mm_move_ss(a, simde_mm_maskz_mov_ps(k, simde_mm_add_ps(a, b))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_add_ss + #define _mm_maskz_add_ss(k, a, b) simde_mm_maskz_add_ss(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_add_epi16(simde__m256i src, simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_add_epi16(src, k, a, b); + #else + return simde_mm256_mask_mov_epi16(src, k, simde_mm256_add_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_add_epi16 + #define _mm256_mask_add_epi16(src, k, a, b) simde_mm256_mask_add_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_add_epi16(simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_maskz_add_epi16(k, a, b); + #else + return simde_mm256_maskz_mov_epi16(k, simde_mm256_add_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_add_epi16 + #define _mm256_maskz_add_epi16(k, a, b) simde_mm256_maskz_add_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_add_epi32(simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_add_epi32(src, k, a, b); + #else + return simde_mm256_mask_mov_epi32(src, k, simde_mm256_add_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_add_epi32 + #define _mm256_mask_add_epi32(src, k, a, b) simde_mm256_mask_add_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_add_epi32(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_add_epi32(k, a, b); + #else + return simde_mm256_maskz_mov_epi32(k, simde_mm256_add_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_add_epi32 + #define _mm256_maskz_add_epi32(k, a, b) simde_mm256_maskz_add_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_add_epi64(simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_add_epi64(src, k, a, b); + #else + return simde_mm256_mask_mov_epi64(src, k, simde_mm256_add_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_add_epi64 + #define _mm256_mask_add_epi64(src, k, a, b) simde_mm256_mask_add_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_add_epi64(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_add_epi64(k, a, b); + #else + return simde_mm256_maskz_mov_epi64(k, simde_mm256_add_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_add_epi64 + #define _mm256_maskz_add_epi64(k, a, b) simde_mm256_maskz_add_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_add_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_add_epi8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = a_.i8 + b_.i8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_add_epi8(a_.m256i[i], b_.m256i[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_add_epi8 + #define _mm512_add_epi8(a, b) simde_mm512_add_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_add_epi8 (simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_add_epi8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_add_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_add_epi8 + #define _mm512_mask_add_epi8(src, k, a, b) simde_mm512_mask_add_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_add_epi8 (simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_add_epi8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_add_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_add_epi8 + #define _mm512_maskz_add_epi8(k, a, b) simde_mm512_maskz_add_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_add_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_add_epi16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = a_.i16 + b_.i16; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_add_epi16(a_.m256i[i], b_.m256i[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_add_epi16 + #define _mm512_add_epi16(a, b) simde_mm512_add_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_add_epi16 (simde__m512i src, simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_add_epi16(src, k, a, b); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_add_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_add_epi16 + #define _mm512_mask_add_epi16(src, k, a, b) simde_mm512_mask_add_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_add_epi16 (simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_add_epi16(k, a, b); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_add_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_add_epi16 + #define _mm512_maskz_add_epi16(k, a, b) simde_mm512_maskz_add_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_add_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_add_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 + b_.i32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_add_epi32(a_.m256i[i], b_.m256i[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_add_epi32 + #define _mm512_add_epi32(a, b) simde_mm512_add_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_add_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_add_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_add_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_add_epi32 + #define _mm512_mask_add_epi32(src, k, a, b) simde_mm512_mask_add_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_add_epi32(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_add_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_add_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_add_epi32 + #define _mm512_maskz_add_epi32(k, a, b) simde_mm512_maskz_add_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_add_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_add_epi64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_add_epi64(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_CLANG_BAD_VI64_OPS) + r_.i64 = a_.i64 + b_.i64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_add_epi64(a_.m256i[i], b_.m256i[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_add_epi64 + #define _mm512_add_epi64(a, b) simde_mm512_add_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_add_epi64(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_add_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_add_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_add_epi64 + #define _mm512_mask_add_epi64(src, k, a, b) simde_mm512_mask_add_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_add_epi64(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_add_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_add_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_add_epi64 + #define _mm512_maskz_add_epi64(k, a, b) simde_mm512_maskz_add_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_add_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_add_ps(a, b); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = a_.f32 + b_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_add_ps(a_.m256[i], b_.m256[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_add_ps + #define _mm512_add_ps(a, b) simde_mm512_add_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_add_ps(simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_add_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_add_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_add_ps + #define _mm512_mask_add_ps(src, k, a, b) simde_mm512_mask_add_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_add_ps(simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_add_ps(k, a, b); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_add_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_add_ps + #define _mm512_maskz_add_ps(k, a, b) simde_mm512_maskz_add_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_add_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_add_pd(a, b); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = a_.f64 + b_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_add_pd(a_.m256d[i], b_.m256d[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_add_pd + #define _mm512_add_pd(a, b) simde_mm512_add_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_add_pd(simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_add_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_add_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_add_pd + #define _mm512_mask_add_pd(src, k, a, b) simde_mm512_mask_add_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_add_pd(simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_add_pd(k, a, b); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_add_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_add_pd + #define _mm512_maskz_add_pd(k, a, b) simde_mm512_maskz_add_pd(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_ADD_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/adds.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/adds.h new file mode 100644 index 00000000..64abffaa --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/adds.h @@ -0,0 +1,529 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + */ + +#if !defined(SIMDE_X86_AVX512_ADDS_H) +#define SIMDE_X86_AVX512_ADDS_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_adds_epi8(simde__m128i src, simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_adds_epi8(src, k, a, b); + #else + return simde_mm_mask_mov_epi8(src, k, simde_mm_adds_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_adds_epi8 + #define _mm_mask_adds_epi8(src, k, a, b) simde_mm_mask_adds_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_adds_epi8(simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_maskz_adds_epi8(k, a, b); + #else + return simde_mm_maskz_mov_epi8(k, simde_mm_adds_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_adds_epi8 + #define _mm_maskz_adds_epi8(k, a, b) simde_mm_maskz_adds_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_adds_epi16(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_adds_epi16(src, k, a, b); + #else + return simde_mm_mask_mov_epi16(src, k, simde_mm_adds_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_adds_epi16 + #define _mm_mask_adds_epi16(src, k, a, b) simde_mm_mask_adds_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_adds_epi16(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_maskz_adds_epi16(k, a, b); + #else + return simde_mm_maskz_mov_epi16(k, simde_mm_adds_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_adds_epi16 + #define _mm_maskz_adds_epi16(k, a, b) simde_mm_maskz_adds_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_adds_epi8(simde__m256i src, simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_adds_epi8(src, k, a, b); + #else + return simde_mm256_mask_mov_epi8(src, k, simde_mm256_adds_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_adds_epi8 + #define _mm256_mask_adds_epi8(src, k, a, b) simde_mm256_mask_adds_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_adds_epi8(simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_maskz_adds_epi8(k, a, b); + #else + return simde_mm256_maskz_mov_epi8(k, simde_mm256_adds_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_adds_epi8 + #define _mm256_maskz_adds_epi8(k, a, b) simde_mm256_maskz_adds_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_adds_epi16(simde__m256i src, simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_adds_epi16(src, k, a, b); + #else + return simde_mm256_mask_mov_epi16(src, k, simde_mm256_adds_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_adds_epi16 + #define _mm256_mask_adds_epi16(src, k, a, b) simde_mm256_mask_adds_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_adds_epi16(simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_maskz_adds_epi16(k, a, b); + #else + return simde_mm256_maskz_mov_epi16(k, simde_mm256_adds_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_adds_epi16 + #define _mm256_maskz_adds_epi16(k, a, b) simde_mm256_maskz_adds_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_adds_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_adds_epi8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if !defined(HEDLEY_INTEL_VERSION) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_adds_epi8(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = simde_math_adds_i8(a_.i8[i], b_.i8[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_adds_epi8 + #define _mm512_adds_epi8(a, b) simde_mm512_adds_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_adds_epi8 (simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_adds_epi8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_adds_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_adds_epi8 + #define _mm512_mask_adds_epi8(src, k, a, b) simde_mm512_mask_adds_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_adds_epi8 (simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_adds_epi8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_adds_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_adds_epi8 + #define _mm512_maskz_adds_epi8(k, a, b) simde_mm512_maskz_adds_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_adds_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_adds_epi16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if !defined(HEDLEY_INTEL_VERSION) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_adds_epi16(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = simde_math_adds_i16(a_.i16[i], b_.i16[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_adds_epi16 + #define _mm512_adds_epi16(a, b) simde_mm512_adds_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_adds_epi16 (simde__m512i src, simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_adds_epi16(src, k, a, b); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_adds_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_adds_epi16 + #define _mm512_mask_adds_epi16(src, k, a, b) simde_mm512_mask_adds_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_adds_epi16 (simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_adds_epi16(k, a, b); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_adds_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_adds_epi16 + #define _mm512_maskz_adds_epi16(k, a, b) simde_mm512_maskz_adds_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_adds_epu8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_adds_epu8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if !defined(HEDLEY_INTEL_VERSION) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_adds_epu8(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = simde_math_adds_u8(a_.u8[i], b_.u8[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_adds_epu8 + #define _mm512_adds_epu8(a, b) simde_mm512_adds_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_adds_epu8 (simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_adds_epu8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_adds_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_adds_epu8 + #define _mm512_mask_adds_epu8(src, k, a, b) simde_mm512_mask_adds_epu8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_adds_epu8 (simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_adds_epu8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_adds_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_adds_epu8 + #define _mm512_maskz_adds_epu8(k, a, b) simde_mm512_maskz_adds_epu8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_adds_epu16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_adds_epu16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if !defined(HEDLEY_INTEL_VERSION) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_adds_epu16(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = simde_math_adds_u16(a_.u16[i], b_.u16[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_adds_epu16 + #define _mm512_adds_epu16(a, b) simde_mm512_adds_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_adds_epu16 (simde__m512i src, simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_adds_epu16(src, k, a, b); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_adds_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_adds_epu16 + #define _mm512_mask_adds_epu16(src, k, a, b) simde_mm512_mask_adds_epu16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_adds_epu16 (simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_adds_epu16(k, a, b); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_adds_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_adds_epu16 + #define _mm512_maskz_adds_epu16(k, a, b) simde_mm512_maskz_adds_epu16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_adds_epi32(simde__m128i a, simde__m128i b) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vqaddq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_POWER_ALTIVEC_P6) + r_.altivec_i32 = vec_adds(a_.altivec_i32, b_.altivec_i32); + #else + #if defined(SIMDE_X86_SSE2_NATIVE) + /* https://stackoverflow.com/a/56544654/501126 */ + const __m128i int_max = _mm_set1_epi32(INT32_MAX); + + /* normal result (possibly wraps around) */ + const __m128i sum = _mm_add_epi32(a_.n, b_.n); + + /* If result saturates, it has the same sign as both a and b */ + const __m128i sign_bit = _mm_srli_epi32(a_.n, 31); /* shift sign to lowest bit */ + + #if defined(SIMDE_X86_AVX512VL_NATIVE) + const __m128i overflow = _mm_ternarylogic_epi32(a_.n, b_.n, sum, 0x42); + #else + const __m128i sign_xor = _mm_xor_si128(a_.n, b_.n); + const __m128i overflow = _mm_andnot_si128(sign_xor, _mm_xor_si128(a_.n, sum)); + #endif + + #if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + r_.n = _mm_mask_add_epi32(sum, _mm_movepi32_mask(overflow), int_max, sign_bit); + #else + const __m128i saturated = _mm_add_epi32(int_max, sign_bit); + + #if defined(SIMDE_X86_SSE4_1_NATIVE) + r_.n = + _mm_castps_si128( + _mm_blendv_ps( + _mm_castsi128_ps(sum), + _mm_castsi128_ps(saturated), + _mm_castsi128_ps(overflow) + ) + ); + #else + const __m128i overflow_mask = _mm_srai_epi32(overflow, 31); + r_.n = + _mm_or_si128( + _mm_and_si128(overflow_mask, saturated), + _mm_andnot_si128(overflow_mask, sum) + ); + #endif + #endif + #elif defined(SIMDE_VECTOR_SCALAR) + uint32_t au SIMDE_VECTOR(16) = HEDLEY_REINTERPRET_CAST(__typeof__(au), a_.i32); + uint32_t bu SIMDE_VECTOR(16) = HEDLEY_REINTERPRET_CAST(__typeof__(bu), b_.i32); + uint32_t ru SIMDE_VECTOR(16) = au + bu; + + au = (au >> 31) + INT32_MAX; + + uint32_t m SIMDE_VECTOR(16) = HEDLEY_REINTERPRET_CAST(__typeof__(m), HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (au ^ bu) | ~(bu ^ ru)) < 0); + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (au & ~m) | (ru & m)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = simde_math_adds_i32(a_.i32[i], b_.i32[i]); + } + #endif + #endif + + return simde__m128i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_adds_epi32(simde__m256i a, simde__m256i b) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_adds_epi32(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SCALAR) + uint32_t au SIMDE_VECTOR(32) = HEDLEY_REINTERPRET_CAST(__typeof__(au), a_.i32); + uint32_t bu SIMDE_VECTOR(32) = HEDLEY_REINTERPRET_CAST(__typeof__(bu), b_.i32); + uint32_t ru SIMDE_VECTOR(32) = au + bu; + + au = (au >> 31) + INT32_MAX; + + uint32_t m SIMDE_VECTOR(32) = HEDLEY_REINTERPRET_CAST(__typeof__(m), HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (au ^ bu) | ~(bu ^ ru)) < 0); + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (au & ~m) | (ru & m)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = simde_math_adds_i32(a_.i32[i], b_.i32[i]); + } + #endif + + return simde__m256i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_adds_epi32(simde__m512i a, simde__m512i b) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_adds_epi32(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_adds_epi32(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SCALAR) + uint32_t au SIMDE_VECTOR(64) = HEDLEY_REINTERPRET_CAST(__typeof__(au), a_.i32); + uint32_t bu SIMDE_VECTOR(64) = HEDLEY_REINTERPRET_CAST(__typeof__(bu), b_.i32); + uint32_t ru SIMDE_VECTOR(64) = au + bu; + + au = (au >> 31) + INT32_MAX; + + uint32_t m SIMDE_VECTOR(64) = HEDLEY_REINTERPRET_CAST(__typeof__(m), HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (au ^ bu) | ~(bu ^ ru)) < 0); + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (au & ~m) | (ru & m)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = simde_math_adds_i32(a_.i32[i], b_.i32[i]); + } + #endif + + return simde__m512i_from_private(r_); +} + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_ADDS_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/and.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/and.h new file mode 100644 index 00000000..fd7118f8 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/and.h @@ -0,0 +1,305 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_AND_H) +#define SIMDE_X86_AVX512_AND_H + +#include "types.h" +#include "../avx2.h" + +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_and_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_and_pd(a, b); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + #if defined(SIMDE_X86_AVX_NATIVE) + r_.m256d[0] = simde_mm256_and_pd(a_.m256d[0], b_.m256d[0]); + r_.m256d[1] = simde_mm256_and_pd(a_.m256d[1], b_.m256d[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f & b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] & b_.i32f[i]; + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_and_pd + #define _mm512_and_pd(a, b) simde_mm512_and_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_and_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_and_ps(a, b); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + #if defined(SIMDE_X86_AVX_NATIVE) + r_.m256[0] = simde_mm256_and_ps(a_.m256[0], b_.m256[0]); + r_.m256[1] = simde_mm256_and_ps(a_.m256[1], b_.m256[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f & b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] & b_.i32f[i]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_and_ps + #define _mm512_and_ps(a, b) simde_mm512_and_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_and_ps(simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_mask_and_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_and_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_and_ps + #define _mm512_mask_and_ps(src, k, a, b) simde_mm512_mask_and_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_and_ps(simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_maskz_and_ps(k, a, b); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_and_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_and_ps + #define _mm512_maskz_and_ps(k, a, b) simde_mm512_maskz_and_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_and_pd(simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_mask_and_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_and_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_and_pd + #define _mm512_mask_and_pd(src, k, a, b) simde_mm512_mask_and_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_and_pd(simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_maskz_and_pd(k, a, b); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_and_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_and_pd + #define _mm512_maskz_and_pd(k, a, b) simde_mm512_maskz_and_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_and_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_and_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 & b_.i32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] & b_.i32[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_and_epi32 + #define _mm512_and_epi32(a, b) simde_mm512_and_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_and_epi32(simde__m512i src, simde__mmask16 k, simde__m512i v2, simde__m512i v3) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_and_epi32(src, k, v2, v3); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_and_epi32(v2, v3)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_and_epi32 + #define _mm512_mask_and_epi32(src, k, v2, v3) simde_mm512_mask_and_epi32(src, k, v2, v3) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_and_epi32(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_and_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_and_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_and_epi32 + #define _mm512_maskz_and_epi32(k, a, b) simde_mm512_maskz_and_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_and_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_and_epi64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 & b_.i64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] & b_.i64[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_and_epi64 + #define _mm512_and_epi64(a, b) simde_mm512_and_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_and_epi64(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_and_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_and_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_and_epi64 + #define _mm512_mask_and_epi64(src, k, a, b) simde_mm512_mask_and_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_and_epi64(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_and_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_and_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_and_epi64 + #define _mm512_maskz_and_epi64(k, a, b) simde_mm512_maskz_and_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_and_si512 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_and_si512(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256i[0] = simde_mm256_and_si256(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_and_si256(a_.m256i[1], b_.m256i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f & b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] & b_.i32[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_and_si512 + #define _mm512_and_si512(a, b) simde_mm512_and_si512(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_AND_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/andnot.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/andnot.h new file mode 100644 index 00000000..ddc3dcba --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/andnot.h @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + */ + +#if !defined(SIMDE_X86_AVX512_ANDNOT_H) +#define SIMDE_X86_AVX512_ANDNOT_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_andnot_ps(a, b) _mm512_andnot_ps(a, b) +#else + #define simde_mm512_andnot_ps(a, b) simde_mm512_castsi512_ps(simde_mm512_andnot_si512(simde_mm512_castps_si512(a), simde_mm512_castps_si512(b))) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_andnot_ps + #define _mm512_andnot_ps(a, b) simde_mm512_andnot_ps(a, b) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_mask_andnot_ps(src, k, a, b) _mm512_mask_andnot_ps((src), (k), (a), (b)) +#else + #define simde_mm512_mask_andnot_ps(src, k, a, b) simde_mm512_castsi512_ps(simde_mm512_mask_andnot_epi32(simde_mm512_castps_si512(src), k, simde_mm512_castps_si512(a), simde_mm512_castps_si512(b))) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_andnot_ps + #define _mm512_mask_andnot_ps(src, k, a, b) simde_mm512_mask_andnot_ps(src, k, a, b) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_maskz_andnot_ps(k, a, b) _mm512_maskz_andnot_ps((k), (a), (b)) +#else + #define simde_mm512_maskz_andnot_ps(k, a, b) simde_mm512_castsi512_ps(simde_mm512_maskz_andnot_epi32(k, simde_mm512_castps_si512(a), simde_mm512_castps_si512(b))) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_andnot_ps + #define _mm512_maskz_andnot_ps(k, a, b) simde_mm512_maskz_andnot_ps(k, a, b) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_andnot_pd(a, b) _mm512_andnot_pd(a, b) +#else + #define simde_mm512_andnot_pd(a, b) simde_mm512_castsi512_pd(simde_mm512_andnot_si512(simde_mm512_castpd_si512(a), simde_mm512_castpd_si512(b))) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_andnot_pd + #define _mm512_andnot_pd(a, b) simde_mm512_andnot_pd(a, b) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_mask_andnot_pd(src, k, a, b) _mm512_mask_andnot_pd((src), (k), (a), (b)) +#else + #define simde_mm512_mask_andnot_pd(src, k, a, b) simde_mm512_castsi512_pd(simde_mm512_mask_andnot_epi64(simde_mm512_castpd_si512(src), k, simde_mm512_castpd_si512(a), simde_mm512_castpd_si512(b))) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_andnot_pd + #define _mm512_mask_andnot_pd(src, k, a, b) simde_mm512_mask_andnot_pd(src, k, a, b) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_maskz_andnot_pd(k, a, b) _mm512_maskz_andnot_pd((k), (a), (b)) +#else + #define simde_mm512_maskz_andnot_pd(k, a, b) simde_mm512_castsi512_pd(simde_mm512_maskz_andnot_epi64(k, simde_mm512_castpd_si512(a), simde_mm512_castpd_si512(b))) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_andnot_pd + #define _mm512_maskz_andnot_pd(k, a, b) simde_mm512_maskz_andnot_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_andnot_si512 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_andnot_si512(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256i[0] = simde_mm256_andnot_si256(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_andnot_si256(a_.m256i[1], b_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = ~(a_.i32f[i]) & b_.i32f[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#define simde_mm512_andnot_epi32(a, b) simde_mm512_andnot_si512(a, b) +#define simde_mm512_andnot_epi64(a, b) simde_mm512_andnot_si512(a, b) +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_andnot_si512 + #define _mm512_andnot_si512(a, b) simde_mm512_andnot_si512(a, b) + #undef _mm512_andnot_epi32 + #define _mm512_andnot_epi32(a, b) simde_mm512_andnot_si512(a, b) + #undef _mm512_andnot_epi64 + #define _mm512_andnot_epi64(a, b) simde_mm512_andnot_si512(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_andnot_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_andnot_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_andnot_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_andnot_epi32 + #define _mm512_mask_andnot_epi32(src, k, a, b) simde_mm512_mask_andnot_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_andnot_epi32(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_andnot_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_andnot_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_andnot_epi32 + #define _mm512_maskz_andnot_epi32(k, a, b) simde_mm512_maskz_andnot_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_andnot_epi64(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_andnot_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_andnot_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_andnot_epi64 + #define _mm512_mask_andnot_epi64(src, k, a, b) simde_mm512_mask_andnot_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_andnot_epi64(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_andnot_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_andnot_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_andnot_epi64 + #define _mm512_maskz_andnot_epi64(k, a, b) simde_mm512_maskz_andnot_epi64(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_ANDNOT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/avg.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/avg.h new file mode 100644 index 00000000..2ec34412 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/avg.h @@ -0,0 +1,258 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_AVG_H) +#define SIMDE_X86_AVX512_AVG_H + +#include "types.h" +#include "mov.h" +#include "../avx2.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_avg_epu8(simde__m128i src, simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_avg_epu8(src, k, a, b); + #else + return simde_mm_mask_mov_epi8(src, k, simde_mm_avg_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_avg_epu8 + #define _mm_mask_avg_epu8(src, k, a, b) simde_mm_mask_avg_epu8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_avg_epu8(simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_maskz_avg_epu8(k, a, b); + #else + return simde_mm_maskz_mov_epi8(k, simde_mm_avg_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_avg_epu8 + #define _mm_maskz_avg_epu8(k, a, b) simde_mm_maskz_avg_epu8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_avg_epu16(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_avg_epu16(src, k, a, b); + #else + return simde_mm_mask_mov_epi16(src, k, simde_mm_avg_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_avg_epu16 + #define _mm_mask_avg_epu16(src, k, a, b) simde_mm_mask_avg_epu16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_avg_epu16(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_maskz_avg_epu16(k, a, b); + #else + return simde_mm_maskz_mov_epi16(k, simde_mm_avg_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_avg_epu16 + #define _mm_maskz_avg_epu16(k, a, b) simde_mm_maskz_avg_epu16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_avg_epu8(simde__m256i src, simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_avg_epu8(src, k, a, b); + #else + return simde_mm256_mask_mov_epi8(src, k, simde_mm256_avg_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_avg_epu8 + #define _mm256_mask_avg_epu8(src, k, a, b) simde_mm256_mask_avg_epu8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_avg_epu8(simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_maskz_avg_epu8(k, a, b); + #else + return simde_mm256_maskz_mov_epi8(k, simde_mm256_avg_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_avg_epu8 + #define _mm256_maskz_avg_epu8(k, a, b) simde_mm256_maskz_avg_epu8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_avg_epu16(simde__m256i src, simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_avg_epu16(src, k, a, b); + #else + return simde_mm256_mask_mov_epi16(src, k, simde_mm256_avg_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_avg_epu16 + #define _mm256_mask_avg_epu16(src, k, a, b) simde_mm256_mask_avg_epu16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_avg_epu16(simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_maskz_avg_epu16(k, a, b); + #else + return simde_mm256_maskz_mov_epi16(k, simde_mm256_avg_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_avg_epu16 + #define _mm256_maskz_avg_epu16(k, a, b) simde_mm256_maskz_avg_epu16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_avg_epu8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_avg_epu8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] + b_.u8[i] + 1) >> 1; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_avg_epu8 + #define _mm512_avg_epu8(a, b) simde_mm512_avg_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_avg_epu8 (simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_avg_epu8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_avg_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_avg_epu8 + #define _mm512_mask_avg_epu8(src, k, a, b) simde_mm512_mask_avg_epu8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_avg_epu8 (simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_avg_epu8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_avg_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_avg_epu8 + #define _mm512_maskz_avg_epu8(k, a, b) simde_mm512_maskz_avg_epu8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_avg_epu16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_avg_epu16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] + b_.u16[i] + 1) >> 1; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_avg_epu16 + #define _mm512_avg_epu16(a, b) simde_mm512_avg_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_avg_epu16 (simde__m512i src, simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_avg_epu16(src, k, a, b); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_avg_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_avg_epu16 + #define _mm512_mask_avg_epu16(src, k, a, b) simde_mm512_mask_avg_epu16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_avg_epu16 (simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_avg_epu16(k, a, b); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_avg_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_avg_epu16 + #define _mm512_maskz_avg_epu16(k, a, b) simde_mm512_maskz_avg_epu16(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_AVG_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/bitshuffle.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/bitshuffle.h new file mode 100644 index 00000000..05f4b5c8 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/bitshuffle.h @@ -0,0 +1,202 @@ +#if !defined(SIMDE_X86_AVX512_BITSHUFFLE_H) +#define SIMDE_X86_AVX512_BITSHUFFLE_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_bitshuffle_epi64_mask (simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_bitshuffle_epi64_mask(b, c); + #else + simde__m128i_private + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + simde__mmask16 r = 0; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + __typeof__(b_.u64) rv = { 0, 0 }; + __typeof__(b_.u64) lshift = { 0, 8 }; + + for (int8_t i = 0 ; i < 8 ; i++) { + __typeof__(b_.u64) ct = (HEDLEY_REINTERPRET_CAST(__typeof__(ct), c_.u8) >> (i * 8)) & 63; + rv |= ((b_.u64 >> ct) & 1) << lshift; + lshift += 1; + } + + r = + HEDLEY_STATIC_CAST(simde__mmask16, rv[0]) | + HEDLEY_STATIC_CAST(simde__mmask16, rv[1]); + #else + for (size_t i = 0 ; i < (sizeof(c_.m64_private) / sizeof(c_.m64_private[0])) ; i++) { + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t j = 0 ; j < (sizeof(c_.m64_private[i].u8) / sizeof(c_.m64_private[i].u8[0])) ; j++) { + r |= (((b_.u64[i] >> (c_.m64_private[i].u8[j]) & 63) & 1) << ((i * 8) + j)); + } + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_bitshuffle_epi64_mask + #define _mm_bitshuffle_epi64_mask(b, c) simde_mm_bitshuffle_epi64_mask(b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_mask_bitshuffle_epi64_mask (simde__mmask16 k, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_bitshuffle_epi64_mask(k, b, c); + #else + return (k & simde_mm_bitshuffle_epi64_mask(b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_bitshuffle_epi64_mask + #define _mm_mask_bitshuffle_epi64_mask(k, b, c) simde_mm_mask_bitshuffle_epi64_mask(k, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_bitshuffle_epi64_mask (simde__m256i b, simde__m256i c) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_bitshuffle_epi64_mask(b, c); + #else + simde__m256i_private + b_ = simde__m256i_to_private(b), + c_ = simde__m256i_to_private(c); + simde__mmask32 r = 0; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < sizeof(b_.m128i) / sizeof(b_.m128i[0]) ; i++) { + r |= (HEDLEY_STATIC_CAST(simde__mmask32, simde_mm_bitshuffle_epi64_mask(b_.m128i[i], c_.m128i[i])) << (i * 16)); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + __typeof__(b_.u64) rv = { 0, 0, 0, 0 }; + __typeof__(b_.u64) lshift = { 0, 8, 16, 24 }; + + for (int8_t i = 0 ; i < 8 ; i++) { + __typeof__(b_.u64) ct = (HEDLEY_REINTERPRET_CAST(__typeof__(ct), c_.u8) >> (i * 8)) & 63; + rv |= ((b_.u64 >> ct) & 1) << lshift; + lshift += 1; + } + + r = + HEDLEY_STATIC_CAST(simde__mmask32, rv[0]) | + HEDLEY_STATIC_CAST(simde__mmask32, rv[1]) | + HEDLEY_STATIC_CAST(simde__mmask32, rv[2]) | + HEDLEY_STATIC_CAST(simde__mmask32, rv[3]); + #else + for (size_t i = 0 ; i < (sizeof(c_.m128i_private) / sizeof(c_.m128i_private[0])) ; i++) { + for (size_t j = 0 ; j < (sizeof(c_.m128i_private[i].m64_private) / sizeof(c_.m128i_private[i].m64_private[0])) ; j++) { + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t k = 0 ; k < (sizeof(c_.m128i_private[i].m64_private[j].u8) / sizeof(c_.m128i_private[i].m64_private[j].u8[0])) ; k++) { + r |= (((b_.m128i_private[i].u64[j] >> (c_.m128i_private[i].m64_private[j].u8[k]) & 63) & 1) << ((i * 16) + (j * 8) + k)); + } + } + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_bitshuffle_epi64_mask + #define _mm256_bitshuffle_epi64_mask(b, c) simde_mm256_bitshuffle_epi64_mask(b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_mask_bitshuffle_epi64_mask (simde__mmask32 k, simde__m256i b, simde__m256i c) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_bitshuffle_epi64_mask(k, b, c); + #else + return (k & simde_mm256_bitshuffle_epi64_mask(b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_bitshuffle_epi64_mask + #define _mm256_mask_bitshuffle_epi64_mask(k, b, c) simde_mm256_mask_bitshuffle_epi64_mask(k, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_bitshuffle_epi64_mask (simde__m512i b, simde__m512i c) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) + return _mm512_bitshuffle_epi64_mask(b, c); + #else + simde__m512i_private + b_ = simde__m512i_to_private(b), + c_ = simde__m512i_to_private(c); + simde__mmask64 r = 0; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(b_.m128i) / sizeof(b_.m128i[0])) ; i++) { + r |= (HEDLEY_STATIC_CAST(simde__mmask64, simde_mm_bitshuffle_epi64_mask(b_.m128i[i], c_.m128i[i])) << (i * 16)); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(b_.m256i) / sizeof(b_.m256i[0])) ; i++) { + r |= (HEDLEY_STATIC_CAST(simde__mmask64, simde_mm256_bitshuffle_epi64_mask(b_.m256i[i], c_.m256i[i])) << (i * 32)); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + __typeof__(b_.u64) rv = { 0, 0, 0, 0, 0, 0, 0, 0 }; + __typeof__(b_.u64) lshift = { 0, 8, 16, 24, 32, 40, 48, 56 }; + + for (int8_t i = 0 ; i < 8 ; i++) { + __typeof__(b_.u64) ct = (HEDLEY_REINTERPRET_CAST(__typeof__(ct), c_.u8) >> (i * 8)) & 63; + rv |= ((b_.u64 >> ct) & 1) << lshift; + lshift += 1; + } + + r = + HEDLEY_STATIC_CAST(simde__mmask64, rv[0]) | + HEDLEY_STATIC_CAST(simde__mmask64, rv[1]) | + HEDLEY_STATIC_CAST(simde__mmask64, rv[2]) | + HEDLEY_STATIC_CAST(simde__mmask64, rv[3]) | + HEDLEY_STATIC_CAST(simde__mmask64, rv[4]) | + HEDLEY_STATIC_CAST(simde__mmask64, rv[5]) | + HEDLEY_STATIC_CAST(simde__mmask64, rv[6]) | + HEDLEY_STATIC_CAST(simde__mmask64, rv[7]); + #else + for (size_t i = 0 ; i < (sizeof(c_.m128i_private) / sizeof(c_.m128i_private[0])) ; i++) { + for (size_t j = 0 ; j < (sizeof(c_.m128i_private[i].m64_private) / sizeof(c_.m128i_private[i].m64_private[0])) ; j++) { + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t k = 0 ; k < (sizeof(c_.m128i_private[i].m64_private[j].u8) / sizeof(c_.m128i_private[i].m64_private[j].u8[0])) ; k++) { + r |= (((b_.m128i_private[i].u64[j] >> (c_.m128i_private[i].m64_private[j].u8[k]) & 63) & 1) << ((i * 16) + (j * 8) + k)); + } + } + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) + #undef _mm512_bitshuffle_epi64_mask + #define _mm512_bitshuffle_epi64_mask(b, c) simde_mm512_bitshuffle_epi64_mask(b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_mask_bitshuffle_epi64_mask (simde__mmask64 k, simde__m512i b, simde__m512i c) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) + return _mm512_mask_bitshuffle_epi64_mask(k, b, c); + #else + return (k & simde_mm512_bitshuffle_epi64_mask(b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_bitshuffle_epi64_mask + #define _mm512_mask_bitshuffle_epi64_mask(k, b, c) simde_mm512_mask_bitshuffle_epi64_mask(k, b, c) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_BITSHUFFLE_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/blend.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/blend.h new file mode 100644 index 00000000..e34dd20b --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/blend.h @@ -0,0 +1,293 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_BLEND_H) +#define SIMDE_X86_AVX512_BLEND_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_blend_epi8(simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_blend_epi8(k, a, b); + #else + return simde_mm_mask_mov_epi8(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_blend_epi8 + #define _mm_mask_blend_epi8(k, a, b) simde_mm_mask_blend_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_blend_epi16(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_blend_epi16(k, a, b); + #else + return simde_mm_mask_mov_epi16(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_blend_epi16 + #define _mm_mask_blend_epi16(k, a, b) simde_mm_mask_blend_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_blend_epi32(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_blend_epi32(k, a, b); + #else + return simde_mm_mask_mov_epi32(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_blend_epi32 + #define _mm_mask_blend_epi32(k, a, b) simde_mm_mask_blend_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_blend_epi64(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_blend_epi64(k, a, b); + #else + return simde_mm_mask_mov_epi64(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_blend_epi64 + #define _mm_mask_blend_epi64(k, a, b) simde_mm_mask_blend_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mask_blend_ps(simde__mmask8 k, simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_blend_ps(k, a, b); + #else + return simde_mm_mask_mov_ps(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_blend_ps + #define _mm_mask_blend_ps(k, a, b) simde_mm_mask_blend_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mask_blend_pd(simde__mmask8 k, simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_blend_pd(k, a, b); + #else + return simde_mm_mask_mov_pd(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_blend_pd + #define _mm_mask_blend_pd(k, a, b) simde_mm_mask_blend_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_blend_epi8(simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_blend_epi8(k, a, b); + #else + return simde_mm256_mask_mov_epi8(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_blend_epi8 + #define _mm256_mask_blend_epi8(k, a, b) simde_mm256_mask_blend_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_blend_epi16(simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_blend_epi16(k, a, b); + #else + return simde_mm256_mask_mov_epi16(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_blend_epi16 + #define _mm256_mask_blend_epi16(k, a, b) simde_mm256_mask_blend_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_blend_epi32(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_blend_epi32(k, a, b); + #else + return simde_mm256_mask_mov_epi32(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_blend_epi32 + #define _mm256_mask_blend_epi32(k, a, b) simde_mm256_mask_blend_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_blend_epi64(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_blend_epi64(k, a, b); + #else + return simde_mm256_mask_mov_epi64(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_blend_epi64 + #define _mm256_mask_blend_epi64(k, a, b) simde_mm256_mask_blend_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask_blend_ps(simde__mmask8 k, simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_blend_ps(k, a, b); + #else + return simde_mm256_mask_mov_ps(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_blend_ps + #define _mm256_mask_blend_ps(k, a, b) simde_mm256_mask_blend_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mask_blend_pd(simde__mmask8 k, simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_blend_pd(k, a, b); + #else + return simde_mm256_mask_mov_pd(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_blend_pd + #define _mm256_mask_blend_pd(k, a, b) simde_mm256_mask_blend_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_blend_epi8(simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_blend_epi8(k, a, b); + #else + return simde_mm512_mask_mov_epi8(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_blend_epi8 + #define _mm512_mask_blend_epi8(k, a, b) simde_mm512_mask_blend_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_blend_epi16(simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_blend_epi16(k, a, b); + #else + return simde_mm512_mask_mov_epi16(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_blend_epi16 + #define _mm512_mask_blend_epi16(k, a, b) simde_mm512_mask_blend_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_blend_epi32(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_blend_epi32(k, a, b); + #else + return simde_mm512_mask_mov_epi32(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_blend_epi32 + #define _mm512_mask_blend_epi32(k, a, b) simde_mm512_mask_blend_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_blend_epi64(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_blend_epi64(k, a, b); + #else + return simde_mm512_mask_mov_epi64(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_blend_epi64 + #define _mm512_mask_blend_epi64(k, a, b) simde_mm512_mask_blend_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_blend_ps(simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_blend_ps(k, a, b); + #else + return simde_mm512_mask_mov_ps(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_blend_ps + #define _mm512_mask_blend_ps(k, a, b) simde_mm512_mask_blend_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_blend_pd(simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_blend_pd(k, a, b); + #else + return simde_mm512_mask_mov_pd(a, k, b); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_blend_pd + #define _mm512_mask_blend_pd(k, a, b) simde_mm512_mask_blend_pd(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_BLEND_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/broadcast.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/broadcast.h new file mode 100644 index 00000000..33b41abd --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/broadcast.h @@ -0,0 +1,897 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + * 2020 Christopher Moore + */ + +#if !defined(SIMDE_X86_AVX512_BROADCAST_H) +#define SIMDE_X86_AVX512_BROADCAST_H + +#include "types.h" +#include "../avx2.h" + +#include "mov.h" +#include "cast.h" +#include "set1.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_broadcast_f32x2 (simde__m128 a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm256_broadcast_f32x2(a); + #else + simde__m256_private r_; + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) + r_.f32 = __builtin_shufflevector(a_.f32, a_.f32, 0, 1, 0, 1, 0, 1, 0, 1); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i += 2) { + r_.f32[ i ] = a_.f32[0]; + r_.f32[i + 1] = a_.f32[1]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm256_broadcast_f32x2 + #define _mm256_broadcast_f32x2(a) simde_mm256_broadcast_f32x2(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask_broadcast_f32x2(simde__m256 src, simde__mmask8 k, simde__m128 a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm256_mask_broadcast_f32x2(src, k, a); + #else + return simde_mm256_mask_mov_ps(src, k, simde_mm256_broadcast_f32x2(a)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_broadcast_f32x2 + #define _mm256_mask_broadcast_f32x2(src, k, a) simde_mm256_mask_broadcast_f32x2(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_maskz_broadcast_f32x2(simde__mmask8 k, simde__m128 a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm256_maskz_broadcast_f32x2(k, a); + #else + return simde_mm256_maskz_mov_ps(k, simde_mm256_broadcast_f32x2(a)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_broadcast_f32x2 + #define _mm256_maskz_broadcast_f32x2(k, a) simde_mm256_maskz_broadcast_f32x2(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_broadcast_f32x2 (simde__m128 a) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_broadcast_f32x2(a); + #else + simde__m512_private r_; + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) + r_.f32 = __builtin_shufflevector(a_.f32, a_.f32, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i+=2) { + r_.f32[ i ] = a_.f32[0]; + r_.f32[i + 1] = a_.f32[1]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_broadcast_f32x2 + #define _mm512_broadcast_f32x2(a) simde_mm512_broadcast_f32x2(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_broadcast_f32x2(simde__m512 src, simde__mmask16 k, simde__m128 a) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_mask_broadcast_f32x2(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_broadcast_f32x2(a)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_broadcast_f32x2 + #define _mm512_mask_broadcast_f32x2(src, k, a) simde_mm512_mask_broadcast_f32x2(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_broadcast_f32x2(simde__mmask16 k, simde__m128 a) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_maskz_broadcast_f32x2(k, a); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_broadcast_f32x2(a)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_broadcast_f32x2 + #define _mm512_maskz_broadcast_f32x2(k, a) simde_mm512_maskz_broadcast_f32x2(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_broadcast_f32x8 (simde__m256 a) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_broadcast_f32x8(a); + #else + simde__m512_private r_; + simde__m256_private a_ = simde__m256_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) + r_.f32 = __builtin_shufflevector(a_.f32, a_.f32, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i+=8) { + r_.f32[ i ] = a_.f32[0]; + r_.f32[i + 1] = a_.f32[1]; + r_.f32[i + 2] = a_.f32[2]; + r_.f32[i + 3] = a_.f32[3]; + r_.f32[i + 4] = a_.f32[4]; + r_.f32[i + 5] = a_.f32[5]; + r_.f32[i + 6] = a_.f32[6]; + r_.f32[i + 7] = a_.f32[7]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_broadcast_f32x8 + #define _mm512_broadcast_f32x8(a) simde_mm512_broadcast_f32x8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_broadcast_f32x8(simde__m512 src, simde__mmask16 k, simde__m256 a) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_mask_broadcast_f32x8(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_broadcast_f32x8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_broadcast_f32x8 + #define _mm512_mask_broadcast_f32x8(src, k, a) simde_mm512_mask_broadcast_f32x8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_broadcast_f32x8(simde__mmask16 k, simde__m256 a) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_maskz_broadcast_f32x8(k, a); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_broadcast_f32x8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_broadcast_f32x8 + #define _mm512_maskz_broadcast_f32x8(k, a) simde_mm512_maskz_broadcast_f32x8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_broadcast_f64x2 (simde__m128d a) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_broadcast_f64x2(a); + #else + simde__m512d_private r_; + simde__m128d_private a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) && !defined(SIMDE_BUG_CLANG_BAD_VI64_OPS) + r_.f64 = __builtin_shufflevector(a_.f64, a_.f64, 0, 1, 0, 1, 0, 1, 0, 1); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i += 2) { + r_.f64[ i ] = a_.f64[0]; + r_.f64[i + 1] = a_.f64[1]; + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_broadcast_f64x2 + #define _mm512_broadcast_f64x2(a) simde_mm512_broadcast_f64x2(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_broadcast_f64x2(simde__m512d src, simde__mmask8 k, simde__m128d a) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_mask_broadcast_f64x2(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_broadcast_f64x2(a)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_broadcast_f64x2 + #define _mm512_mask_broadcast_f64x2(src, k, a) simde_mm512_mask_broadcast_f64x2(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_broadcast_f64x2(simde__mmask8 k, simde__m128d a) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_maskz_broadcast_f64x2(k, a); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_broadcast_f64x2(a)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_broadcast_f64x2 + #define _mm512_maskz_broadcast_f64x2(k, a) simde_mm512_maskz_broadcast_f64x2(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_broadcast_f32x4 (simde__m128 a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_broadcast_f32x4(a); + #else + simde__m256_private r_; + simde__m128_private a_ = simde__m128_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128_private[0] = a_; + r_.m128_private[1] = a_; + #elif defined(SIMDE_VECTOR_SUBSCRIPT) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) + r_.f32 = __builtin_shufflevector(a_.f32, a_.f32, 0, 1, 2, 3, 0, 1, 2, 3); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i += 4) { + r_.f32[ i ] = a_.f32[0]; + r_.f32[i + 1] = a_.f32[1]; + r_.f32[i + 2] = a_.f32[2]; + r_.f32[i + 3] = a_.f32[3]; + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm256_broadcast_f32x4 + #define _mm256_broadcast_f32x4(a) simde_mm256_broadcast_f32x4(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask_broadcast_f32x4(simde__m256 src, simde__mmask8 k, simde__m128 a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_broadcast_f32x4(src, k, a); + #else + return simde_mm256_mask_mov_ps(src, k, simde_mm256_broadcast_f32x4(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_broadcast_f32x4 + #define _mm256_mask_broadcast_f32x4(src, k, a) simde_mm256_mask_broadcast_f32x4(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_maskz_broadcast_f32x4(simde__mmask8 k, simde__m128 a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_broadcast_f32x4(k, a); + #else + return simde_mm256_maskz_mov_ps(k, simde_mm256_broadcast_f32x4(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_broadcast_f32x4 + #define _mm256_maskz_broadcast_f32x4(k, a) simde_mm256_maskz_broadcast_f32x4(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_broadcast_f64x2 (simde__m128d a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm256_broadcast_f64x2(a); + #else + simde__m256d_private r_; + simde__m128d_private a_ = simde__m128d_to_private(a); + + /* I don't have a bug # for this, but when compiled with clang-10 without optimization on aarch64 + * the __builtin_shufflevector version doesn't work correctly. clang 9 and 11 aren't a problem */ + #if defined(SIMDE_VECTOR_SUBSCRIPT) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) && \ + (!defined(__clang__) || (SIMDE_DETECT_CLANG_VERSION < 100000 || SIMDE_DETECT_CLANG_VERSION > 100000)) + r_.f64 = __builtin_shufflevector(a_.f64, a_.f64, 0, 1, 0, 1); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i += 2) { + r_.f64[ i ] = a_.f64[0]; + r_.f64[i + 1] = a_.f64[1]; + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_broadcast_f64x2 + #define _mm256_broadcast_f64x2(a) simde_mm256_broadcast_f64x2(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mask_broadcast_f64x2(simde__m256d src, simde__mmask8 k, simde__m128d a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm256_mask_broadcast_f64x2(src, k, a); + #else + return simde_mm256_mask_mov_pd(src, k, simde_mm256_broadcast_f64x2(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_broadcast_f64x2 + #define _mm256_mask_broadcast_f64x2(src, k, a) simde_mm256_mask_broadcast_f64x2(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_maskz_broadcast_f64x2(simde__mmask8 k, simde__m128d a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm256_maskz_broadcast_f64x2(k, a); + #else + return simde_mm256_maskz_mov_pd(k, simde_mm256_broadcast_f64x2(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_broadcast_f64x2 + #define _mm256_maskz_broadcast_f64x2(k, a) simde_mm256_maskz_broadcast_f64x2(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_broadcast_f32x4 (simde__m128 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_broadcast_f32x4(a); + #else + simde__m512_private r_; + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256[1] = r_.m256[0] = simde_mm256_castsi256_ps(simde_mm256_broadcastsi128_si256(simde_mm_castps_si128(a))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = a; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_broadcast_f32x4 + #define _mm512_broadcast_f32x4(a) simde_mm512_broadcast_f32x4(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_broadcast_f32x4(simde__m512 src, simde__mmask16 k, simde__m128 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_broadcast_f32x4(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_broadcast_f32x4(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_broadcast_f32x4 + #define _mm512_mask_broadcast_f32x4(src, k, a) simde_mm512_mask_broadcast_f32x4(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_broadcast_f32x4(simde__mmask16 k, simde__m128 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_broadcast_f32x4(k, a); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_broadcast_f32x4(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_broadcast_f32x4 + #define _mm512_maskz_broadcast_f32x4(k, a) simde_mm512_maskz_broadcast_f32x4(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_broadcast_f64x4 (simde__m256d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_broadcast_f64x4(a); + #else + simde__m512d_private r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = a; + } + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_broadcast_f64x4 + #define _mm512_broadcast_f64x4(a) simde_mm512_broadcast_f64x4(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_broadcast_f64x4(simde__m512d src, simde__mmask8 k, simde__m256d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_broadcast_f64x4(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_broadcast_f64x4(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_broadcast_f64x4 + #define _mm512_mask_broadcast_f64x4(src, k, a) simde_mm512_mask_broadcast_f64x4(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_broadcast_f64x4(simde__mmask8 k, simde__m256d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_broadcast_f64x4(k, a); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_broadcast_f64x4(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_broadcast_f64x4 + #define _mm512_maskz_broadcast_f64x4(k, a) simde_mm512_maskz_broadcast_f64x4(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_broadcast_i32x4 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_broadcast_i32x4(a); + #else + simde__m512i_private r_; + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256i[1] = r_.m256i[0] = simde_mm256_broadcastsi128_si256(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + r_.m128i[3] = r_.m128i[2] = r_.m128i[1] = r_.m128i[0] = a; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = a; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_broadcast_i32x4 + #define _mm512_broadcast_i32x4(a) simde_mm512_broadcast_i32x4(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_broadcast_i32x4(simde__m512i src, simde__mmask16 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_broadcast_i32x4(src, k, a); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_broadcast_i32x4(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_broadcast_i32x4 + #define _mm512_mask_broadcast_i32x4(src, k, a) simde_mm512_mask_broadcast_i32x4(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_broadcast_i32x4(simde__mmask16 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_broadcast_i32x4(k, a); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_broadcast_i32x4(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_broadcast_i32x4 + #define _mm512_maskz_broadcast_i32x4(k, a) simde_mm512_maskz_broadcast_i32x4(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_broadcast_i64x4 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_broadcast_i64x4(a); + #else + simde__m512i_private r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = a; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_broadcast_i64x4 + #define _mm512_broadcast_i64x4(a) simde_mm512_broadcast_i64x4(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_broadcast_i64x4(simde__m512i src, simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_broadcast_i64x4(src, k, a); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_broadcast_i64x4(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_broadcast_i64x4 + #define _mm512_mask_broadcast_i64x4(src, k, a) simde_mm512_mask_broadcast_i64x4(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_broadcast_i64x4(simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_broadcast_i64x4(k, a); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_broadcast_i64x4(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_broadcast_i64x4 + #define _mm512_maskz_broadcast_i64x4(k, a) simde_mm512_maskz_broadcast_i64x4(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_broadcastd_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_broadcastd_epi32(a); + #else + simde__m512i_private r_; + simde__m128i_private a_= simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[0]; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_broadcastd_epi32 + #define _mm512_broadcastd_epi32(a) simde_mm512_broadcastd_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_broadcastd_epi32(simde__m512i src, simde__mmask16 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_broadcastd_epi32(src, k, a); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_broadcastd_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_broadcastd_epi32 + #define _mm512_mask_broadcastd_epi32(src, k, a) simde_mm512_mask_broadcastd_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_broadcastd_epi32(simde__mmask16 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_broadcastd_epi32(k, a); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_broadcastd_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_broadcastd_epi32 + #define _mm512_maskz_broadcastd_epi32(k, a) simde_mm512_maskz_broadcastd_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_broadcastq_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_broadcastq_epi64(a); + #else + simde__m512i_private r_; + simde__m128i_private a_= simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[0]; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_broadcastq_epi64 + #define _mm512_broadcastq_epi64(a) simde_mm512_broadcastq_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_broadcastq_epi64(simde__m512i src, simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_broadcastq_epi64(src, k, a); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_broadcastq_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_broadcastq_epi64 + #define _mm512_mask_broadcastq_epi64(src, k, a) simde_mm512_mask_broadcastq_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_broadcastq_epi64(simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_broadcastq_epi64(k, a); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_broadcastq_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_broadcastq_epi64 + #define _mm512_maskz_broadcastq_epi64(k, a) simde_mm512_maskz_broadcastq_epi64(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_broadcastss_ps (simde__m128 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_broadcastss_ps(a); + #else + simde__m512_private r_; + simde__m128_private a_= simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[0]; + } + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_broadcastss_ps + #define _mm512_broadcastss_ps(a) simde_mm512_broadcastss_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_broadcastss_ps(simde__m512 src, simde__mmask16 k, simde__m128 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_broadcastss_ps(src, k, a); + #else + simde__m512_private + src_ = simde__m512_to_private(src), + r_; + simde__m128_private + a_ = simde__m128_to_private(a); + + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = ((k >> i) & 1) ? a_.f32[0] : src_.f32[i]; + } + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_broadcastss_ps + #define _mm512_mask_broadcastss_ps(src, k, a) simde_mm512_mask_broadcastss_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_broadcastss_ps(simde__mmask16 k, simde__m128 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_broadcastss_ps(k, a); + #else + simde__m512_private + r_; + simde__m128_private + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = ((k >> i) & 1) ? a_.f32[0] : INT32_C(0); + } + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_broadcastss_ps + #define _mm512_maskz_broadcastss_ps(k, a) simde_mm512_maskz_broadcastss_ps(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_broadcastsd_pd (simde__m128d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_broadcastsd_pd(a); + #else + simde__m512d_private r_; + simde__m128d_private a_= simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[0]; + } + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_broadcastsd_pd + #define _mm512_broadcastsd_pd(a) simde_mm512_broadcastsd_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_broadcastsd_pd(simde__m512d src, simde__mmask8 k, simde__m128d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_broadcastsd_pd(src, k, a); + #else + simde__m512d_private + src_ = simde__m512d_to_private(src), + r_; + simde__m128d_private + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = ((k >> i) & 1) ? a_.f64[0] : src_.f64[i]; + } + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_broadcastsd_pd + #define _mm512_mask_broadcastsd_pd(src, k, a) simde_mm512_mask_broadcastsd_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_broadcastsd_pd(simde__mmask8 k, simde__m128d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_broadcastsd_pd(k, a); + #else + simde__m512d_private + r_; + simde__m128d_private + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = ((k >> i) & 1) ? a_.f64[0] : INT64_C(0); + } + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_broadcastsd_pd + #define _mm512_maskz_broadcastsd_pd(k, a) simde_mm512_maskz_broadcastsd_pd(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_broadcastb_epi8 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_broadcastb_epi8(a); + #else + simde__m128i_private a_= simde__m128i_to_private(a); + return simde_mm512_set1_epi8(a_.i8[0]); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_broadcastb_epi8 + #define _mm512_broadcastb_epi8(a) simde_mm512_broadcastb_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_broadcastb_epi8 (simde__m512i src, simde__mmask64 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_broadcastb_epi8(src, k, a); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_broadcastb_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_broadcastb_epi8 + #define _mm512_mask_broadcastb_epi8(src, k, a) simde_mm512_mask_broadcastb_epi8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_broadcastb_epi8 (simde__mmask64 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_broadcastb_epi8(k, a); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_broadcastb_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_broadcastb_epi8 + #define _mm512_maskz_broadcastb_epi8(k, a) simde_mm512_maskz_broadcastb_epi8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_broadcastw_epi16 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_broadcastw_epi16(a); + #else + simde__m128i_private a_= simde__m128i_to_private(a); + return simde_mm512_set1_epi16(a_.i16[0]); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_broadcastw_epi16 + #define _mm512_broadcastw_epi16(a) simde_mm512_broadcastw_epi16(a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_BROADCAST_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cast.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cast.h new file mode 100644 index 00000000..7f67a573 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cast.h @@ -0,0 +1,357 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + * 2020 Hidayat Khan + * 2020 Christopher Moore + */ + +#if !defined(SIMDE_X86_AVX512_CAST_H) +#define SIMDE_X86_AVX512_CAST_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_castpd_ps (simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castpd_ps(a); + #else + simde__m512 r; + simde_memcpy(&r, &a, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castpd_ps + #define _mm512_castpd_ps(a) simde_mm512_castpd_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_castpd_si512 (simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castpd_si512(a); + #else + simde__m512i r; + simde_memcpy(&r, &a, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castpd_si512 + #define _mm512_castpd_si512(a) simde_mm512_castpd_si512(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_castps_pd (simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castps_pd(a); + #else + simde__m512d r; + simde_memcpy(&r, &a, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castps_pd + #define _mm512_castps_pd(a) simde_mm512_castps_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_castps_si512 (simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castps_si512(a); + #else + simde__m512i r; + simde_memcpy(&r, &a, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castps_si512 + #define _mm512_castps_si512(a) simde_mm512_castps_si512(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_castph_si512 (simde__m512h a) { + #if defined(SIMDE_X86_AVX512FP16_NATIVE) + return _mm512_castph_si512(a); + #else + simde__m512i r; + simde_memcpy(&r, &a, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) + #undef _mm512_castph_si512 + #define _mm512_castph_si512(a) simde_mm512_castph_si512(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512h +simde_mm512_castsi512_ph (simde__m512i a) { + #if defined(SIMDE_X86_AVX512FP16_NATIVE) + return _mm512_castsi512_ph(a); + #else + simde__m512h r; + simde_memcpy(&r, &a, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) + #undef _mm512_castsi512_ph + #define _mm512_castsi512_ph(a) simde_mm512_castsi512_ph(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_castsi512_ps (simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castsi512_ps(a); + #else + simde__m512 r; + simde_memcpy(&r, &a, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castsi512_ps + #define _mm512_castsi512_ps(a) simde_mm512_castsi512_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_castsi512_pd (simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castsi512_pd(a); + #else + simde__m512d r; + simde_memcpy(&r, &a, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castsi512_pd + #define _mm512_castsi512_pd(a) simde_mm512_castsi512_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_castpd128_pd512 (simde__m128d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castpd128_pd512(a); + #else + simde__m512d_private r_; + r_.m128d[0] = a; + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castpd128_pd512 + #define _mm512_castpd128_pd512(a) simde_mm512_castpd128_pd512(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_castpd256_pd512 (simde__m256d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castpd256_pd512(a); + #else + simde__m512d_private r_; + r_.m256d[0] = a; + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castpd256_pd512 + #define _mm512_castpd256_pd512(a) simde_mm512_castpd256_pd512(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm512_castpd512_pd128 (simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castpd512_pd128(a); + #else + simde__m512d_private a_ = simde__m512d_to_private(a); + return a_.m128d[0]; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castpd512_pd128 + #define _mm512_castpd512_pd128(a) simde_mm512_castpd512_pd128(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm512_castpd512_pd256 (simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castpd512_pd256(a); + #else + simde__m512d_private a_ = simde__m512d_to_private(a); + return a_.m256d[0]; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castpd512_pd256 + #define _mm512_castpd512_pd256(a) simde_mm512_castpd512_pd256(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_castps128_ps512 (simde__m128 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castps128_ps512(a); + #else + simde__m512_private r_; + r_.m128[0] = a; + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castps128_ps512 + #define _mm512_castps128_ps512(a) simde_mm512_castps128_ps512(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_castps256_ps512 (simde__m256 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castps256_ps512(a); + #else + simde__m512_private r_; + r_.m256[0] = a; + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castps256_ps512 + #define _mm512_castps256_ps512(a) simde_mm512_castps256_ps512(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm512_castps512_ps128 (simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castps512_ps128(a); + #else + simde__m512_private a_ = simde__m512_to_private(a); + return a_.m128[0]; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castps512_ps128 + #define _mm512_castps512_ps128(a) simde_mm512_castps512_ps128(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm512_castps512_ps256 (simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castps512_ps256(a); + #else + simde__m512_private a_ = simde__m512_to_private(a); + return a_.m256[0]; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castps512_ps256 + #define _mm512_castps512_ps256(a) simde_mm512_castps512_ps256(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_castsi128_si512 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castsi128_si512(a); + #else + simde__m512i_private r_; + r_.m128i[0] = a; + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castsi128_si512 + #define _mm512_castsi128_si512(a) simde_mm512_castsi128_si512(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_castsi256_si512 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castsi256_si512(a); + #else + simde__m512i_private r_; + r_.m256i[0] = a; + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castsi256_si512 + #define _mm512_castsi256_si512(a) simde_mm512_castsi256_si512(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm512_castsi512_si128 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castsi512_si128(a); + #else + simde__m512i_private a_ = simde__m512i_to_private(a); + return a_.m128i[0]; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castsi512_si128 + #define _mm512_castsi512_si128(a) simde_mm512_castsi512_si128(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_castsi512_si256 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_castsi512_si256(a); + #else + simde__m512i_private a_ = simde__m512i_to_private(a); + return a_.m256i[0]; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_castsi512_si256 + #define _mm512_castsi512_si256(a) simde_mm512_castsi512_si256(a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_CAST_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmp.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmp.h new file mode 100644 index 00000000..2a3b99c3 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmp.h @@ -0,0 +1,1714 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020-2021 Evan Nemerson + * 2020 Himanshi Mathur + */ + +#if !defined(SIMDE_X86_AVX512_CMP_H) +#define SIMDE_X86_AVX512_CMP_H + +#include "types.h" +#include "mov.h" +#include "mov_mask.h" +#include "setzero.h" +#include "setone.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#if defined(__clang__) && SIMDE_FLOAT16_API == SIMDE_FLOAT16_API_FP16 +SIMDE_DIAGNOSTIC_DISABLE_DOUBLE_PROMOTION_ +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_cmp_epi8_mask (simde__m512i a, simde__m512i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + switch (imm8) { + case SIMDE_MM_CMPINT_EQ: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), (a_.i8 == b_.i8)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] == b_.i8[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), (a_.i8 < b_.i8)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] < b_.i8[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), (a_.i8 <= b_.i8)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] <= b_.i8[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_FALSE: + r_ = simde__m512i_to_private(simde_mm512_setzero_si512()); + break; + + + case SIMDE_MM_CMPINT_NE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), (a_.i8 != b_.i8)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] != b_.i8[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), ~(a_.i8 < b_.i8)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = !(a_.i8[i] < b_.i8[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), ~(a_.i8 <= b_.i8)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = !(a_.i8[i] <= b_.i8[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_TRUE: + r_ = simde__m512i_to_private(simde_x_mm512_setone_si512()); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde_mm512_movepi8_mask(simde__m512i_from_private(r_)); +} +#if defined(SIMDE_X86_AVX512BW_NATIVE) + #define simde_mm512_cmp_epi8_mask(a, b, imm8) _mm512_cmp_epi8_mask((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmp_epi8_mask + #define _mm512_cmp_epi8_mask(a, b, imm8) simde_mm512_cmp_epi8_mask((a), (b), (imm8)) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_cmp_epi32_mask (simde__m256i a, simde__m256i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + switch (imm8) { + case SIMDE_MM_CMPINT_EQ: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.i32 == b_.i32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] == b_.i32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.i32 < b_.i32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] < b_.i32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.i32 <= b_.i32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] <= b_.i32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_FALSE: + r_ = simde__m256i_to_private(simde_mm256_setzero_si256()); + break; + + + case SIMDE_MM_CMPINT_NE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.i32 != b_.i32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] != b_.i32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), ~(a_.i32 < b_.i32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = !(a_.i32[i] < b_.i32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), ~(a_.i32 <= b_.i32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = !(a_.i32[i] <= b_.i32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_TRUE: + r_ = simde__m256i_to_private(simde_x_mm256_setone_si256()); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde_mm256_movepi32_mask(simde__m256i_from_private(r_)); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_cmp_epi32_mask(a, b, imm8) _mm256_cmp_epi32_mask((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmp_epi32_mask + #define _mm256_cmp_epi32_mask(a, b, imm8) simde_mm256_cmp_epi32_mask((a), (b), (imm8)) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_cmp_ps_mask (simde__m512 a, simde__m512 b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 31) { + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + switch (imm8) { + case SIMDE_CMP_EQ_OQ: + case SIMDE_CMP_EQ_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 == b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = (a_.f32[i] == b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_LT_OQ: + case SIMDE_CMP_LT_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 < b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = (a_.f32[i] < b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_LE_OQ: + case SIMDE_CMP_LE_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 <= b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = (a_.f32[i] <= b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_UNORD_Q: + case SIMDE_CMP_UNORD_S: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 != a_.f32) | (b_.f32 != b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = ((a_.f32[i] != a_.f32[i]) || (b_.f32[i] != b_.f32[i])) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NEQ_UQ: + case SIMDE_CMP_NEQ_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 != b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = (a_.f32[i] != b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NEQ_OQ: + case SIMDE_CMP_NEQ_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 == a_.f32) & (b_.f32 == b_.f32) & (a_.f32 != b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = ((a_.f32[i] == a_.f32[i]) & (b_.f32[i] == b_.f32[i]) & (a_.f32[i] != b_.f32[i])) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NLT_UQ: + case SIMDE_CMP_NLT_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), ~(a_.f32 < b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = !(a_.f32[i] < b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NLE_UQ: + case SIMDE_CMP_NLE_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), ~(a_.f32 <= b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = !(a_.f32[i] <= b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_ORD_Q: + case SIMDE_CMP_ORD_S: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), ((a_.f32 == a_.f32) & (b_.f32 == b_.f32))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = ((a_.f32[i] == a_.f32[i]) & (b_.f32[i] == b_.f32[i])) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_EQ_UQ: + case SIMDE_CMP_EQ_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 != a_.f32) | (b_.f32 != b_.f32) | (a_.f32 == b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = ((a_.f32[i] != a_.f32[i]) | (b_.f32[i] != b_.f32[i]) | (a_.f32[i] == b_.f32[i])) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NGE_UQ: + case SIMDE_CMP_NGE_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), ~(a_.f32 >= b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = !(a_.f32[i] >= b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_NGT_UQ: + case SIMDE_CMP_NGT_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), ~(a_.f32 > b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = !(a_.f32[i] > b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_FALSE_OQ: + case SIMDE_CMP_FALSE_OS: + r_ = simde__m512_to_private(simde_mm512_setzero_ps()); + break; + + case SIMDE_CMP_GE_OQ: + case SIMDE_CMP_GE_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 >= b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = (a_.f32[i] >= b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_GT_OQ: + case SIMDE_CMP_GT_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 > b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.i32[i] = (a_.f32[i] > b_.f32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + break; + + case SIMDE_CMP_TRUE_UQ: + case SIMDE_CMP_TRUE_US: + r_ = simde__m512_to_private(simde_x_mm512_setone_ps()); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde_mm512_movepi32_mask(simde_mm512_castps_si512(simde__m512_from_private(r_))); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_cmp_ps_mask(a, b, imm8) _mm512_cmp_ps_mask((a), (b), (imm8)) +#elif defined(SIMDE_STATEMENT_EXPR_) && SIMDE_NATURAL_VECTOR_SIZE_LE(128) + #define simde_mm512_cmp_ps_mask(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512_private \ + simde_mm512_cmp_ps_mask_r_ = simde__m512_to_private(simde_mm512_setzero_ps()), \ + simde_mm512_cmp_ps_mask_a_ = simde__m512_to_private((a)), \ + simde_mm512_cmp_ps_mask_b_ = simde__m512_to_private((b)); \ + \ + for (size_t i = 0 ; i < (sizeof(simde_mm512_cmp_ps_mask_r_.m128) / sizeof(simde_mm512_cmp_ps_mask_r_.m128[0])) ; i++) { \ + simde_mm512_cmp_ps_mask_r_.m128[i] = simde_mm_cmp_ps(simde_mm512_cmp_ps_mask_a_.m128[i], simde_mm512_cmp_ps_mask_b_.m128[i], (imm8)); \ + } \ + \ + simde_mm512_movepi32_mask(simde_mm512_castps_si512(simde__m512_from_private(simde_mm512_cmp_ps_mask_r_))); \ + })) +#elif defined(SIMDE_STATEMENT_EXPR_) && SIMDE_NATURAL_VECTOR_SIZE_LE(256) + #define simde_mm512_cmp_ps_mask(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512_private \ + simde_mm512_cmp_ps_mask_r_ = simde__m512_to_private(simde_mm512_setzero_ps()), \ + simde_mm512_cmp_ps_mask_a_ = simde__m512_to_private((a)), \ + simde_mm512_cmp_ps_mask_b_ = simde__m512_to_private((b)); \ + \ + for (size_t i = 0 ; i < (sizeof(simde_mm512_cmp_ps_mask_r_.m256) / sizeof(simde_mm512_cmp_ps_mask_r_.m256[0])) ; i++) { \ + simde_mm512_cmp_ps_mask_r_.m256[i] = simde_mm256_cmp_ps(simde_mm512_cmp_ps_mask_a_.m256[i], simde_mm512_cmp_ps_mask_b_.m256[i], (imm8)); \ + } \ + \ + simde_mm512_movepi32_mask(simde_mm512_castps_si512(simde__m512_from_private(simde_mm512_cmp_ps_mask_r_))); \ + })) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmp_ps_mask + #define _mm512_cmp_ps_mask(a, b, imm8) simde_mm512_cmp_ps_mask((a), (b), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_cmp_ps_mask(a, b, imm8) _mm256_cmp_ps_mask((a), (b), (imm8)) +#else + #define simde_mm256_cmp_ps_mask(a, b, imm8) simde_mm256_movepi32_mask(simde_mm256_castps_si256(simde_mm256_cmp_ps((a), (b), (imm8)))) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmp_ps_mask + #define _mm256_cmp_ps_mask(a, b, imm8) simde_mm256_cmp_ps_mask((a), (b), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_cmp_ps_mask(a, b, imm8) _mm_cmp_ps_mask((a), (b), (imm8)) +#else + #define simde_mm_cmp_ps_mask(a, b, imm8) simde_mm_movepi32_mask(simde_mm_castps_si128(simde_mm_cmp_ps((a), (b), (imm8)))) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cmp_ps_mask + #define _mm_cmp_ps_mask(a, b, imm8) simde_mm_cmp_ps_mask((a), (b), (imm8)) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_cmp_pd_mask (simde__m512d a, simde__m512d b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 31) { + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + switch (imm8) { + case SIMDE_CMP_EQ_OQ: + case SIMDE_CMP_EQ_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 == b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = (a_.f64[i] == b_.f64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + break; + + case SIMDE_CMP_LT_OQ: + case SIMDE_CMP_LT_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 < b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = (a_.f64[i] < b_.f64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + break; + + case SIMDE_CMP_LE_OQ: + case SIMDE_CMP_LE_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 <= b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = (a_.f64[i] <= b_.f64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + break; + + case SIMDE_CMP_UNORD_Q: + case SIMDE_CMP_UNORD_S: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 != a_.f64) | (b_.f64 != b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = ((a_.f64[i] != a_.f64[i]) || (b_.f64[i] != b_.f64[i])) ? ~INT64_C(0) : INT64_C(0); + } + #endif + break; + + case SIMDE_CMP_NEQ_UQ: + case SIMDE_CMP_NEQ_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 != b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = (a_.f64[i] != b_.f64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + break; + + case SIMDE_CMP_NEQ_OQ: + case SIMDE_CMP_NEQ_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 == a_.f64) & (b_.f64 == b_.f64) & (a_.f64 != b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = ((a_.f64[i] == a_.f64[i]) & (b_.f64[i] == b_.f64[i]) & (a_.f64[i] != b_.f64[i])) ? ~INT64_C(0) : INT64_C(0); + } + #endif + break; + + case SIMDE_CMP_NLT_UQ: + case SIMDE_CMP_NLT_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), ~(a_.f64 < b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = !(a_.f64[i] < b_.f64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + break; + + case SIMDE_CMP_NLE_UQ: + case SIMDE_CMP_NLE_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), ~(a_.f64 <= b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = !(a_.f64[i] <= b_.f64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + break; + + case SIMDE_CMP_ORD_Q: + case SIMDE_CMP_ORD_S: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), ((a_.f64 == a_.f64) & (b_.f64 == b_.f64))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = ((a_.f64[i] == a_.f64[i]) & (b_.f64[i] == b_.f64[i])) ? ~INT64_C(0) : INT64_C(0); + } + #endif + break; + + case SIMDE_CMP_EQ_UQ: + case SIMDE_CMP_EQ_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 != a_.f64) | (b_.f64 != b_.f64) | (a_.f64 == b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = ((a_.f64[i] != a_.f64[i]) | (b_.f64[i] != b_.f64[i]) | (a_.f64[i] == b_.f64[i])) ? ~INT64_C(0) : INT64_C(0); + } + #endif + break; + + case SIMDE_CMP_NGE_UQ: + case SIMDE_CMP_NGE_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), ~(a_.f64 >= b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = !(a_.f64[i] >= b_.f64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + break; + + case SIMDE_CMP_NGT_UQ: + case SIMDE_CMP_NGT_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), ~(a_.f64 > b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = !(a_.f64[i] > b_.f64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + break; + + case SIMDE_CMP_FALSE_OQ: + case SIMDE_CMP_FALSE_OS: + r_ = simde__m512d_to_private(simde_mm512_setzero_pd()); + break; + + case SIMDE_CMP_GE_OQ: + case SIMDE_CMP_GE_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 >= b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = (a_.f64[i] >= b_.f64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + break; + + case SIMDE_CMP_GT_OQ: + case SIMDE_CMP_GT_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 > b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.i64[i] = (a_.f64[i] > b_.f64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + break; + + case SIMDE_CMP_TRUE_UQ: + case SIMDE_CMP_TRUE_US: + r_ = simde__m512d_to_private(simde_x_mm512_setone_pd()); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde_mm512_movepi64_mask(simde_mm512_castpd_si512(simde__m512d_from_private(r_))); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_cmp_pd_mask(a, b, imm8) _mm512_cmp_pd_mask((a), (b), (imm8)) +#elif defined(SIMDE_STATEMENT_EXPR_) && SIMDE_NATURAL_VECTOR_SIZE_LE(128) + #define simde_mm512_cmp_pd_mask(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d_private \ + simde_mm512_cmp_pd_mask_r_ = simde__m512d_to_private(simde_mm512_setzero_pd()), \ + simde_mm512_cmp_pd_mask_a_ = simde__m512d_to_private((a)), \ + simde_mm512_cmp_pd_mask_b_ = simde__m512d_to_private((b)); \ + \ + for (size_t simde_mm512_cmp_pd_mask_i = 0 ; simde_mm512_cmp_pd_mask_i < (sizeof(simde_mm512_cmp_pd_mask_r_.m128d) / sizeof(simde_mm512_cmp_pd_mask_r_.m128d[0])) ; simde_mm512_cmp_pd_mask_i++) { \ + simde_mm512_cmp_pd_mask_r_.m128d[simde_mm512_cmp_pd_mask_i] = simde_mm_cmp_pd(simde_mm512_cmp_pd_mask_a_.m128d[simde_mm512_cmp_pd_mask_i], simde_mm512_cmp_pd_mask_b_.m128d[simde_mm512_cmp_pd_mask_i], (imm8)); \ + } \ + \ + simde_mm512_movepi64_mask(simde_mm512_castpd_si512(simde__m512d_from_private(simde_mm512_cmp_pd_mask_r_))); \ + })) +#elif defined(SIMDE_STATEMENT_EXPR_) && SIMDE_NATURAL_VECTOR_SIZE_LE(256) + #define simde_mm512_cmp_pd_mask(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d_private \ + simde_mm512_cmp_pd_mask_r_ = simde__m512d_to_private(simde_mm512_setzero_pd()), \ + simde_mm512_cmp_pd_mask_a_ = simde__m512d_to_private((a)), \ + simde_mm512_cmp_pd_mask_b_ = simde__m512d_to_private((b)); \ + \ + for (size_t simde_mm512_cmp_pd_mask_i = 0 ; simde_mm512_cmp_pd_mask_i < (sizeof(simde_mm512_cmp_pd_mask_r_.m256d) / sizeof(simde_mm512_cmp_pd_mask_r_.m256d[0])) ; simde_mm512_cmp_pd_mask_i++) { \ + simde_mm512_cmp_pd_mask_r_.m256d[simde_mm512_cmp_pd_mask_i] = simde_mm256_cmp_pd(simde_mm512_cmp_pd_mask_a_.m256d[simde_mm512_cmp_pd_mask_i], simde_mm512_cmp_pd_mask_b_.m256d[simde_mm512_cmp_pd_mask_i], (imm8)); \ + } \ + \ + simde_mm512_movepi64_mask(simde_mm512_castpd_si512(simde__m512d_from_private(simde_mm512_cmp_pd_mask_r_))); \ + })) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmp_pd_mask + #define _mm512_cmp_pd_mask(a, b, imm8) simde_mm512_cmp_pd_mask((a), (b), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_cmp_pd_mask(a, b, imm8) _mm256_cmp_pd_mask((a), (b), (imm8)) +#else + #define simde_mm256_cmp_pd_mask(a, b, imm8) simde_mm256_movepi64_mask(simde_mm256_castpd_si256(simde_mm256_cmp_pd((a), (b), (imm8)))) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmp_pd_mask + #define _mm256_cmp_pd_mask(a, b, imm8) simde_mm256_cmp_pd_mask((a), (b), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_cmp_pd_mask(a, b, imm8) _mm_cmp_pd_mask((a), (b), (imm8)) +#else + #define simde_mm_cmp_pd_mask(a, b, imm8) simde_mm_movepi64_mask(simde_mm_castpd_si128(simde_mm_cmp_pd((a), (b), (imm8)))) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cmp_pd_mask + #define _mm_cmp_pd_mask(a, b, imm8) simde_mm_cmp_pd_mask((a), (b), (imm8)) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_cmp_ph_mask (simde__m512h a, simde__m512h b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 31) { + simde__m512h_private + r_, + a_ = simde__m512h_to_private(a), + b_ = simde__m512h_to_private(b); + + switch (imm8) { + case SIMDE_CMP_EQ_OQ: + case SIMDE_CMP_EQ_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_FLOAT16_VECTOR) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.f16 == b_.f16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.i16[i] = ( + simde_float16_as_uint16(a_.f16[i]) == simde_float16_as_uint16(b_.f16[i]) + && !simde_isnanhf(a_.f16[i]) && !simde_isnanhf(b_.f16[i]) + ) ? ~INT16_C(0) : INT16_C(0); + } + #endif + break; + + case SIMDE_CMP_LT_OQ: + case SIMDE_CMP_LT_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_FLOAT16_VECTOR) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.f16 < b_.f16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.i16[i] = (simde_float16_to_float32(a_.f16[i]) < simde_float16_to_float32(b_.f16[i])) ? ~INT16_C(0) : INT16_C(0); + } + #endif + break; + + case SIMDE_CMP_LE_OQ: + case SIMDE_CMP_LE_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_FLOAT16_VECTOR) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.f16 <= b_.f16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.i16[i] = (simde_float16_to_float32(a_.f16[i]) <= simde_float16_to_float32(b_.f16[i])) ? ~INT16_C(0) : INT16_C(0); + } + #endif + break; + + case SIMDE_CMP_UNORD_Q: + case SIMDE_CMP_UNORD_S: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_FLOAT16_VECTOR) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.f16 != a_.f16) | (b_.f16 != b_.f16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.i16[i] = ( + (simde_float16_to_float32(a_.f16[i]) != simde_float16_to_float32(a_.f16[i])) + || (simde_float16_to_float32(b_.f16[i]) != simde_float16_to_float32(b_.f16[i])) + ) ? ~INT16_C(0) : INT16_C(0); + } + #endif + break; + + case SIMDE_CMP_NEQ_UQ: + case SIMDE_CMP_NEQ_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_FLOAT16_VECTOR) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.f16 != b_.f16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.i16[i] = ( + (simde_float16_as_uint16(a_.f16[i]) != simde_float16_as_uint16(b_.f16[i])) + || simde_isnanhf(a_.f16[i]) || simde_isnanhf(b_.f16[i]) + ) ? ~INT16_C(0) : INT16_C(0); + } + #endif + break; + + case SIMDE_CMP_NEQ_OQ: + case SIMDE_CMP_NEQ_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_FLOAT16_VECTOR) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.f16 == a_.f16) & (b_.f16 == b_.f16) & (a_.f16 != b_.f16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.i16[i] = ( + !(simde_isnanhf(a_.f16[i]) || simde_isnanhf(b_.f16[i])) + && (simde_float16_as_uint16(a_.f16[i]) != simde_float16_as_uint16(b_.f16[i])) + ) ? ~INT16_C(0) : INT16_C(0); + } + #endif + break; + + case SIMDE_CMP_NLT_UQ: + case SIMDE_CMP_NLT_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_FLOAT16_VECTOR) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), ~(a_.f16 < b_.f16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.i16[i] = !( + simde_float16_to_float32(a_.f16[i]) < simde_float16_to_float32(b_.f16[i]) + ) ? ~INT16_C(0) : INT16_C(0); + } + #endif + break; + + case SIMDE_CMP_NLE_UQ: + case SIMDE_CMP_NLE_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_FLOAT16_VECTOR) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), ~(a_.f16 <= b_.f16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.i16[i] = !( + simde_float16_to_float32(a_.f16[i]) <= simde_float16_to_float32(b_.f16[i]) + ) ? ~INT16_C(0) : INT16_C(0); + } + #endif + break; + + case SIMDE_CMP_ORD_Q: + case SIMDE_CMP_ORD_S: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_FLOAT16_VECTOR) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), ((a_.f16 == a_.f16) & (b_.f16 == b_.f16))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.i16[i] = (simde_isnanhf(a_.f16[i]) || simde_isnanhf(b_.f16[i])) ? INT16_C(0) : ~INT16_C(0); + } + #endif + break; + + case SIMDE_CMP_EQ_UQ: + case SIMDE_CMP_EQ_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_FLOAT16_VECTOR) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.f16 != a_.f16) | (b_.f16 != b_.f16) | (a_.f16 == b_.f16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.i16[i] = ( + (simde_isnanhf(a_.f16[i]) || simde_isnanhf(b_.f16[i])) + || (simde_float16_as_uint16(a_.f16[i]) == simde_float16_as_uint16(b_.f16[i])) + ) ? ~INT16_C(0) : INT16_C(0); + } + #endif + break; + + case SIMDE_CMP_NGE_UQ: + case SIMDE_CMP_NGE_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_FLOAT16_VECTOR) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), ~(a_.f16 >= b_.f16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.i16[i] = !( + simde_float16_to_float32(a_.f16[i]) >= simde_float16_to_float32(b_.f16[i]) + ) ? ~INT16_C(0) : INT16_C(0); + } + #endif + break; + + case SIMDE_CMP_NGT_UQ: + case SIMDE_CMP_NGT_US: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_FLOAT16_VECTOR) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), ~(a_.f16 > b_.f16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.i16[i] = !( + simde_float16_to_float32(a_.f16[i]) > simde_float16_to_float32(b_.f16[i]) + ) ? ~INT16_C(0) : INT16_C(0); + } + #endif + break; + + case SIMDE_CMP_FALSE_OQ: + case SIMDE_CMP_FALSE_OS: + r_ = simde__m512h_to_private(simde_mm512_setzero_ph()); + break; + + case SIMDE_CMP_GE_OQ: + case SIMDE_CMP_GE_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_FLOAT16_VECTOR) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.f16 >= b_.f16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.i16[i] = ( + simde_float16_to_float32(a_.f16[i]) >= simde_float16_to_float32(b_.f16[i]) + ) ? ~INT16_C(0) : INT16_C(0); + } + #endif + break; + + case SIMDE_CMP_GT_OQ: + case SIMDE_CMP_GT_OS: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_FLOAT16_VECTOR) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.f16 > b_.f16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.i16[i] = ( + simde_float16_to_float32(a_.f16[i]) > simde_float16_to_float32(b_.f16[i]) + ) ? ~INT16_C(0) : INT16_C(0); + } + #endif + break; + + case SIMDE_CMP_TRUE_UQ: + case SIMDE_CMP_TRUE_US: + r_ = simde__m512h_to_private(simde_x_mm512_setone_ph()); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde_mm512_movepi16_mask(simde_mm512_castph_si512(simde__m512h_from_private(r_))); +} +#if defined(SIMDE_X86_AVX512FP16_NATIVE) + #define simde_mm512_cmp_ph_mask(a, b, imm8) _mm512_cmp_ph_mask((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmp_ph_mask + #define _mm512_cmp_ph_mask(a, b, imm8) simde_mm512_cmp_ph_mask((a), (b), (imm8)) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_cmp_epi16_mask (simde__m512i a, simde__m512i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + switch (imm8) { + case SIMDE_MM_CMPINT_EQ: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.i16 == b_.i16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] == b_.i16[i]) ? ~UINT16_C(0) : UINT16_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.i16 < b_.i16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] < b_.i16[i]) ? ~UINT16_C(0) : UINT16_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.i16 <= b_.i16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] <= b_.i16[i]) ? ~UINT16_C(0) : UINT16_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_FALSE: + r_ = simde__m512i_to_private(simde_mm512_setzero_si512()); + break; + + + case SIMDE_MM_CMPINT_NE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.i16 != b_.i16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] != b_.i16[i]) ? ~UINT16_C(0) : UINT16_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), ~(a_.i16 < b_.i16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = !(a_.i16[i] < b_.i16[i]) ? ~UINT16_C(0) : UINT16_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), ~(a_.i16 <= b_.i16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = !(a_.i16[i] <= b_.i16[i]) ? ~UINT16_C(0) : UINT16_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_TRUE: + r_ = simde__m512i_to_private(simde_x_mm512_setone_si512()); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde_mm512_movepi16_mask(simde__m512i_from_private(r_)); +} +#if defined(SIMDE_X86_AVX512BW_NATIVE) + #define simde_mm512_cmp_epi16_mask(a, b, imm8) _mm512_cmp_epi16_mask((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmp_epi16_mask + #define _mm512_cmp_epi16_mask(a, b, imm8) simde_mm512_cmp_epi16_mask((a), (b), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512BW_NATIVE) + #define simde_mm512_mask_cmp_epi16_mask(k1, a, b, imm8) _mm512_mask_cmp_epi16_mask(k1, a, b, imm8) +#else + #define simde_mm512_mask_cmp_epi16_mask(k1, a, b, imm8) (k1) & simde_mm512_cmp_epi16_mask(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmp_epi16_mask +#define _mm512_mask_cmp_epi16_mask(k1, a, b, imm8) simde_mm512_mask_cmp_epi16_mask((k1), (a), (b), (imm8)) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_cmp_epi32_mask (simde__m512i a, simde__m512i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + switch (imm8) { + case SIMDE_MM_CMPINT_EQ: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.i32 == b_.i32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] == b_.i32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.i32 < b_.i32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] < b_.i32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.i32 <= b_.i32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] <= b_.i32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_FALSE: + r_ = simde__m512i_to_private(simde_mm512_setzero_si512()); + break; + + + case SIMDE_MM_CMPINT_NE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.i32 != b_.i32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] != b_.i32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), ~(a_.i32 < b_.i32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = !(a_.i32[i] < b_.i32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), ~(a_.i32 <= b_.i32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = !(a_.i32[i] <= b_.i32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_TRUE: + r_ = simde__m512i_to_private(simde_x_mm512_setone_si512()); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde_mm512_movepi32_mask(simde__m512i_from_private(r_)); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_cmp_epi32_mask(a, b, imm8) _mm512_cmp_epi32_mask((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmp_epi32_mask + #define _mm512_cmp_epi32_mask(a, b, imm8) simde_mm512_cmp_epi32_mask((a), (b), (imm8)) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_cmp_epi64_mask (simde__m512i a, simde__m512i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + switch (imm8) { + case SIMDE_MM_CMPINT_EQ: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.i64 == b_.i64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] == b_.i64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.i64 < b_.i64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] < b_.i64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.i64 <= b_.i64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] <= b_.i64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_FALSE: + r_ = simde__m512i_to_private(simde_mm512_setzero_si512()); + break; + + + case SIMDE_MM_CMPINT_NE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.i64 != b_.i64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] != b_.i64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), ~(a_.i64 < b_.i64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = !(a_.i64[i] < b_.i64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), ~(a_.i64 <= b_.i64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = !(a_.i64[i] <= b_.i64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_TRUE: + r_ = simde__m512i_to_private(simde_x_mm512_setone_si512()); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde_mm512_movepi64_mask(simde__m512i_from_private(r_)); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_cmp_epi64_mask(a, b, imm8) _mm512_cmp_epi64_mask((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmp_epi64_mask + #define _mm512_cmp_epi64_mask(a, b, imm8) simde_mm512_cmp_epi64_mask((a), (b), (imm8)) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_cmp_epu16_mask (simde__m512i a, simde__m512i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + switch (imm8) { + case SIMDE_MM_CMPINT_EQ: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), (a_.u16 == b_.u16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] == b_.u16[i]) ? ~UINT16_C(0) : UINT16_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), (a_.u16 < b_.u16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] < b_.u16[i]) ? ~UINT16_C(0) : UINT16_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), (a_.u16 <= b_.u16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] <= b_.u16[i]) ? ~UINT16_C(0) : UINT16_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_FALSE: + r_ = simde__m512i_to_private(simde_mm512_setzero_si512()); + break; + + + case SIMDE_MM_CMPINT_NE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), (a_.u16 != b_.u16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] != b_.u16[i]) ? ~UINT16_C(0) : UINT16_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), ~(a_.u16 < b_.u16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = !(a_.u16[i] < b_.u16[i]) ? ~UINT16_C(0) : UINT16_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), ~(a_.u16 <= b_.u16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = !(a_.u16[i] <= b_.u16[i]) ? ~UINT16_C(0) : UINT16_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_TRUE: + r_ = simde__m512i_to_private(simde_x_mm512_setone_si512()); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde_mm512_movepi16_mask(simde__m512i_from_private(r_)); +} +#if defined(SIMDE_X86_AVX512BW_NATIVE) + #define simde_mm512_cmp_epu16_mask(a, b, imm8) _mm512_cmp_epu16_mask((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmp_epu16_mask + #define _mm512_cmp_epu16_mask(a, b, imm8) simde_mm512_cmp_epu16_mask((a), (b), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512BW_NATIVE) + #define simde_mm512_mask_cmp_epu16_mask(k1, a, b, imm8) _mm512_mask_cmp_epu16_mask(k1, a, b, imm8) +#else + #define simde_mm512_mask_cmp_epu16_mask(k1, a, b, imm8) (k1) & simde_mm512_cmp_epu16_mask(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmp_epu16_mask +#define _mm512_mask_cmp_epu16_mask(k1, a, b, imm8) simde_mm512_mask_cmp_epu16_mask((k1), (a), (b), (imm8)) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_cmp_epu32_mask (simde__m256i a, simde__m256i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + switch (imm8) { + case SIMDE_MM_CMPINT_EQ: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), (a_.u32 == b_.u32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] == b_.u32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), (a_.u32 < b_.u32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] < b_.u32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), (a_.u32 <= b_.u32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] <= b_.u32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_FALSE: + r_ = simde__m256i_to_private(simde_mm256_setzero_si256()); + break; + + + case SIMDE_MM_CMPINT_NE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), (a_.u32 != b_.u32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] != b_.u32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), ~(a_.u32 < b_.u32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = !(a_.u32[i] < b_.u32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), ~(a_.u32 <= b_.u32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = !(a_.u32[i] <= b_.u32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_TRUE: + r_ = simde__m256i_to_private(simde_x_mm256_setone_si256()); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde_mm256_movepi32_mask(simde__m256i_from_private(r_)); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_cmp_epu32_mask(a, b, imm8) _mm256_cmp_epu32_mask((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmp_epu32_mask + #define _mm256_cmp_epu32_mask(a, b, imm8) simde_mm256_cmp_epu32_mask((a), (b), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_cmp_epu32_mask(k1, a, b, imm8) _mm256_mask_cmp_epu32_mask(k1, a, b, imm8) +#else + #define simde_mm256_mask_cmp_epu32_mask(k1, a, b, imm8) (k1) & simde_mm256_cmp_epu32_mask((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmp_epu32_mask +#define _mm256_mask_cmp_epu32_mask(a, b, imm8) simde_mm256_mask_cmp_epu32_mask((a), (b), (imm8)) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_cmp_epu32_mask (simde__m512i a, simde__m512i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + switch (imm8) { + case SIMDE_MM_CMPINT_EQ: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), (a_.u32 == b_.u32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] == b_.u32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), (a_.u32 < b_.u32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] < b_.u32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), (a_.u32 <= b_.u32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] <= b_.u32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_FALSE: + r_ = simde__m512i_to_private(simde_mm512_setzero_si512()); + break; + + + case SIMDE_MM_CMPINT_NE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), (a_.u32 != b_.u32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] != b_.u32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), ~(a_.u32 < b_.u32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = !(a_.u32[i] < b_.u32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), ~(a_.u32 <= b_.u32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = !(a_.u32[i] <= b_.u32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_TRUE: + r_ = simde__m512i_to_private(simde_x_mm512_setone_si512()); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde_mm512_movepi32_mask(simde__m512i_from_private(r_)); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_cmp_epu32_mask(a, b, imm8) _mm512_cmp_epu32_mask((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmp_epu32_mask + #define _mm512_cmp_epu32_mask(a, b, imm8) simde_mm512_cmp_epu32_mask((a), (b), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_cmp_epu32_mask(k1, a, b, imm8) _mm512_mask_cmp_epu32_mask(k1, a, b, imm8) +#else + #define simde_mm512_mask_cmp_epu32_mask(k1, a, b, imm8) (k1) & simde_mm512_cmp_epu32_mask(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmp_epu32_mask +#define _mm512_mask_cmp_epu32_mask(k1, a, b, imm8) simde_mm512_mask_cmp_epu32_mask((k1), (a), (b), (imm8)) +#endif + +SIMDE_HUGE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_cmp_epu64_mask (simde__m512i a, simde__m512i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + switch (imm8) { + case SIMDE_MM_CMPINT_EQ: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), (a_.u64 == b_.u64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] == b_.u64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), (a_.u64 < b_.u64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] < b_.u64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_LE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), (a_.u64 <= b_.u64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] <= b_.u64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_FALSE: + r_ = simde__m512i_to_private(simde_mm512_setzero_si512()); + break; + + + case SIMDE_MM_CMPINT_NE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), (a_.u64 != b_.u64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] != b_.u64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLT: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), ~(a_.u64 < b_.u64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = !(a_.u64[i] < b_.u64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_NLE: + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), ~(a_.u64 <= b_.u64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = !(a_.u64[i] <= b_.u64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + break; + + case SIMDE_MM_CMPINT_TRUE: + r_ = simde__m512i_to_private(simde_x_mm512_setone_si512()); + break; + + default: + HEDLEY_UNREACHABLE(); + } + + return simde_mm512_movepi64_mask(simde__m512i_from_private(r_)); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_cmp_epu64_mask(a, b, imm8) _mm512_cmp_epu64_mask((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmp_epu64_mask + #define _mm512_cmp_epu64_mask(a, b, imm8) simde_mm512_cmp_epu64_mask((a), (b), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_cmp_epu64_mask(k1, a, b, imm8) _mm512_mask_cmp_epu64_mask(k1, a, b, imm8) +#else + #define simde_mm512_mask_cmp_epu64_mask(k1, a, b, imm8) (k1) & simde_mm512_cmp_epu64_mask(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmp_epu64_mask +#define _mm512_mask_cmp_epu64_mask(k1, a, b, imm8) simde_mm512_mask_cmp_epu64_mask((k1), (a), (b), (imm8)) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_CMP_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpeq.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpeq.h new file mode 100644 index 00000000..41f90b3e --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpeq.h @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020-2021 Evan Nemerson + * 2020 Himanshi Mathur + */ + +#if !defined(SIMDE_X86_AVX512_CMPEQ_H) +#define SIMDE_X86_AVX512_CMPEQ_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" +#include "mov_mask.h" +#include "cmp.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_cmpeq_epi8_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmpeq_epi8_mask(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + simde__mmask64 r; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.m256i) / sizeof(a_.m256i[0])) ; i++) { + const uint32_t t = HEDLEY_STATIC_CAST(uint32_t, simde_mm256_movemask_epi8(simde_mm256_cmpeq_epi8(a_.m256i[i], b_.m256i[i]))); + r |= HEDLEY_STATIC_CAST(uint64_t, t) << (i * 32); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + simde__m512i_private tmp; + + tmp.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(tmp.i8), a_.i8 == b_.i8); + r = simde_mm512_movepi8_mask(simde__m512i_from_private(tmp)); + #else + r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0])) ; i++) { + r |= (a_.u8[i] == b_.u8[i]) ? (UINT64_C(1) << i) : 0; + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpeq_epi8_mask + #define _mm512_cmpeq_epi8_mask(a, b) simde_mm512_cmpeq_epi8_mask(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_mask_cmpeq_epi8_mask(simde__mmask64 k1, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_cmpeq_epi8_mask(k1, a, b); + #else + return simde_mm512_cmpeq_epi8_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmpeq_epi8_mask + #define _mm512_mask_cmpeq_epi8_mask(k1, a, b) simde_mm512_mask_cmpeq_epi8_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_cmpeq_epi32_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cmpeq_epi32_mask(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_cmpeq_epi32(a_.m256i[i], b_.m256i[i]); + } + + return simde_mm512_movepi32_mask(simde__m512i_from_private(r_)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpeq_epi32_mask + #define _mm512_cmpeq_epi32_mask(a, b) simde_mm512_cmpeq_epi32_mask(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_mask_cmpeq_epi32_mask (simde__mmask16 k1, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cmpeq_epi32_mask(k1, a, b); + #else + return simde_mm512_cmpeq_epi32_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmpeq_epi32_mask + #define _mm512_mask_cmpeq_epi32_mask(k1, a, b) simde_mm512_mask_cmpeq_epi32_mask(k1, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_cmpeq_epi64_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cmpeq_epi64_mask(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_cmpeq_epi64(a_.m256i[i], b_.m256i[i]); + } + + return simde_mm512_movepi64_mask(simde__m512i_from_private(r_)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpeq_epi64_mask + #define _mm512_cmpeq_epi64_mask(a, b) simde_mm512_cmpeq_epi64_mask(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_mask_cmpeq_epi64_mask (simde__mmask8 k1, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cmpeq_epi64_mask(k1, a, b); + #else + return simde_mm512_cmpeq_epi64_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmpeq_epi64_mask + #define _mm512_mask_cmpeq_epi64_mask(k1, a, b) simde_mm512_mask_cmpeq_epi64_mask(k1, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_cmpeq_epu16_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmpeq_epu16_mask(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + simde__mmask32 r; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + simde__m512i_private tmp; + + tmp.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(tmp.u16), a_.u16 == b_.u16); + r = simde_mm512_movepi16_mask(simde__m512i_from_private(tmp)); + #else + r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u16) / sizeof(a_.u16[0])) ; i++) { + r |= (a_.u16[i] == b_.u16[i]) ? (UINT16_C(1) << i) : 0; + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpeq_epu16_mask + #define _mm512_cmpeq_epu16_mask(a, b) simde_mm512_cmpeq_epu16_mask((a), (b)) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_mask_cmpeq_epu16_mask(simde__mmask32 k1, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_cmpeq_epu16_mask(k1, a, b); + #else + return k1 & simde_mm512_cmpeq_epu16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmpeq_epu16_mask + #define _mm512_mask_cmpeq_epu16_mask(k1, a, b) simde_mm512_mask_cmpeq_epu16_mask(k1, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_cmpeq_ps_mask (simde__m512 a, simde__m512 b) { + return simde_mm512_cmp_ps_mask(a, b, SIMDE_CMP_EQ_OQ); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpeq_ps_mask + #define _mm512_cmpeq_ps_mask(a, b) simde_mm512_cmp_ps_mask(a, b, SIMDE_CMP_EQ_OQ) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_cmpeq_pd_mask (simde__m512d a, simde__m512d b) { + return simde_mm512_cmp_pd_mask(a, b, SIMDE_CMP_EQ_OQ); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpeq_pd_mask + #define _mm512_cmpeq_pd_mask(a, b) simde_mm512_cmp_pd_mask(a, b, SIMDE_CMP_EQ_OQ) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_CMPEQ_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpge.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpge.h new file mode 100644 index 00000000..d0d42879 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpge.h @@ -0,0 +1,1434 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020-2021 Evan Nemerson + * 2020 Christopher Moore + * 2021 Andrew Rodriguez + */ + +#if !defined(SIMDE_X86_AVX512_CMPGE_H) +#define SIMDE_X86_AVX512_CMPGE_H + +#include "types.h" +#include "mov.h" +#include "mov_mask.h" +#include "movm.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmpge_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_movm_epi8(_mm_cmpge_epi8_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcgeq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_ge(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i8 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed char), vec_cmpge(a_.altivec_i8, b_.altivec_i8)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), a_.i8 >= b_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i8) / sizeof(a_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] >= b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_cmpge_epi8_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cmpge_epi8_mask(a, b); + #else + return simde_mm_movepi8_mask(simde_x_mm_cmpge_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpge_epi8_mask + #define _mm_cmpge_epi8_mask(a, b) simde_mm_cmpge_epi8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_mask_cmpge_epi8_mask(simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_cmpge_epi8_mask(k, a, b); + #else + return k & simde_mm_cmpge_epi8_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VBW_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpge_epi8_mask + #define _mm_mask_cmpge_epi8_mask(k, a, b) simde_mm_mask_cmpge_epi8_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmpge_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm256_movm_epi8(_mm256_cmpge_epi8_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epi8(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), a_.i8 >= b_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i8) / sizeof(a_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] >= b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_cmpge_epi8_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cmpge_epi8_mask(a, b); + #else + return simde_mm256_movepi8_mask(simde_x_mm256_cmpge_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VBW_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpge_epi8_mask + #define _mm256_cmpge_epi8_mask(a, b) simde_mm256_cmpge_epi8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_mask_cmpge_epi8_mask(simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_cmpge_epi8_mask(k, a, b); + #else + return k & simde_mm256_cmpge_epi8_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpge_epi8_mask + #define _mm256_mask_cmpge_epi8_mask(k, a, b) simde_mm256_mask_cmpge_epi8_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmpge_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm512_movm_epi8(_mm512_cmpge_epi8_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epi8(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmpge_epi8(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), a_.i8 >= b_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i8) / sizeof(a_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] >= b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_cmpge_epi8_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmpge_epi8_mask(a, b); + #else + return simde_mm512_movepi8_mask(simde_x_mm512_cmpge_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpge_epi8_mask + #define _mm512_cmpge_epi8_mask(a, b) simde_mm512_cmpge_epi8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_mask_cmpge_epi8_mask(simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_cmpge_epi8_mask(k, a, b); + #else + return k & simde_mm512_cmpge_epi8_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmpge_epi8_mask + #define _mm512_mask_cmpge_epi8_mask(k, a, b) simde_mm512_mask_cmpge_epi8_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmpge_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_movm_epi8(_mm_cmpge_epu8_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcgeq_u8(a_.neon_u8, b_.neon_u8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u8x16_ge(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u8 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), vec_cmpge(a_.altivec_u8, b_.altivec_u8)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u8), a_.u8 >= b_.u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] >= b_.u8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_cmpge_epu8_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cmpge_epu8_mask(a, b); + #else + return simde_mm_movepi8_mask(simde_x_mm_cmpge_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpge_epu8_mask + #define _mm_cmpge_epu8_mask(a, b) simde_mm_cmpge_epu8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_mask_cmpge_epu8_mask(simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_cmpge_epu8_mask(k, a, b); + #else + return k & simde_mm_cmpge_epu8_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpge_epu8_mask + #define _mm_mask_cmpge_epu8_mask(k, a, b) simde_mm_mask_cmpge_epu8_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmpge_epu8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm256_movm_epi8(_mm256_cmpge_epu8_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epu8(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u8), a_.u8 >= b_.u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] >= b_.u8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_cmpge_epu8_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cmpge_epu8_mask(a, b); + #else + return simde_mm256_movepi8_mask(simde_x_mm256_cmpge_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpge_epu8_mask + #define _mm256_cmpge_epu8_mask(a, b) simde_mm256_cmpge_epu8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_mask_cmpge_epu8_mask(simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_cmpge_epu8_mask(k, a, b); + #else + return k & simde_mm256_cmpge_epu8_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpge_epu8_mask + #define _mm256_mask_cmpge_epu8_mask(k, a, b) simde_mm256_mask_cmpge_epu8_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmpge_epu8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm512_movm_epi8(_mm512_cmpge_epu8_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epu8(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmpge_epu8(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u8), a_.u8 >= b_.u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] >= b_.u8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_cmpge_epu8_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmpge_epu8_mask(a, b); + #else + return simde_mm512_movepi8_mask(simde_x_mm512_cmpge_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpge_epu8_mask + #define _mm512_cmpge_epu8_mask(a, b) simde_mm512_cmpge_epu8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_mask_cmpge_epu8_mask(simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_cmpge_epu8_mask(k, a, b); + #else + return k & simde_mm512_cmpge_epu8_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmpge_epu8_mask + #define _mm512_mask_cmpge_epu8_mask(k, a, b) simde_mm512_mask_cmpge_epu8_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmpge_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_movm_epi16(_mm_cmpge_epi16_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vcgeq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_ge(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i16 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed short), vec_cmpge(a_.altivec_i16, b_.altivec_i16)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), a_.i16 >= b_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] >= b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmpge_epi16_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cmpge_epi16_mask(a, b); + #else + return simde_mm_movepi16_mask(simde_x_mm_cmpge_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpge_epi16_mask + #define _mm_cmpge_epi16_mask(a, b) simde_mm_cmpge_epi16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmpge_epi16_mask(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_cmpge_epi16_mask(k, a, b); + #else + return k & simde_mm_cmpge_epi16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpge_epi16_mask + #define _mm_mask_cmpge_epi16_mask(k, a, b) simde_mm_mask_cmpge_epi16_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmpge_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm256_movm_epi16(_mm256_cmpge_epi16_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epi16(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), a_.i16 >= b_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] >= b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm256_cmpge_epi16_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cmpge_epi16_mask(a, b); + #else + return simde_mm256_movepi16_mask(simde_x_mm256_cmpge_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpge_epi16_mask + #define _mm256_cmpge_epi16_mask(a, b) simde_mm256_cmpge_epi16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm256_mask_cmpge_epi16_mask(simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_cmpge_epi16_mask(k, a, b); + #else + return k & simde_mm256_cmpge_epi16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpge_epi16_mask + #define _mm256_mask_cmpge_epi16_mask(k, a, b) simde_mm256_mask_cmpge_epi16_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmpge_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm512_movm_epi16(_mm512_cmpge_epi16_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epi16(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmpge_epi16(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), a_.i16 >= b_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] >= b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_cmpge_epi16_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmpge_epi16_mask(a, b); + #else + return simde_mm512_movepi16_mask(simde_x_mm512_cmpge_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpge_epi16_mask + #define _mm512_cmpge_epi16_mask(a, b) simde_mm512_cmpge_epi16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_mask_cmpge_epi16_mask(simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_cmpge_epi16_mask(k, a, b); + #else + return k & simde_mm512_cmpge_epi16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmpge_epi16_mask + #define _mm512_mask_cmpge_epi16_mask(k, a, b) simde_mm512_mask_cmpge_epi16_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmpge_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_movm_epi16(_mm_cmpge_epu16_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vcgeq_u16(a_.neon_u16, b_.neon_u16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u16x8_ge(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u16 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned short), vec_cmpge(a_.altivec_u16, b_.altivec_u16)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), a_.u16 >= b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u16) / sizeof(a_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] >= b_.u16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmpge_epu16_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cmpge_epu16_mask(a, b); + #else + return simde_mm_movepi16_mask(simde_x_mm_cmpge_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpge_epu16_mask + #define _mm_cmpge_epu16_mask(a, b) simde_mm_cmpge_epu16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmpge_epu16_mask(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_cmpge_epu16_mask(k, a, b); + #else + return k & simde_mm_cmpge_epu16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpge_epu16_mask + #define _mm_mask_cmpge_epu16_mask(k, a, b) simde_mm_mask_cmpge_epu16_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmpge_epu16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm256_movm_epi16(_mm256_cmpge_epu16_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epu16(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), a_.u16 >= b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u16) / sizeof(a_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] >= b_.u16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm256_cmpge_epu16_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cmpge_epu16_mask(a, b); + #else + return simde_mm256_movepi16_mask(simde_x_mm256_cmpge_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpge_epu16_mask + #define _mm256_cmpge_epu16_mask(a, b) simde_mm256_cmpge_epu16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm256_mask_cmpge_epu16_mask(simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_cmpge_epu16_mask(k, a, b); + #else + return k & simde_mm256_cmpge_epu16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpge_epu16_mask + #define _mm256_mask_cmpge_epu16_mask(k, a, b) simde_mm256_mask_cmpge_epu16_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmpge_epu16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm512_movm_epi16(_mm512_cmpge_epu16_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epu16(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmpge_epu16(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), a_.u16 >= b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u16) / sizeof(a_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] >= b_.u16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_cmpge_epu16_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmpge_epu16_mask(a, b); + #else + return simde_mm512_movepi16_mask(simde_x_mm512_cmpge_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpge_epu16_mask + #define _mm512_cmpge_epu16_mask(a, b) simde_mm512_cmpge_epu16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_mask_cmpge_epu16_mask(simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_cmpge_epu16_mask(k, a, b); + #else + return k & simde_mm512_cmpge_epu16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmpge_epu16_mask + #define _mm512_mask_cmpge_epu16_mask(k, a, b) simde_mm512_mask_cmpge_epu16_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmpge_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm_movm_epi32(_mm_cmpge_epi32_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcgeq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_ge(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed int), vec_cmpge(a_.altivec_i32, b_.altivec_i32)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.i32 >= b_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] >= b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmpge_epi32_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_cmpge_epi32_mask(a, b); + #else + return simde_mm_movepi32_mask(simde_x_mm_cmpge_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpge_epi32_mask + #define _mm_cmpge_epi32_mask(a, b) simde_mm_cmpge_epi32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmpge_epi32_mask(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_cmpge_epi32_mask(k, a, b); + #else + return k & simde_mm_cmpge_epi32_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpge_epi32_mask + #define _mm_mask_cmpge_epi32_mask(k, a, b) simde_mm_mask_cmpge_epi32_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmpge_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm256_movm_epi32(_mm256_cmpge_epi32_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epi32(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.i32 >= b_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] >= b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_cmpge_epi32_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_cmpge_epi32_mask(a, b); + #else + return simde_mm256_movepi32_mask(simde_x_mm256_cmpge_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpge_epi32_mask + #define _mm256_cmpge_epi32_mask(a, b) simde_mm256_cmpge_epi32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_mask_cmpge_epi32_mask(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_cmpge_epi32_mask(k, a, b); + #else + return k & simde_mm256_cmpge_epi32_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpge_epi32_mask + #define _mm256_mask_cmpge_epi32_mask(k, a, b) simde_mm256_mask_cmpge_epi32_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmpge_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return simde_mm512_movm_epi32(_mm512_cmpge_epi32_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epi32(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmpge_epi32(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.i32 >= b_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] >= b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_cmpge_epi32_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cmpge_epi32_mask(a, b); + #else + return simde_mm512_movepi32_mask(simde_x_mm512_cmpge_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpge_epi32_mask + #define _mm512_cmpge_epi32_mask(a, b) simde_mm512_cmpge_epi32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_mask_cmpge_epi32_mask(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cmpge_epi32_mask(k, a, b); + #else + return k & simde_mm512_cmpge_epi32_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmpge_epi32_mask + #define _mm512_mask_cmpge_epi32_mask(k, a, b) simde_mm512_mask_cmpge_epi32_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmpge_epu32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm_movm_epi32(_mm_cmpge_epu32_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcgeq_u32(a_.neon_u32, b_.neon_u32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u32x4_ge(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned int), vec_cmpge(a_.altivec_u32, b_.altivec_u32)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), a_.u32 >= b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] >= b_.u32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmpge_epu32_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_cmpge_epu32_mask(a, b); + #else + return simde_mm_movepi32_mask(simde_x_mm_cmpge_epu32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpge_epu32_mask + #define _mm_cmpge_epu32_mask(a, b) simde_mm_cmpge_epu32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmpge_epu32_mask(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_cmpge_epu32_mask(k, a, b); + #else + return k & simde_mm_cmpge_epu32_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpge_epu32_mask + #define _mm_mask_cmpge_epu32_mask(k, a, b) simde_mm_mask_cmpge_epu32_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmpge_epu32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm256_movm_epi32(_mm256_cmpge_epu32_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epu32(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), a_.u32 >= b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] >= b_.u32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_cmpge_epu32_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_cmpge_epu32_mask(a, b); + #else + return simde_mm256_movepi32_mask(simde_x_mm256_cmpge_epu32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpge_epu32_mask + #define _mm256_cmpge_epu32_mask(a, b) simde_mm256_cmpge_epu32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_mask_cmpge_epu32_mask(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_cmpge_epu32_mask(k, a, b); + #else + return k & simde_mm256_cmpge_epu32_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpge_epu32_mask + #define _mm256_mask_cmpge_epu32_mask(k, a, b) simde_mm256_mask_cmpge_epu32_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmpge_epu32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return simde_mm512_movm_epi32(_mm512_cmpge_epu32_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epu32(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmpge_epu32(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), a_.u32 >= b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] >= b_.u32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_cmpge_epu32_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cmpge_epu32_mask(a, b); + #else + return simde_mm512_movepi32_mask(simde_x_mm512_cmpge_epu32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpge_epu32_mask + #define _mm512_cmpge_epu32_mask(a, b) simde_mm512_cmpge_epu32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_mask_cmpge_epu32_mask(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cmpge_epu32_mask(k, a, b); + #else + return k & simde_mm512_cmpge_epu32_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmpge_epu32_mask + #define _mm512_mask_cmpge_epu32_mask(k, a, b) simde_mm512_mask_cmpge_epu32_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmpge_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm_movm_epi64(_mm_cmpge_epi64_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcgeq_s64(a_.neon_i64, b_.neon_i64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_ge(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed long long), vec_cmpge(a_.altivec_i64, b_.altivec_i64)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 >= b_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] >= b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmpge_epi64_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_cmpge_epi64_mask(a, b); + #else + return simde_mm_movepi64_mask(simde_x_mm_cmpge_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpge_epi64_mask + #define _mm_cmpge_epi64_mask(a, b) simde_mm_cmpge_epi64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmpge_epi64_mask(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_cmpge_epi64_mask(k, a, b); + #else + return k & simde_mm_cmpge_epi64_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpge_epi64_mask + #define _mm_mask_cmpge_epi64_mask(k, a, b) simde_mm_mask_cmpge_epi64_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmpge_epi64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm256_movm_epi64(_mm256_cmpge_epi64_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epi64(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 >= b_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] >= b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_cmpge_epi64_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_cmpge_epi64_mask(a, b); + #else + return simde_mm256_movepi64_mask(simde_x_mm256_cmpge_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpge_epi64_mask + #define _mm256_cmpge_epi64_mask(a, b) simde_mm256_cmpge_epi64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_mask_cmpge_epi64_mask(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_cmpge_epi64_mask(k, a, b); + #else + return k & simde_mm256_cmpge_epi64_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpge_epi64_mask + #define _mm256_mask_cmpge_epi64_mask(k, a, b) simde_mm256_mask_cmpge_epi64_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmpge_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return simde_mm512_movm_epi64(_mm512_cmpge_epi64_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epi64(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmpge_epi64(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 >= b_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] >= b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_cmpge_epi64_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cmpge_epi64_mask(a, b); + #else + return simde_mm512_movepi64_mask(simde_x_mm512_cmpge_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpge_epi64_mask + #define _mm512_cmpge_epi64_mask(a, b) simde_mm512_cmpge_epi64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_mask_cmpge_epi64_mask(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cmpge_epi64_mask(k, a, b); + #else + return k & simde_mm512_cmpge_epi64_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmpge_epi64_mask + #define _mm512_mask_cmpge_epi64_mask(k, a, b) simde_mm512_mask_cmpge_epi64_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmpge_epu64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm_movm_epi64(_mm_cmpge_epu64_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcgeq_u64(a_.neon_u64, b_.neon_u64); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long), vec_cmpge(a_.altivec_u64, b_.altivec_u64)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), a_.u64 >= b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] >= b_.u64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmpge_epu64_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_cmpge_epu64_mask(a, b); + #else + return simde_mm_movepi64_mask(simde_x_mm_cmpge_epu64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpge_epu64_mask + #define _mm_cmpge_epu64_mask(a, b) simde_mm_cmpge_epu64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmpge_epu64_mask(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_cmpge_epu64_mask(k, a, b); + #else + return k & simde_mm_cmpge_epu64_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpge_epu64_mask + #define _mm_mask_cmpge_epu64_mask(k, a, b) simde_mm_mask_cmpge_epu64_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmpge_epu64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm256_movm_epi64(_mm256_cmpge_epu64_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epu64(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), a_.u64 >= b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] >= b_.u64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_cmpge_epu64_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_cmpge_epu64_mask(a, b); + #else + return simde_mm256_movepi64_mask(simde_x_mm256_cmpge_epu64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpge_epu64_mask + #define _mm256_cmpge_epu64_mask(a, b) simde_mm256_cmpge_epu64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_mask_cmpge_epu64_mask(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_cmpge_epu64_mask(k, a, b); + #else + return k & simde_mm256_cmpge_epu64_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpge_epu64_mask + #define _mm256_mask_cmpge_epu64_mask(k, a, b) simde_mm256_mask_cmpge_epu64_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmpge_epu64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm512_movm_epi64(_mm512_cmpge_epu64_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmpge_epu64(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmpge_epu64(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), a_.u64 >= b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] >= b_.u64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_cmpge_epu64_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cmpge_epu64_mask(a, b); + #else + return simde_mm512_movepi64_mask(simde_x_mm512_cmpge_epu64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpge_epu64_mask + #define _mm512_cmpge_epu64_mask(a, b) simde_mm512_cmpge_epu64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_mask_cmpge_epu64_mask(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cmpge_epu64_mask(k, a, b); + #else + return k & simde_mm512_cmpge_epu64_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmpge_epu64_mask + #define _mm512_mask_cmpge_epu64_mask(k, a, b) simde_mm512_mask_cmpge_epu64_mask((k), (a), (b)) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_CMPGE_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpgt.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpgt.h new file mode 100644 index 00000000..15245f96 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpgt.h @@ -0,0 +1,212 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Christopher Moore + */ + +#if !defined(SIMDE_X86_AVX512_CMPGT_H) +#define SIMDE_X86_AVX512_CMPGT_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" +#include "mov_mask.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_cmpgt_epi8_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmpgt_epi8_mask(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + simde__mmask64 r; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) && !defined(HEDLEY_INTEL_VERSION) + r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.m256i) / sizeof(a_.m256i[0])) ; i++) { + const uint32_t t = HEDLEY_STATIC_CAST(uint32_t, simde_mm256_movemask_epi8(simde_mm256_cmpgt_epi8(a_.m256i[i], b_.m256i[i]))); + r |= HEDLEY_STATIC_CAST(uint64_t, t) << HEDLEY_STATIC_CAST(uint64_t, i * 32); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + simde__m512i_private tmp; + + tmp.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(tmp.i8), a_.i8 > b_.i8); + r = simde_mm512_movepi8_mask(simde__m512i_from_private(tmp)); + #else + r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i8) / sizeof(a_.i8[0])) ; i++) { + r |= (a_.i8[i] > b_.i8[i]) ? (UINT64_C(1) << i) : 0; + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpgt_epi8_mask + #define _mm512_cmpgt_epi8_mask(a, b) simde_mm512_cmpgt_epi8_mask(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_cmpgt_epu8_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmpgt_epu8_mask(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + simde__mmask64 r = 0; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + simde__m512i_private tmp; + + tmp.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(tmp.i8), a_.u8 > b_.u8); + r = simde_mm512_movepi8_mask(simde__m512i_from_private(tmp)); + #else + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0])) ; i++) { + r |= (a_.u8[i] > b_.u8[i]) ? (UINT64_C(1) << i) : 0; + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpgt_epu8_mask + #define _mm512_cmpgt_epu8_mask(a, b) simde_mm512_cmpgt_epu8_mask(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_cmpgt_epi16_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmpgt_epi16_mask(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_cmpgt_epi16(a_.m256i[i], b_.m256i[i]); + } + + return simde_mm512_movepi16_mask(simde__m512i_from_private(r_)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpgt_epi16_mask + #define _mm512_cmpgt_epi16_mask(a, b) simde_mm512_cmpgt_epi16_mask(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_cmpgt_epi32_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cmpgt_epi32_mask(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_cmpgt_epi32(a_.m256i[i], b_.m256i[i]); + } + + return simde_mm512_movepi32_mask(simde__m512i_from_private(r_)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpgt_epi32_mask + #define _mm512_cmpgt_epi32_mask(a, b) simde_mm512_cmpgt_epi32_mask(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_mask_cmpgt_epi32_mask (simde__mmask16 k1, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cmpgt_epi32_mask(k1, a, b); + #else + return simde_mm512_cmpgt_epi32_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmpgt_epi32_mask + #define _mm512_mask_cmpgt_epi32_mask(k1, a, b) simde_mm512_mask_cmpgt_epi32_mask(k1, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_cmpgt_epi64_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cmpgt_epi64_mask(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_cmpgt_epi64(a_.m256i[i], b_.m256i[i]); + } + + return simde_mm512_movepi64_mask(simde__m512i_from_private(r_)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmpgt_epi64_mask + #define _mm512_cmpgt_epi64_mask(a, b) simde_mm512_cmpgt_epi64_mask(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_mask_cmpgt_epi64_mask (simde__mmask8 k1, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cmpgt_epi64_mask(k1, a, b); + #else + return simde_mm512_cmpgt_epi64_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmpgt_epi64_mask + #define _mm512_mask_cmpgt_epi64_mask(k1, a, b) simde_mm512_mask_cmpgt_epi64_mask(k1, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_CMPGT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmple.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmple.h new file mode 100644 index 00000000..9b3c3aad --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmple.h @@ -0,0 +1,1432 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020-2021 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_CMPLE_H) +#define SIMDE_X86_AVX512_CMPLE_H + +#include "types.h" +#include "mov.h" +#include "mov_mask.h" +#include "movm.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmple_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_movm_epi8(_mm_cmple_epi8_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcleq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_le(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i8 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed char), vec_cmple(a_.altivec_i8, b_.altivec_i8)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), a_.i8 <= b_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i8) / sizeof(a_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] <= b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_cmple_epi8_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cmple_epi8_mask(a, b); + #else + return simde_mm_movepi8_mask(simde_x_mm_cmple_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_cmple_epi8_mask + #define _mm_cmple_epi8_mask(a, b) simde_mm_cmple_epi8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_mask_cmple_epi8_mask(simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_cmple_epi8_mask(k, a, b); + #else + return k & simde_mm_cmple_epi8_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VBW_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmple_epi8_mask + #define _mm_mask_cmple_epi8_mask(k, a, b) simde_mm_mask_cmple_epi8_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmple_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm256_movm_epi8(_mm256_cmple_epi8_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epi8(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), a_.i8 <= b_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i8) / sizeof(a_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] <= b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_cmple_epi8_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cmple_epi8_mask(a, b); + #else + return simde_mm256_movepi8_mask(simde_x_mm256_cmple_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VBW_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmple_epi8_mask + #define _mm256_cmple_epi8_mask(a, b) simde_mm256_cmple_epi8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_mask_cmple_epi8_mask(simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_cmple_epi8_mask(k, a, b); + #else + return k & simde_mm256_cmple_epi8_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmple_epi8_mask + #define _mm256_mask_cmple_epi8_mask(k, a, b) simde_mm256_mask_cmple_epi8_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmple_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm512_movm_epi8(_mm512_cmple_epi8_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epi8(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmple_epi8(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), a_.i8 <= b_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i8) / sizeof(a_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] <= b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_cmple_epi8_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmple_epi8_mask(a, b); + #else + return simde_mm512_movepi8_mask(simde_x_mm512_cmple_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmple_epi8_mask + #define _mm512_cmple_epi8_mask(a, b) simde_mm512_cmple_epi8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_mask_cmple_epi8_mask(simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_cmple_epi8_mask(k, a, b); + #else + return k & simde_mm512_cmple_epi8_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmple_epi8_mask + #define _mm512_mask_cmple_epi8_mask(k, a, b) simde_mm512_mask_cmple_epi8_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmple_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_movm_epi8(_mm_cmple_epu8_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcleq_u8(a_.neon_u8, b_.neon_u8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u8x16_le(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u8 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), vec_cmple(a_.altivec_u8, b_.altivec_u8)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u8), a_.u8 <= b_.u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] <= b_.u8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_cmple_epu8_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cmple_epu8_mask(a, b); + #else + return simde_mm_movepi8_mask(simde_x_mm_cmple_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_cmple_epu8_mask + #define _mm_cmple_epu8_mask(a, b) simde_mm_cmple_epu8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_mask_cmple_epu8_mask(simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_cmple_epu8_mask(k, a, b); + #else + return k & simde_mm_cmple_epu8_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmple_epu8_mask + #define _mm_mask_cmple_epu8_mask(k, a, b) simde_mm_mask_cmple_epu8_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmple_epu8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm256_movm_epi8(_mm256_cmple_epu8_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epu8(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u8), a_.u8 <= b_.u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] <= b_.u8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_cmple_epu8_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cmple_epu8_mask(a, b); + #else + return simde_mm256_movepi8_mask(simde_x_mm256_cmple_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmple_epu8_mask + #define _mm256_cmple_epu8_mask(a, b) simde_mm256_cmple_epu8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_mask_cmple_epu8_mask(simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_cmple_epu8_mask(k, a, b); + #else + return k & simde_mm256_cmple_epu8_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmple_epu8_mask + #define _mm256_mask_cmple_epu8_mask(k, a, b) simde_mm256_mask_cmple_epu8_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmple_epu8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm512_movm_epi8(_mm512_cmple_epu8_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epu8(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmple_epu8(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u8), a_.u8 <= b_.u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] <= b_.u8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_cmple_epu8_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmple_epu8_mask(a, b); + #else + return simde_mm512_movepi8_mask(simde_x_mm512_cmple_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmple_epu8_mask + #define _mm512_cmple_epu8_mask(a, b) simde_mm512_cmple_epu8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_mask_cmple_epu8_mask(simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_cmple_epu8_mask(k, a, b); + #else + return k & simde_mm512_cmple_epu8_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmple_epu8_mask + #define _mm512_mask_cmple_epu8_mask(k, a, b) simde_mm512_mask_cmple_epu8_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmple_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_movm_epi16(_mm_cmple_epi16_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vcleq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_le(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i16 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed short), vec_cmple(a_.altivec_i16, b_.altivec_i16)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), a_.i16 <= b_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] <= b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmple_epi16_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cmple_epi16_mask(a, b); + #else + return simde_mm_movepi16_mask(simde_x_mm_cmple_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_cmple_epi16_mask + #define _mm_cmple_epi16_mask(a, b) simde_mm_cmple_epi16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmple_epi16_mask(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_cmple_epi16_mask(k, a, b); + #else + return k & simde_mm_cmple_epi16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmple_epi16_mask + #define _mm_mask_cmple_epi16_mask(k, a, b) simde_mm_mask_cmple_epi16_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmple_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm256_movm_epi16(_mm256_cmple_epi16_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epi16(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), a_.i16 <= b_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] <= b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm256_cmple_epi16_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cmple_epi16_mask(a, b); + #else + return simde_mm256_movepi16_mask(simde_x_mm256_cmple_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmple_epi16_mask + #define _mm256_cmple_epi16_mask(a, b) simde_mm256_cmple_epi16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm256_mask_cmple_epi16_mask(simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_cmple_epi16_mask(k, a, b); + #else + return k & simde_mm256_cmple_epi16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmple_epi16_mask + #define _mm256_mask_cmple_epi16_mask(k, a, b) simde_mm256_mask_cmple_epi16_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmple_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm512_movm_epi16(_mm512_cmple_epi16_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epi16(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmple_epi16(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), a_.i16 <= b_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] <= b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_cmple_epi16_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmple_epi16_mask(a, b); + #else + return simde_mm512_movepi16_mask(simde_x_mm512_cmple_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmple_epi16_mask + #define _mm512_cmple_epi16_mask(a, b) simde_mm512_cmple_epi16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_mask_cmple_epi16_mask(simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_cmple_epi16_mask(k, a, b); + #else + return k & simde_mm512_cmple_epi16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmple_epi16_mask + #define _mm512_mask_cmple_epi16_mask(k, a, b) simde_mm512_mask_cmple_epi16_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmple_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_movm_epi16(_mm_cmple_epu16_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vcleq_u16(a_.neon_u16, b_.neon_u16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u16x8_le(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u16 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned short), vec_cmple(a_.altivec_u16, b_.altivec_u16)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), a_.u16 <= b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u16) / sizeof(a_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] <= b_.u16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmple_epu16_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cmple_epu16_mask(a, b); + #else + return simde_mm_movepi16_mask(simde_x_mm_cmple_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_cmple_epu16_mask + #define _mm_cmple_epu16_mask(a, b) simde_mm_cmple_epu16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmple_epu16_mask(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_cmple_epu16_mask(k, a, b); + #else + return k & simde_mm_cmple_epu16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmple_epu16_mask + #define _mm_mask_cmple_epu16_mask(k, a, b) simde_mm_mask_cmple_epu16_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmple_epu16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm256_movm_epi16(_mm256_cmple_epu16_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epu16(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), a_.u16 <= b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u16) / sizeof(a_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] <= b_.u16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm256_cmple_epu16_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cmple_epu16_mask(a, b); + #else + return simde_mm256_movepi16_mask(simde_x_mm256_cmple_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmple_epu16_mask + #define _mm256_cmple_epu16_mask(a, b) simde_mm256_cmple_epu16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm256_mask_cmple_epu16_mask(simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_cmple_epu16_mask(k, a, b); + #else + return k & simde_mm256_cmple_epu16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmple_epu16_mask + #define _mm256_mask_cmple_epu16_mask(k, a, b) simde_mm256_mask_cmple_epu16_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmple_epu16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return simde_mm512_movm_epi16(_mm512_cmple_epu16_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epu16(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmple_epu16(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), a_.u16 <= b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u16) / sizeof(a_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] <= b_.u16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_cmple_epu16_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmple_epu16_mask(a, b); + #else + return simde_mm512_movepi16_mask(simde_x_mm512_cmple_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmple_epu16_mask + #define _mm512_cmple_epu16_mask(a, b) simde_mm512_cmple_epu16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_mask_cmple_epu16_mask(simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_cmple_epu16_mask(k, a, b); + #else + return k & simde_mm512_cmple_epu16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmple_epu16_mask + #define _mm512_mask_cmple_epu16_mask(k, a, b) simde_mm512_mask_cmple_epu16_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmple_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm_movm_epi32(_mm_cmple_epi32_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcleq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_le(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed int), vec_cmple(a_.altivec_i32, b_.altivec_i32)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.i32 <= b_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] <= b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmple_epi32_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_cmple_epi32_mask(a, b); + #else + return simde_mm_movepi32_mask(simde_x_mm_cmple_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cmple_epi32_mask + #define _mm_cmple_epi32_mask(a, b) simde_mm_cmple_epi32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmple_epi32_mask(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_cmple_epi32_mask(k, a, b); + #else + return k & simde_mm_cmple_epi32_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmple_epi32_mask + #define _mm_mask_cmple_epi32_mask(k, a, b) simde_mm_mask_cmple_epi32_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmple_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm256_movm_epi32(_mm256_cmple_epi32_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epi32(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.i32 <= b_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] <= b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_cmple_epi32_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_cmple_epi32_mask(a, b); + #else + return simde_mm256_movepi32_mask(simde_x_mm256_cmple_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmple_epi32_mask + #define _mm256_cmple_epi32_mask(a, b) simde_mm256_cmple_epi32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_mask_cmple_epi32_mask(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_cmple_epi32_mask(k, a, b); + #else + return k & simde_mm256_cmple_epi32_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmple_epi32_mask + #define _mm256_mask_cmple_epi32_mask(k, a, b) simde_mm256_mask_cmple_epi32_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmple_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return simde_mm512_movm_epi32(_mm512_cmple_epi32_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epi32(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmple_epi32(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.i32 <= b_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] <= b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_cmple_epi32_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cmple_epi32_mask(a, b); + #else + return simde_mm512_movepi32_mask(simde_x_mm512_cmple_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmple_epi32_mask + #define _mm512_cmple_epi32_mask(a, b) simde_mm512_cmple_epi32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_mask_cmple_epi32_mask(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cmple_epi32_mask(k, a, b); + #else + return k & simde_mm512_cmple_epi32_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmple_epi32_mask + #define _mm512_mask_cmple_epi32_mask(k, a, b) simde_mm512_mask_cmple_epi32_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmple_epu32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm_movm_epi32(_mm_cmple_epu32_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcleq_u32(a_.neon_u32, b_.neon_u32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u32x4_le(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned int), vec_cmple(a_.altivec_u32, b_.altivec_u32)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), a_.u32 <= b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] <= b_.u32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmple_epu32_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_cmple_epu32_mask(a, b); + #else + return simde_mm_movepi32_mask(simde_x_mm_cmple_epu32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cmple_epu32_mask + #define _mm_cmple_epu32_mask(a, b) simde_mm_cmple_epu32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmple_epu32_mask(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_cmple_epu32_mask(k, a, b); + #else + return k & simde_mm_cmple_epu32_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmple_epu32_mask + #define _mm_mask_cmple_epu32_mask(k, a, b) simde_mm_mask_cmple_epu32_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmple_epu32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm256_movm_epi32(_mm256_cmple_epu32_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epu32(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), a_.u32 <= b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] <= b_.u32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_cmple_epu32_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_cmple_epu32_mask(a, b); + #else + return simde_mm256_movepi32_mask(simde_x_mm256_cmple_epu32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmple_epu32_mask + #define _mm256_cmple_epu32_mask(a, b) simde_mm256_cmple_epu32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_mask_cmple_epu32_mask(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_cmple_epu32_mask(k, a, b); + #else + return k & simde_mm256_cmple_epu32_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmple_epu32_mask + #define _mm256_mask_cmple_epu32_mask(k, a, b) simde_mm256_mask_cmple_epu32_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmple_epu32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return simde_mm512_movm_epi32(_mm512_cmple_epu32_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epu32(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmple_epu32(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), a_.u32 <= b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] <= b_.u32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_cmple_epu32_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cmple_epu32_mask(a, b); + #else + return simde_mm512_movepi32_mask(simde_x_mm512_cmple_epu32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmple_epu32_mask + #define _mm512_cmple_epu32_mask(a, b) simde_mm512_cmple_epu32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_mask_cmple_epu32_mask(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cmple_epu32_mask(k, a, b); + #else + return k & simde_mm512_cmple_epu32_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmple_epu32_mask + #define _mm512_mask_cmple_epu32_mask(k, a, b) simde_mm512_mask_cmple_epu32_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmple_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm_movm_epi64(_mm_cmple_epi64_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcleq_s64(a_.neon_i64, b_.neon_i64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_le(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed long long), vec_cmple(a_.altivec_i64, b_.altivec_i64)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 <= b_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] <= b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmple_epi64_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_cmple_epi64_mask(a, b); + #else + return simde_mm_movepi64_mask(simde_x_mm_cmple_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cmple_epi64_mask + #define _mm_cmple_epi64_mask(a, b) simde_mm_cmple_epi64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmple_epi64_mask(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_cmple_epi64_mask(k, a, b); + #else + return k & simde_mm_cmple_epi64_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmple_epi64_mask + #define _mm_mask_cmple_epi64_mask(k, a, b) simde_mm_mask_cmple_epi64_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmple_epi64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm256_movm_epi64(_mm256_cmple_epi64_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epi64(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 <= b_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] <= b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_cmple_epi64_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_cmple_epi64_mask(a, b); + #else + return simde_mm256_movepi64_mask(simde_x_mm256_cmple_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmple_epi64_mask + #define _mm256_cmple_epi64_mask(a, b) simde_mm256_cmple_epi64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_mask_cmple_epi64_mask(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_cmple_epi64_mask(k, a, b); + #else + return k & simde_mm256_cmple_epi64_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmple_epi64_mask + #define _mm256_mask_cmple_epi64_mask(k, a, b) simde_mm256_mask_cmple_epi64_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmple_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return simde_mm512_movm_epi64(_mm512_cmple_epi64_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epi64(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmple_epi64(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 <= b_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] <= b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_cmple_epi64_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cmple_epi64_mask(a, b); + #else + return simde_mm512_movepi64_mask(simde_x_mm512_cmple_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmple_epi64_mask + #define _mm512_cmple_epi64_mask(a, b) simde_mm512_cmple_epi64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_mask_cmple_epi64_mask(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cmple_epi64_mask(k, a, b); + #else + return k & simde_mm512_cmple_epi64_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmple_epi64_mask + #define _mm512_mask_cmple_epi64_mask(k, a, b) simde_mm512_mask_cmple_epi64_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cmple_epu64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm_movm_epi64(_mm_cmple_epu64_mask(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcleq_u64(a_.neon_u64, b_.neon_u64); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long), vec_cmple(a_.altivec_u64, b_.altivec_u64)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), a_.u64 <= b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] <= b_.u64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmple_epu64_mask (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_cmple_epu64_mask(a, b); + #else + return simde_mm_movepi64_mask(simde_x_mm_cmple_epu64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cmple_epu64_mask + #define _mm_cmple_epu64_mask(a, b) simde_mm_cmple_epu64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmple_epu64_mask(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_cmple_epu64_mask(k, a, b); + #else + return k & simde_mm_cmple_epu64_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmple_epu64_mask + #define _mm_mask_cmple_epu64_mask(k, a, b) simde_mm_mask_cmple_epu64_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_cmple_epu64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return simde_mm256_movm_epi64(_mm256_cmple_epu64_mask(a, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epu64(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), a_.u64 <= b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] <= b_.u64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_cmple_epu64_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_cmple_epu64_mask(a, b); + #else + return simde_mm256_movepi64_mask(simde_x_mm256_cmple_epu64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmple_epu64_mask + #define _mm256_cmple_epu64_mask(a, b) simde_mm256_cmple_epu64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_mask_cmple_epu64_mask(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_cmple_epu64_mask(k, a, b); + #else + return k & simde_mm256_cmple_epu64_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmple_epu64_mask + #define _mm256_mask_cmple_epu64_mask(k, a, b) simde_mm256_mask_cmple_epu64_mask((k), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_cmple_epu64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return simde_mm512_movm_epi64(_mm512_cmple_epu64_mask(a, b)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_cmple_epu64(a_.m128i[i], b_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_cmple_epu64(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), a_.u64 <= b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] <= b_.u64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_cmple_epu64_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cmple_epu64_mask(a, b); + #else + return simde_mm512_movepi64_mask(simde_x_mm512_cmple_epu64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmple_epu64_mask + #define _mm512_cmple_epu64_mask(a, b) simde_mm512_cmple_epu64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_mask_cmple_epu64_mask(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cmple_epu64_mask(k, a, b); + #else + return k & simde_mm512_cmple_epu64_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cmple_epu64_mask + #define _mm512_mask_cmple_epu64_mask(k, a, b) simde_mm512_mask_cmple_epu64_mask((k), (a), (b)) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_CMPLE_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmplt.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmplt.h new file mode 100644 index 00000000..550e9015 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmplt.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_CMPLT_H) +#define SIMDE_X86_AVX512_CMPLT_H + +#include "types.h" +#include "mov.h" +#include "cmp.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_cmplt_ps_mask (simde__m512 a, simde__m512 b) { + return simde_mm512_cmp_ps_mask(a, b, SIMDE_CMP_LT_OQ); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmplt_ps_mask + #define _mm512_cmplt_ps_mask(a, b) simde_mm512_cmp_ps_mask(a, b, SIMDE_CMP_LT_OQ) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_cmplt_pd_mask (simde__m512d a, simde__m512d b) { + return simde_mm512_cmp_pd_mask(a, b, SIMDE_CMP_LT_OQ); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmplt_pd_mask + #define _mm512_cmplt_pd_mask(a, b) simde_mm512_cmp_pd_mask(a, b, SIMDE_CMP_LT_OQ) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_cmplt_epi8_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmplt_epi8_mask(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + simde__mmask64 r = 0; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + simde__m512i_private tmp; + + tmp.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(tmp.i8), a_.i8 < b_.i8); + r = simde_mm512_movepi8_mask(simde__m512i_from_private(tmp)); + #else + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i8) / sizeof(a_.i8[0])) ; i++) { + r |= (a_.i8[i] < b_.i8[i]) ? (UINT64_C(1) << i) : 0; + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmplt_epi8_mask + #define _mm512_cmplt_epi8_mask(a, b) simde_mm512_cmplt_epi8_mask(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_cmplt_epu8_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cmplt_epu8_mask(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + simde__mmask64 r = 0; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + simde__m512i_private tmp; + + tmp.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(tmp.i8), a_.u8 < b_.u8); + r = simde_mm512_movepi8_mask(simde__m512i_from_private(tmp)); + #else + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0])) ; i++) { + r |= (a_.u8[i] < b_.u8[i]) ? (UINT64_C(1) << i) : 0; + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cmplt_epu8_mask + #define _mm512_cmplt_epu8_mask(a, b) simde_mm512_cmplt_epu8_mask(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_CMPLT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpneq.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpneq.h new file mode 100644 index 00000000..6e9bf336 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cmpneq.h @@ -0,0 +1,490 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2021 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_CMPNEQ_H) +#define SIMDE_X86_AVX512_CMPNEQ_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" +#include "mov_mask.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_cmpneq_epi8_mask(simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cmpneq_epi8_mask(a, b); + #else + return ~simde_mm_movepi8_mask(simde_mm_cmpeq_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpneq_epi8_mask + #define _mm_cmpneq_epi8_mask(a, b) simde_mm_cmpneq_epi8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_mask_cmpneq_epi8_mask(simde__mmask16 k1, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_cmpneq_epi8_mask(k1, a, b); + #else + return simde_mm_cmpneq_epi8_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpneq_epi8_mask + #define _mm_mask_cmpneq_epi8_mask(k1, a, b) simde_mm_mask_cmpneq_epi8_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_cmpneq_epu8_mask(simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cmpneq_epu8_mask(a, b); + #else + return simde_mm_cmpneq_epi8_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpneq_epu8_mask + #define _mm_cmpneq_epu8_mask(a, b) simde_mm_cmpneq_epu8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_mask_cmpneq_epu8_mask(simde__mmask16 k1, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_cmpneq_epu8_mask(k1, a, b); + #else + return simde_mm_mask_cmpneq_epi8_mask(k1, a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpneq_epu8_mask + #define _mm_mask_cmpneq_epu8_mask(k1, a, b) simde_mm_mask_cmpneq_epu8_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmpneq_epi16_mask(simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cmpneq_epi16_mask(a, b); + #else + return ~simde_mm_movepi16_mask(simde_mm_cmpeq_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpneq_epi16_mask + #define _mm_cmpneq_epi16_mask(a, b) simde_mm_cmpneq_epi16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmpneq_epi16_mask(simde__mmask8 k1, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_cmpneq_epi16_mask(k1, a, b); + #else + return simde_mm_cmpneq_epi16_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpneq_epi16_mask + #define _mm_mask_cmpneq_epi16_mask(k1, a, b) simde_mm_mask_cmpneq_epi16_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmpneq_epu16_mask(simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cmpneq_epu16_mask(a, b); + #else + return simde_mm_cmpneq_epi16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpneq_epu16_mask + #define _mm_cmpneq_epu16_mask(a, b) simde_mm_cmpneq_epu16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmpneq_epu16_mask(simde__mmask8 k1, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_cmpneq_epu16_mask(k1, a, b); + #else + return simde_mm_mask_cmpneq_epi16_mask(k1, a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpneq_epu16_mask + #define _mm_mask_cmpneq_epu16_mask(k1, a, b) simde_mm_mask_cmpneq_epu16_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmpneq_epi32_mask(simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_cmpneq_epi32_mask(a, b); + #else + return (~simde_mm_movepi32_mask(simde_mm_cmpeq_epi32(a, b))) & 15; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpneq_epi32_mask + #define _mm_cmpneq_epi32_mask(a, b) simde_mm_cmpneq_epi32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmpneq_epi32_mask(simde__mmask8 k1, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_cmpneq_epi32_mask(k1, a, b); + #else + return simde_mm_cmpneq_epi32_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpneq_epi32_mask + #define _mm_mask_cmpneq_epi32_mask(k1, a, b) simde_mm_mask_cmpneq_epi32_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmpneq_epu32_mask(simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_cmpneq_epu32_mask(a, b); + #else + return simde_mm_cmpneq_epi32_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpneq_epu32_mask + #define _mm_cmpneq_epu32_mask(a, b) simde_mm_cmpneq_epu32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmpneq_epu32_mask(simde__mmask8 k1, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_cmpneq_epu32_mask(k1, a, b); + #else + return simde_mm_mask_cmpneq_epi32_mask(k1, a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpneq_epu32_mask + #define _mm_mask_cmpneq_epu32_mask(k1, a, b) simde_mm_mask_cmpneq_epu32_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmpneq_epi64_mask(simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_cmpneq_epi64_mask(a, b); + #else + return (~simde_mm_movepi64_mask(simde_mm_cmpeq_epi64(a, b))) & 3; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpneq_epi64_mask + #define _mm_cmpneq_epi64_mask(a, b) simde_mm_cmpneq_epi64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmpneq_epi64_mask(simde__mmask8 k1, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_cmpneq_epi64_mask(k1, a, b); + #else + return simde_mm_cmpneq_epi64_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpneq_epi64_mask + #define _mm_mask_cmpneq_epi64_mask(k1, a, b) simde_mm_mask_cmpneq_epi64_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_cmpneq_epu64_mask(simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_cmpneq_epu64_mask(a, b); + #else + return simde_mm_cmpneq_epi64_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpneq_epu64_mask + #define _mm_cmpneq_epu64_mask(a, b) simde_mm_cmpneq_epu64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_mask_cmpneq_epu64_mask(simde__mmask8 k1, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_cmpneq_epu64_mask(k1, a, b); + #else + return simde_mm_mask_cmpneq_epi64_mask(k1, a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cmpneq_epu64_mask + #define _mm_mask_cmpneq_epu64_mask(k1, a, b) simde_mm_mask_cmpneq_epu64_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_cmpneq_epi8_mask(simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cmpneq_epi8_mask(a, b); + #else + return ~simde_mm256_movepi8_mask(simde_mm256_cmpeq_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpneq_epi8_mask + #define _mm256_cmpneq_epi8_mask(a, b) simde_mm256_cmpneq_epi8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_mask_cmpneq_epi8_mask(simde__mmask32 k1, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_cmpneq_epi8_mask(k1, a, b); + #else + return simde_mm256_cmpneq_epi8_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpneq_epi8_mask + #define _mm256_mask_cmpneq_epi8_mask(k1, a, b) simde_mm256_mask_cmpneq_epi8_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_cmpneq_epu8_mask(simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cmpneq_epu8_mask(a, b); + #else + return simde_mm256_cmpneq_epi8_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpneq_epu8_mask + #define _mm256_cmpneq_epu8_mask(a, b) simde_mm256_cmpneq_epu8_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_mask_cmpneq_epu8_mask(simde__mmask32 k1, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_cmpneq_epu8_mask(k1, a, b); + #else + return simde_mm256_mask_cmpneq_epi8_mask(k1, a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpneq_epu8_mask + #define _mm256_mask_cmpneq_epu8_mask(k1, a, b) simde_mm256_mask_cmpneq_epu8_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm256_cmpneq_epi16_mask(simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cmpneq_epi16_mask(a, b); + #else + return ~simde_mm256_movepi16_mask(simde_mm256_cmpeq_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpneq_epi16_mask + #define _mm256_cmpneq_epi16_mask(a, b) simde_mm256_cmpneq_epi16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm256_mask_cmpneq_epi16_mask(simde__mmask16 k1, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_cmpneq_epi16_mask(k1, a, b); + #else + return simde_mm256_cmpneq_epi16_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpneq_epi16_mask + #define _mm256_mask_cmpneq_epi16_mask(k1, a, b) simde_mm256_mask_cmpneq_epi16_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm256_cmpneq_epu16_mask(simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cmpneq_epu16_mask(a, b); + #else + return simde_mm256_cmpneq_epi16_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpneq_epu16_mask + #define _mm256_cmpneq_epu16_mask(a, b) simde_mm256_cmpneq_epu16_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm256_mask_cmpneq_epu16_mask(simde__mmask16 k1, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_mask_cmpneq_epu16_mask(k1, a, b); + #else + return simde_mm256_mask_cmpneq_epi16_mask(k1, a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpneq_epu16_mask + #define _mm256_mask_cmpneq_epu16_mask(k1, a, b) simde_mm256_mask_cmpneq_epu16_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_cmpneq_epi32_mask(simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_cmpneq_epi32_mask(a, b); + #else + return (~simde_mm256_movepi32_mask(simde_mm256_cmpeq_epi32(a, b))); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpneq_epi32_mask + #define _mm256_cmpneq_epi32_mask(a, b) simde_mm256_cmpneq_epi32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_mask_cmpneq_epi32_mask(simde__mmask8 k1, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_cmpneq_epi32_mask(k1, a, b); + #else + return simde_mm256_cmpneq_epi32_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpneq_epi32_mask + #define _mm256_mask_cmpneq_epi32_mask(k1, a, b) simde_mm256_mask_cmpneq_epi32_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_cmpneq_epu32_mask(simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_cmpneq_epu32_mask(a, b); + #else + return simde_mm256_cmpneq_epi32_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpneq_epu32_mask + #define _mm256_cmpneq_epu32_mask(a, b) simde_mm256_cmpneq_epu32_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_mask_cmpneq_epu32_mask(simde__mmask8 k1, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_cmpneq_epu32_mask(k1, a, b); + #else + return simde_mm256_mask_cmpneq_epi32_mask(k1, a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpneq_epu32_mask + #define _mm256_mask_cmpneq_epu32_mask(k1, a, b) simde_mm256_mask_cmpneq_epu32_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_cmpneq_epi64_mask(simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_cmpneq_epi64_mask(a, b); + #else + return (~simde_mm256_movepi64_mask(simde_mm256_cmpeq_epi64(a, b))) & 15; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpneq_epi64_mask + #define _mm256_cmpneq_epi64_mask(a, b) simde_mm256_cmpneq_epi64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_mask_cmpneq_epi64_mask(simde__mmask8 k1, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_cmpneq_epi64_mask(k1, a, b); + #else + return simde_mm256_cmpneq_epi64_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpneq_epi64_mask + #define _mm256_mask_cmpneq_epi64_mask(k1, a, b) simde_mm256_mask_cmpneq_epi64_mask((k1), (a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_cmpneq_epu64_mask(simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_cmpneq_epu64_mask(a, b); + #else + return simde_mm256_cmpneq_epi64_mask(a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cmpneq_epu64_mask + #define _mm256_cmpneq_epu64_mask(a, b) simde_mm256_cmpneq_epu64_mask((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_mask_cmpneq_epu64_mask(simde__mmask8 k1, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_cmpneq_epu64_mask(k1, a, b); + #else + return simde_mm256_mask_cmpneq_epi64_mask(k1, a, b); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_cmpneq_epu64_mask + #define _mm256_mask_cmpneq_epu64_mask(k1, a, b) simde_mm256_mask_cmpneq_epu64_mask((k1), (a), (b)) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_CMPNEQ_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/compress.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/compress.h new file mode 100644 index 00000000..06fffc73 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/compress.h @@ -0,0 +1,755 @@ +#if !defined(SIMDE_X86_AVX512_COMPRESS_H) +#define SIMDE_X86_AVX512_COMPRESS_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mask_compress_pd (simde__m256d src, simde__mmask8 k, simde__m256d a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm256_mask_compress_pd(src, k, a); + #else + simde__m256d_private + a_ = simde__m256d_to_private(a), + src_ = simde__m256d_to_private(src); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + if ((k >> i) & 1) { + a_.f64[ri++] = a_.f64[i]; + } + } + + for ( ; ri < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; ri++) { + a_.f64[ri] = src_.f64[ri]; + } + + return simde__m256d_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_compress_pd + #define _mm256_mask_compress_pd(src, k, a) simde_mm256_mask_compress_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_mask_compressstoreu_pd (void* base_addr, simde__mmask8 k, simde__m256d a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && !defined(__znver4__) + _mm256_mask_compressstoreu_pd(base_addr, k, a); + #elif defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && defined(__znver4__) + simde__mmask8 store_mask = _pext_u32(-1, k); + _mm256_mask_storeu_pd(base_addr, store_mask, _mm256_maskz_compress_pd(k, a)); + #else + simde__m256d_private + a_ = simde__m256d_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + if ((k >> i) & 1) { + a_.f64[ri++] = a_.f64[i]; + } + } + + simde_memcpy(base_addr, &a_, ri * sizeof(a_.f64[0])); + + return; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_compressstoreu_pd + #define _mm256_mask_compressstoreu_pd(base_addr, k, a) simde_mm256_mask_compressstoreu_pd(base_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_maskz_compress_pd (simde__mmask8 k, simde__m256d a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm256_maskz_compress_pd(k, a); + #else + simde__m256d_private + a_ = simde__m256d_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + if ((k >> i) & 1) { + a_.f64[ri++] = a_.f64[i]; + } + } + + for ( ; ri < (sizeof(a_.f64) / sizeof(a_.f64[0])); ri++) { + a_.f64[ri] = SIMDE_FLOAT64_C(0.0); + } + + return simde__m256d_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_compress_pd + #define _mm256_maskz_compress_pd(k, a) simde_mm256_maskz_compress_pd(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask_compress_ps (simde__m256 src, simde__mmask8 k, simde__m256 a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm256_mask_compress_ps(src, k, a); + #else + simde__m256_private + a_ = simde__m256_to_private(a), + src_ = simde__m256_to_private(src); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + if ((k >> i) & 1) { + a_.f32[ri++] = a_.f32[i]; + } + } + + for ( ; ri < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; ri++) { + a_.f32[ri] = src_.f32[ri]; + } + + return simde__m256_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_compress_ps + #define _mm256_mask_compress_ps(src, k, a) simde_mm256_mask_compress_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_mask_compressstoreu_ps (void* base_addr, simde__mmask8 k, simde__m256 a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && !defined(__znver4__) + _mm256_mask_compressstoreu_ps(base_addr, k, a); + #elif defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && defined(__znver4__) + simde__mmask8 store_mask = _pext_u32(-1, k); + _mm256_mask_storeu_ps(base_addr, store_mask, _mm256_maskz_compress_ps(k, a)); + #else + simde__m256_private + a_ = simde__m256_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + if ((k >> i) & 1) { + a_.f32[ri++] = a_.f32[i]; + } + } + + simde_memcpy(base_addr, &a_, ri * sizeof(a_.f32[0])); + + return; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_compressstoreu_ps + #define _mm256_mask_compressstoreu_ps(base_addr, k, a) simde_mm256_mask_compressstoreu_ps(base_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_maskz_compress_ps (simde__mmask8 k, simde__m256 a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm256_maskz_compress_ps(k, a); + #else + simde__m256_private + a_ = simde__m256_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + if ((k >> i) & 1) { + a_.f32[ri++] = a_.f32[i]; + } + } + + for ( ; ri < (sizeof(a_.f32) / sizeof(a_.f32[0])); ri++) { + a_.f32[ri] = SIMDE_FLOAT32_C(0.0); + } + + return simde__m256_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_compress_ps + #define _mm256_maskz_compress_ps(k, a) simde_mm256_maskz_compress_ps(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_compress_epi32 (simde__m256i src, simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm256_mask_compress_epi32(src, k, a); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + src_ = simde__m256i_to_private(src); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + if ((k >> i) & 1) { + a_.i32[ri++] = a_.i32[i]; + } + } + + for ( ; ri < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; ri++) { + a_.i32[ri] = src_.i32[ri]; + } + + return simde__m256i_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_compress_epi32 + #define _mm256_mask_compress_epi32(src, k, a) simde_mm256_mask_compress_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_mask_compressstoreu_epi32 (void* base_addr, simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && !defined(__znver4__) + _mm256_mask_compressstoreu_epi32(base_addr, k, a); + #elif defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && defined(__znver4__) + simde__mmask8 store_mask = _pext_u32(-1, k); + _mm256_mask_storeu_epi32(base_addr, store_mask, _mm256_maskz_compress_epi32(k, a)); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + if ((k >> i) & 1) { + a_.i32[ri++] = a_.i32[i]; + } + } + + simde_memcpy(base_addr, &a_, ri * sizeof(a_.i32[0])); + + return; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_compressstoreu_epi32 + #define _mm256_mask_compressstoreu_epi32(base_addr, k, a) simde_mm256_mask_compressstoreu_epi32(base_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_compress_epi32 (simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm256_maskz_compress_epi32(k, a); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + if ((k >> i) & 1) { + a_.i32[ri++] = a_.i32[i]; + } + } + + for ( ; ri < (sizeof(a_.i32) / sizeof(a_.i32[0])); ri++) { + a_.f32[ri] = INT32_C(0); + } + + return simde__m256i_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_compress_epi32 + #define _mm256_maskz_compress_epi32(k, a) simde_mm256_maskz_compress_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_compress_epi64 (simde__m256i src, simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm256_mask_compress_epi64(src, k, a); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + src_ = simde__m256i_to_private(src); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + if ((k >> i) & 1) { + a_.i64[ri++] = a_.i64[i]; + } + } + + for ( ; ri < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; ri++) { + a_.i64[ri] = src_.i64[ri]; + } + + return simde__m256i_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_compress_epi64 + #define _mm256_mask_compress_epi64(src, k, a) simde_mm256_mask_compress_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_mask_compressstoreu_epi64 (void* base_addr, simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && !defined(__znver4__) + _mm256_mask_compressstoreu_epi64(base_addr, k, a); + #elif defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && defined(__znver4__) + simde__mmask8 store_mask = _pext_u32(-1, k); + _mm256_mask_storeu_epi64(base_addr, store_mask, _mm256_maskz_compress_epi64(k, a)); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + if ((k >> i) & 1) { + a_.i64[ri++] = a_.i64[i]; + } + } + + simde_memcpy(base_addr, &a_, ri * sizeof(a_.i64[0])); + + return; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_compressstoreu_epi64 + #define _mm256_mask_compressstoreu_epi64(base_addr, k, a) simde_mm256_mask_compressstoreu_epi64(base_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_compress_epi64 (simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm256_maskz_compress_epi64(k, a); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + if ((k >> i) & 1) { + a_.i64[ri++] = a_.i64[i]; + } + } + + for ( ; ri < (sizeof(a_.i64) / sizeof(a_.i64[0])); ri++) { + a_.i64[ri] = INT64_C(0); + } + + return simde__m256i_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_compress_epi64 + #define _mm256_maskz_compress_epi64(k, a) simde_mm256_maskz_compress_epi64(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_compress_pd (simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_compress_pd(src, k, a); + #else + simde__m512d_private + a_ = simde__m512d_to_private(a), + src_ = simde__m512d_to_private(src); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + if ((k >> i) & 1) { + a_.f64[ri++] = a_.f64[i]; + } + } + + for ( ; ri < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; ri++) { + a_.f64[ri] = src_.f64[ri]; + } + + return simde__m512d_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_compress_pd + #define _mm512_mask_compress_pd(src, k, a) simde_mm512_mask_compress_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_mask_compressstoreu_pd (void* base_addr, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && !defined(__znver4__) + _mm512_mask_compressstoreu_pd(base_addr, k, a); + #elif defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && defined(__znver4__) + simde__mmask8 store_mask = _pext_u32(-1, k); + _mm512_mask_storeu_pd(base_addr, store_mask, _mm512_maskz_compress_pd(k, a)); + #else + simde__m512d_private + a_ = simde__m512d_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + if ((k >> i) & 1) { + a_.f64[ri++] = a_.f64[i]; + } + } + + simde_memcpy(base_addr, &a_, ri * sizeof(a_.f64[0])); + + return; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_compressstoreu_pd + #define _mm512_mask_compressstoreu_pd(base_addr, k, a) simde_mm512_mask_compressstoreu_pd(base_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_compress_pd (simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_compress_pd(k, a); + #else + simde__m512d_private + a_ = simde__m512d_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + if ((k >> i) & 1) { + a_.f64[ri++] = a_.f64[i]; + } + } + + for ( ; ri < (sizeof(a_.f64) / sizeof(a_.f64[0])); ri++) { + a_.f64[ri] = SIMDE_FLOAT64_C(0.0); + } + + return simde__m512d_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_compress_pd + #define _mm512_maskz_compress_pd(k, a) simde_mm512_maskz_compress_pd(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_compress_ps (simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_compress_ps(src, k, a); + #else + simde__m512_private + a_ = simde__m512_to_private(a), + src_ = simde__m512_to_private(src); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + if ((k >> i) & 1) { + a_.f32[ri++] = a_.f32[i]; + } + } + + for ( ; ri < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; ri++) { + a_.f32[ri] = src_.f32[ri]; + } + + return simde__m512_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_compress_ps + #define _mm512_mask_compress_ps(src, k, a) simde_mm512_mask_compress_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_mask_compressstoreu_ps (void* base_addr, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && !defined(__znver4__) + _mm512_mask_compressstoreu_ps(base_addr, k, a); + #elif defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && defined(__znver4__) + simde__mmask16 store_mask = _pext_u32(-1, k); + _mm512_mask_storeu_ps(base_addr, store_mask, _mm512_maskz_compress_ps(k, a)); + #else + simde__m512_private + a_ = simde__m512_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + if ((k >> i) & 1) { + a_.f32[ri++] = a_.f32[i]; + } + } + + simde_memcpy(base_addr, &a_, ri * sizeof(a_.f32[0])); + + return; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_compressstoreu_ps + #define _mm512_mask_compressstoreu_ps(base_addr, k, a) simde_mm512_mask_compressstoreu_ps(base_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_compress_ps (simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_compress_ps(k, a); + #else + simde__m512_private + a_ = simde__m512_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + if ((k >> i) & 1) { + a_.f32[ri++] = a_.f32[i]; + } + } + + for ( ; ri < (sizeof(a_.f32) / sizeof(a_.f32[0])); ri++) { + a_.f32[ri] = SIMDE_FLOAT32_C(0.0); + } + + return simde__m512_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_compress_ps + #define _mm512_maskz_compress_ps(k, a) simde_mm512_maskz_compress_ps(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_compress_epi32 (simde__m512i src, simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_compress_epi32(src, k, a); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + src_ = simde__m512i_to_private(src); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + if ((k >> i) & 1) { + a_.i32[ri++] = a_.i32[i]; + } + } + + for ( ; ri < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; ri++) { + a_.i32[ri] = src_.i32[ri]; + } + + return simde__m512i_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_compress_epi32 + #define _mm512_mask_compress_epi32(src, k, a) simde_mm512_mask_compress_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_mask_compressstoreu_epi16 (void* base_addr, simde__mmask32 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512VBMI2_NATIVE) && !defined(__znver4__) + _mm512_mask_compressstoreu_epi16(base_addr, k, a); + #elif defined(SIMDE_X86_AVX512VBMI2_NATIVE) && defined(__znver4__) + simde__mmask32 store_mask = _pext_u32(-1, k); + _mm512_mask_storeu_epi16(base_addr, store_mask, _mm512_maskz_compress_epi16(k, a)); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0])) ; i++) { + if ((k >> i) & 1) { + a_.i16[ri++] = a_.i16[i]; + } + } + + simde_memcpy(base_addr, &a_, ri * sizeof(a_.i16[0])); + + return; + #endif +} +#if defined(SIMDE_X86_AVX512VBMI2_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_compressstoreu_epi16 + #define _mm512_mask_compressstoreu_epi16(base_addr, k, a) simde_mm512_mask_compressstoreu_epi16(base_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_mask_compressstoreu_epi32 (void* base_addr, simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && !defined(__znver4__) + _mm512_mask_compressstoreu_epi32(base_addr, k, a); + #elif defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && defined(__znver4__) + simde__mmask16 store_mask = _pext_u32(-1, k); + _mm512_mask_storeu_epi32(base_addr, store_mask, _mm512_maskz_compress_epi32(k, a)); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + if ((k >> i) & 1) { + a_.i32[ri++] = a_.i32[i]; + } + } + + simde_memcpy(base_addr, &a_, ri * sizeof(a_.i32[0])); + + return; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_compressstoreu_epi32 + #define _mm512_mask_compressstoreu_epi32(base_addr, k, a) simde_mm512_mask_compressstoreu_epi32(base_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_compress_epi32 (simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_compress_epi32(k, a); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + if ((k >> i) & 1) { + a_.i32[ri++] = a_.i32[i]; + } + } + + for ( ; ri < (sizeof(a_.i32) / sizeof(a_.i32[0])); ri++) { + a_.f32[ri] = INT32_C(0); + } + + return simde__m512i_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_compress_epi32 + #define _mm512_maskz_compress_epi32(k, a) simde_mm512_maskz_compress_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_compress_epi64 (simde__m512i src, simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_compress_epi64(src, k, a); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + src_ = simde__m512i_to_private(src); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + if ((k >> i) & 1) { + a_.i64[ri++] = a_.i64[i]; + } + } + + for ( ; ri < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; ri++) { + a_.i64[ri] = src_.i64[ri]; + } + + return simde__m512i_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_compress_epi64 + #define _mm512_mask_compress_epi64(src, k, a) simde_mm512_mask_compress_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_mask_compressstoreu_epi64 (void* base_addr, simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && !defined(__znver4__) + _mm512_mask_compressstoreu_epi64(base_addr, k, a); + #elif defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) && defined(__znver4__) + simde__mmask8 store_mask = _pext_u32(-1, k); + _mm512_mask_storeu_epi64(base_addr, store_mask, _mm512_maskz_compress_epi64(k, a)); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + if ((k >> i) & 1) { + a_.i64[ri++] = a_.i64[i]; + } + } + + simde_memcpy(base_addr, &a_, ri * sizeof(a_.i64[0])); + + return; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_compressstoreu_epi64 + #define _mm512_mask_compressstoreu_epi64(base_addr, k, a) simde_mm512_mask_compressstoreu_epi64(base_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_compress_epi64 (simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_compress_epi64(k, a); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a); + size_t ri = 0; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + if ((k >> i) & 1) { + a_.i64[ri++] = a_.i64[i]; + } + } + + for ( ; ri < (sizeof(a_.i64) / sizeof(a_.i64[0])); ri++) { + a_.i64[ri] = INT64_C(0); + } + + return simde__m512i_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_compress_epi64 + #define _mm512_maskz_compress_epi64(k, a) simde_mm512_maskz_compress_epi64(k, a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_COMPRESS_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/conflict.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/conflict.h new file mode 100644 index 00000000..239aef9b --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/conflict.h @@ -0,0 +1,351 @@ +#if !defined(SIMDE_X86_AVX512_CONFLICT_H) +#define SIMDE_X86_AVX512_CONFLICT_H + +#include "types.h" +#include "mov_mask.h" +#include "mov.h" +#include "cmpeq.h" +#include "set1.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_conflict_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm_conflict_epi32(a); + #else + simde__m128i_private + r_ = simde__m128i_to_private(simde_mm_setzero_si128()), + a_ = simde__m128i_to_private(a); + + for (size_t i = 1 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = + simde_mm_movemask_ps( + simde_mm_castsi128_ps( + simde_mm_cmpeq_epi32(simde_mm_set1_epi32(a_.i32[i]), a) + ) + ) & ((1 << i) - 1); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm_conflict_epi32 + #define _mm_conflict_epi32(a) simde_mm_conflict_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_conflict_epi32 (simde__m128i src, simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm_mask_conflict_epi32(src, k, a); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_conflict_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_conflict_epi32 + #define _mm_mask_conflict_epi32(src, k, a) simde_mm_mask_conflict_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_conflict_epi32 (simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm_maskz_conflict_epi32(k, a); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_conflict_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_conflict_epi32 + #define _mm_maskz_conflict_epi32(k, a) simde_mm_maskz_conflict_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_conflict_epi32 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm256_conflict_epi32(a); + #else + simde__m256i_private + r_ = simde__m256i_to_private(simde_mm256_setzero_si256()), + a_ = simde__m256i_to_private(a); + + for (size_t i = 1 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = + simde_mm256_movemask_ps( + simde_mm256_castsi256_ps( + simde_mm256_cmpeq_epi32(simde_mm256_set1_epi32(a_.i32[i]), a) + ) + ) & ((1 << i) - 1); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm256_conflict_epi32 + #define _mm256_conflict_epi32(a) simde_mm256_conflict_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_conflict_epi32 (simde__m256i src, simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm256_mask_conflict_epi32(src, k, a); + #else + return simde_mm256_mask_mov_epi32(src, k, simde_mm256_conflict_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_conflict_epi32 + #define _mm256_mask_conflict_epi32(src, k, a) simde_mm256_mask_conflict_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_conflict_epi32 (simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm256_maskz_conflict_epi32(k, a); + #else + return simde_mm256_maskz_mov_epi32(k, simde_mm256_conflict_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_conflict_epi32 + #define _mm256_maskz_conflict_epi32(k, a) simde_mm256_maskz_conflict_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_conflict_epi32 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm512_conflict_epi32(a); + #else + simde__m512i_private + r_ = simde__m512i_to_private(simde_mm512_setzero_si512()), + a_ = simde__m512i_to_private(a); + + for (size_t i = 1 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = + HEDLEY_STATIC_CAST( + int32_t, + simde_mm512_cmpeq_epi32_mask(simde_mm512_set1_epi32(a_.i32[i]), a) + ) & ((1 << i) - 1); + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm512_conflict_epi32 + #define _mm512_conflict_epi32(a) simde_mm512_conflict_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_conflict_epi32 (simde__m512i src, simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm512_mask_conflict_epi32(src, k, a); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_conflict_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_conflict_epi32 + #define _mm512_mask_conflict_epi32(src, k, a) simde_mm512_mask_conflict_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_conflict_epi32 (simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm512_maskz_conflict_epi32(k, a); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_conflict_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_conflict_epi32 + #define _mm512_maskz_conflict_epi32(k, a) simde_mm512_maskz_conflict_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_conflict_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm_conflict_epi64(a); + #else + simde__m128i_private + r_ = simde__m128i_to_private(simde_mm_setzero_si128()), + a_ = simde__m128i_to_private(a); + + for (size_t i = 1 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = + HEDLEY_STATIC_CAST( + int64_t, + simde_mm_movemask_pd( + simde_mm_castsi128_pd( + simde_mm_cmpeq_epi64(simde_mm_set1_epi64x(a_.i64[i]), a) + ) + ) + ) & ((1 << i) - 1); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm_conflict_epi64 + #define _mm_conflict_epi64(a) simde_mm_conflict_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_conflict_epi64 (simde__m128i src, simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm_mask_conflict_epi64(src, k, a); + #else + return simde_mm_mask_mov_epi64(src, k, simde_mm_conflict_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_conflict_epi64 + #define _mm_mask_conflict_epi64(src, k, a) simde_mm_mask_conflict_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_conflict_epi64 (simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm_maskz_conflict_epi64(k, a); + #else + return simde_mm_maskz_mov_epi64(k, simde_mm_conflict_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_conflict_epi64 + #define _mm_maskz_conflict_epi64(k, a) simde_mm_maskz_conflict_epi64(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_conflict_epi64 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm256_conflict_epi64(a); + #else + simde__m256i_private + r_ = simde__m256i_to_private(simde_mm256_setzero_si256()), + a_ = simde__m256i_to_private(a); + + for (size_t i = 1 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = + HEDLEY_STATIC_CAST( + int64_t, + simde_mm256_movemask_pd( + simde_mm256_castsi256_pd( + simde_mm256_cmpeq_epi64(simde_mm256_set1_epi64x(a_.i64[i]), a) + ) + ) + ) & ((1 << i) - 1); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm256_conflict_epi64 + #define _mm256_conflict_epi64(a) simde_mm256_conflict_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_conflict_epi64 (simde__m256i src, simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm256_mask_conflict_epi64(src, k, a); + #else + return simde_mm256_mask_mov_epi64(src, k, simde_mm256_conflict_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_conflict_epi64 + #define _mm256_mask_conflict_epi64(src, k, a) simde_mm256_mask_conflict_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_conflict_epi64 (simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm256_maskz_conflict_epi64(k, a); + #else + return simde_mm256_maskz_mov_epi64(k, simde_mm256_conflict_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_conflict_epi64 + #define _mm256_maskz_conflict_epi64(k, a) simde_mm256_maskz_conflict_epi64(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_conflict_epi64 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm512_conflict_epi64(a); + #else + simde__m512i_private + r_ = simde__m512i_to_private(simde_mm512_setzero_si512()), + a_ = simde__m512i_to_private(a); + + for (size_t i = 1 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = + HEDLEY_STATIC_CAST( + int64_t, + simde_mm512_cmpeq_epi64_mask(simde_mm512_set1_epi64(a_.i64[i]), a) + ) & ((1 << i) - 1); + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm512_conflict_epi64 + #define _mm512_conflict_epi64(a) simde_mm512_conflict_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_conflict_epi64 (simde__m512i src, simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm512_mask_conflict_epi64(src, k, a); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_conflict_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_conflict_epi64 + #define _mm512_mask_conflict_epi64(src, k, a) simde_mm512_mask_conflict_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_conflict_epi64 (simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm512_maskz_conflict_epi64(k, a); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_conflict_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_conflict_epi64 + #define _mm512_maskz_conflict_epi64(k, a) simde_mm512_maskz_conflict_epi64(k, a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_CONFLICT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/copysign.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/copysign.h new file mode 100644 index 00000000..eba37e9f --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/copysign.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + */ + +#if !defined(SIMDE_X86_AVX512_COPYSIGN_H) +#define SIMDE_X86_AVX512_COPYSIGN_H + +#include "types.h" +#include "mov.h" +#include "and.h" +#include "andnot.h" +#include "xor.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_x_mm512_copysign_ps(simde__m512 dest, simde__m512 src) { + simde__m512_private + r_, + dest_ = simde__m512_to_private(dest), + src_ = simde__m512_to_private(src); + + #if defined(simde_math_copysignf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_copysignf(dest_.f32[i], src_.f32[i]); + } + #else + simde__m512 sgnbit = simde_mm512_xor_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.0)), simde_mm512_set1_ps(-SIMDE_FLOAT32_C(0.0))); + return simde_mm512_xor_ps(simde_mm512_and_ps(sgnbit, src), simde_mm512_andnot_ps(sgnbit, dest)); + #endif + + return simde__m512_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_x_mm512_copysign_pd(simde__m512d dest, simde__m512d src) { + simde__m512d_private + r_, + dest_ = simde__m512d_to_private(dest), + src_ = simde__m512d_to_private(src); + + #if defined(simde_math_copysign) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_copysign(dest_.f64[i], src_.f64[i]); + } + #else + simde__m512d sgnbit = simde_mm512_xor_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.0)), simde_mm512_set1_pd(-SIMDE_FLOAT64_C(0.0))); + return simde_mm512_xor_pd(simde_mm512_and_pd(sgnbit, src), simde_mm512_andnot_pd(sgnbit, dest)); + #endif + + return simde__m512d_from_private(r_); +} + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_COPYSIGN_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvt.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvt.h new file mode 100644 index 00000000..579bcac1 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvt.h @@ -0,0 +1,402 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020-2021 Evan Nemerson + * 2020 Himanshi Mathur + * 2020 Hidayat Khan + * 2021 Andrew Rodriguez + */ + +#if !defined(SIMDE_X86_AVX512_CVT_H) +#define SIMDE_X86_AVX512_CVT_H + +#include "types.h" +#include "mov.h" +#include "../../simde-f16.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cvtepi64_pd (simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm_cvtepi64_pd(a); + #else + simde__m128d_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_X86_SSE2_NATIVE) + /* https://stackoverflow.com/questions/41144668/how-to-efficiently-perform-double-int64-conversions-with-sse-avx */ + __m128i xH = _mm_srai_epi32(a_.n, 16); + #if defined(SIMDE_X86_SSE4_2_NATIVE) + xH = _mm_blend_epi16(xH, _mm_setzero_si128(), 0x33); + #else + xH = _mm_and_si128(xH, _mm_set_epi16(~INT16_C(0), ~INT16_C(0), INT16_C(0), INT16_C(0), ~INT16_C(0), ~INT16_C(0), INT16_C(0), INT16_C(0))); + #endif + xH = _mm_add_epi64(xH, _mm_castpd_si128(_mm_set1_pd(442721857769029238784.0))); + const __m128i e = _mm_castpd_si128(_mm_set1_pd(0x0010000000000000)); + #if defined(SIMDE_X86_SSE4_2_NATIVE) + __m128i xL = _mm_blend_epi16(a_.n, e, 0x88); + #else + __m128i m = _mm_set_epi16(INT16_C(0), ~INT16_C(0), ~INT16_C(0), ~INT16_C(0), INT16_C(0), ~INT16_C(0), ~INT16_C(0), ~INT16_C(0)); + __m128i xL = _mm_or_si128(_mm_and_si128(m, a_.n), _mm_andnot_si128(m, e)); + #endif + __m128d f = _mm_sub_pd(_mm_castsi128_pd(xH), _mm_set1_pd(442726361368656609280.0)); + return _mm_add_pd(f, _mm_castsi128_pd(xL)); + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.f64, a_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = HEDLEY_STATIC_CAST(simde_float64, a_.i64[i]); + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtepi64_pd + #define _mm_cvtepi64_pd(a) simde_mm_cvtepi64_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mask_cvtepi64_pd(simde__m128d src, simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm_mask_cvtepi64_pd(src, k, a); + #else + return simde_mm_mask_mov_pd(src, k, simde_mm_cvtepi64_pd(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cvtepi64_pd + #define _mm_mask_cvtepi64_pd(src, k, a) simde_mm_mask_cvtepi64_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_maskz_cvtepi64_pd(simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm_maskz_cvtepi64_pd(k, a); + #else + return simde_mm_maskz_mov_pd(k, simde_mm_cvtepi64_pd(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_cvtepi64_pd + #define _mm_maskz_cvtepi64_pd(k, a) simde_mm_maskz_cvtepi64_pd((k), (a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_cvtepi16_epi32 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cvtepi16_epi32(a); + #else + simde__m512i_private r_; + simde__m256i_private a_ = simde__m256i_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i32, a_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i16[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtepi16_epi32 + #define _mm512_cvtepi16_epi32(a) simde_mm512_cvtepi16_epi32(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_cvtepi16_epi8 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cvtepi16_epi8(a); + #else + simde__m256i_private r_; + simde__m512i_private a_ = simde__m512i_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i8, a_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = HEDLEY_STATIC_CAST(int8_t, a_.i16[i]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtepi16_epi8 + #define _mm512_cvtepi16_epi8(a) simde_mm512_cvtepi16_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_mask_cvtepi16_epi8 (simde__m256i src, simde__mmask32 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_cvtepi16_epi8(src, k, a); + #else + return simde_mm256_mask_mov_epi8(src, k, simde_mm512_cvtepi16_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cvtepi16_epi8 + #define _mm512_mask_cvtepi16_epi8(src, k, a) simde_mm512_mask_cvtepi16_epi8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_maskz_cvtepi16_epi8 (simde__mmask32 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_cvtepi16_epi8(k, a); + #else + return simde_mm256_maskz_mov_epi8(k, simde_mm512_cvtepi16_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_cvtepi16_epi8 + #define _mm512_maskz_cvtepi16_epi8(k, a) simde_mm512_maskz_cvtepi16_epi8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_cvtepi8_epi16 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cvtepi8_epi16(a); + #else + simde__m512i_private r_; + simde__m256i_private a_ = simde__m256i_to_private(a); + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256i[0] = _mm256_cvtepi8_epi16(a_.m128i[0]); + r_.m256i[1] = _mm256_cvtepi8_epi16(a_.m128i[1]); + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i16, a_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i8[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtepi8_epi16 + #define _mm512_cvtepi8_epi16(a) simde_mm512_cvtepi8_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_cvtepi32_ps (simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cvtepi32_ps(a); + #else + simde__m512_private r_; + simde__m512i_private a_ = simde__m512i_to_private(a); + + #if defined(SIMDE_X86_AVX_NATIVE) + r_.m256[0] = _mm256_cvtepi32_ps(a_.m256i[0]); + r_.m256[1] = _mm256_cvtepi32_ps(a_.m256i[1]); + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.f32, a_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.f32[i] = HEDLEY_STATIC_CAST(simde_float32, a_.i32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtepi32_ps + #define _mm512_cvtepi32_ps(a) simde_mm512_cvtepi32_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_cvtepi64_epi32 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cvtepi64_epi32(a); + #else + simde__m256i_private r_; + simde__m512i_private a_ = simde__m512i_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i32, a_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = HEDLEY_STATIC_CAST(int32_t, a_.i64[i]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtepi64_epi32 + #define _mm512_cvtepi64_epi32(a) simde_mm512_cvtepi64_epi32((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_cvtepu16_epi32 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cvtepu16_epi32(a); + #else + simde__m512i_private r_; + simde__m256i_private a_ = simde__m256i_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i32, a_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = HEDLEY_STATIC_CAST(int32_t, a_.u16[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtepu16_epi32 + #define _mm512_cvtepu16_epi32(a) simde_mm512_cvtepu16_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_cvtepu32_ps (simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cvtepu32_ps(a); + #else + simde__m512_private r_; + simde__m512i_private a_ = simde__m512i_to_private(a); + + #if defined(SIMDE_X86_SSE2_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + /* https://stackoverflow.com/a/34067907/501126 */ + const __m128 tmp = _mm_cvtepi32_ps(_mm_srli_epi32(a_.m128i[i], 1)); + r_.m128[i] = + _mm_add_ps( + _mm_add_ps(tmp, tmp), + _mm_cvtepi32_ps(_mm_and_si128(a_.m128i[i], _mm_set1_epi32(1))) + ); + } + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.f32, a_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.f32[i] = HEDLEY_STATIC_CAST(float, a_.u32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtepu32_ps + #define _mm512_cvtepu32_ps(a) simde_mm512_cvtepu32_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_cvtph_ps(simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cvtph_ps(a); + #endif + simde__m256i_private a_ = simde__m256i_to_private(a); + simde__m512_private r_; + + #if defined(SIMDE_X86_F16C_NATIVE) + r_.m256[0] = _mm256_cvtph_ps(a_.m128i[0]); + r_.m256[1] = _mm256_cvtph_ps(a_.m128i[1]); + #elif defined(SIMDE_FLOAT16_VECTOR) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_float16_to_float32(a_.f16[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_float16_to_float32(simde_uint16_as_float16(a_.u16[i])); + } + #endif + + return simde__m512_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtph_ps + #define _mm512_cvtph_ps(a) simde_mm512_cvtph_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_cvtps_epi32(simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cvtps_epi32(a); + #endif + simde__m512_private a_ = simde__m512_to_private(a); + simde__m512i_private r_; + + #if defined(SIMDE_X86_AVX_NATIVE) + r_.m256i[0] = _mm256_cvtps_epi32(a_.m256[0]); + r_.m256i[1] = _mm256_cvtps_epi32(a_.m256[1]); + #elif defined(simde_math_nearbyintf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = SIMDE_CONVERT_FTOI(int32_t, simde_math_nearbyintf(a_.f32[i])); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtps_epi32 + #define _mm512_cvtps_epi32(a) simde_mm512_cvtps_epi32(a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_CVT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvts.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvts.h new file mode 100644 index 00000000..0194889a --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvts.h @@ -0,0 +1,781 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_CVTS_H) +#define SIMDE_X86_AVX512_CVTS_H + +#include "types.h" +#include "mov.h" +#include "storeu.h" +#include "loadu.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtsepi16_epi8 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cvtsepi16_epi8(a); + #else + simde__m128i_private r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + simde__m128i_private a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0])) ; i++) { + r_.i8[i] = + (a_.i16[i] < INT8_MIN) + ? (INT8_MIN) + : ((a_.i16[i] > INT8_MAX) + ? (INT8_MAX) + : HEDLEY_STATIC_CAST(int8_t, a_.i16[i])); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtsepi16_epi8 + #define _mm_cvtsepi16_epi8(a) simde_mm_cvtsepi16_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm256_cvtsepi16_epi8 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cvtsepi16_epi8(a); + #else + simde__m128i_private r_; + simde__m256i_private a_ = simde__m256i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = + (a_.i16[i] < INT8_MIN) + ? (INT8_MIN) + : ((a_.i16[i] > INT8_MAX) + ? (INT8_MAX) + : HEDLEY_STATIC_CAST(int8_t, a_.i16[i])); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtsepi16_epi8 + #define _mm256_cvtsepi16_epi8(a) simde_mm256_cvtsepi16_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtsepi32_epi8 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cvtsepi32_epi8(a); + #else + simde__m128i_private r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + simde__m128i_private a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i8[i] = + (a_.i32[i] < INT8_MIN) + ? (INT8_MIN) + : ((a_.i32[i] > INT8_MAX) + ? (INT8_MAX) + : HEDLEY_STATIC_CAST(int8_t, a_.i32[i])); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtsepi32_epi8 + #define _mm_cvtsepi32_epi8(a) simde_mm_cvtsepi32_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm256_cvtsepi32_epi8 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cvtsepi32_epi8(a); + #else + simde__m128i_private r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + simde__m256i_private a_ = simde__m256i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i8[i] = + (a_.i32[i] < INT8_MIN) + ? (INT8_MIN) + : ((a_.i32[i] > INT8_MAX) + ? (INT8_MAX) + : HEDLEY_STATIC_CAST(int8_t, a_.i32[i])); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtsepi32_epi8 + #define _mm256_cvtsepi32_epi8(a) simde_mm256_cvtsepi32_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtsepi32_epi16 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cvtsepi32_epi16(a); + #else + simde__m128i_private r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + simde__m128i_private a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i16[i] = + (a_.i32[i] < INT16_MIN) + ? (INT16_MIN) + : ((a_.i32[i] > INT16_MAX) + ? (INT16_MAX) + : HEDLEY_STATIC_CAST(int16_t, a_.i32[i])); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtsepi32_epi16 + #define _mm_cvtsepi32_epi16(a) simde_mm_cvtsepi32_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm256_cvtsepi32_epi16 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cvtsepi32_epi16(a); + #else + simde__m128i_private r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + simde__m256i_private a_ = simde__m256i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i16[i] = + (a_.i32[i] < INT16_MIN) + ? (INT16_MIN) + : ((a_.i32[i] > INT16_MAX) + ? (INT16_MAX) + : HEDLEY_STATIC_CAST(int16_t, a_.i32[i])); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtsepi32_epi16 + #define _mm256_cvtsepi32_epi16(a) simde_mm256_cvtsepi32_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtsepi64_epi8 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_cvtsepi64_epi8(a); + #else + simde__m128i_private r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + simde__m128i_private a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i8[i] = + (a_.i64[i] < INT8_MIN) + ? (INT8_MIN) + : ((a_.i64[i] > INT8_MAX) + ? (INT8_MAX) + : HEDLEY_STATIC_CAST(int8_t, a_.i64[i])); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtsepi64_epi8 + #define _mm_cvtsepi64_epi8(a) simde_mm_cvtsepi64_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm256_cvtsepi64_epi8 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_cvtsepi64_epi8(a); + #else + simde__m128i_private r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + simde__m256i_private a_ = simde__m256i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i8[i] = + (a_.i64[i] < INT8_MIN) + ? (INT8_MIN) + : ((a_.i64[i] > INT8_MAX) + ? (INT8_MAX) + : HEDLEY_STATIC_CAST(int8_t, a_.i64[i])); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_cvtsepi64_epi8 + #define _mm256_cvtsepi64_epi8(a) simde_mm256_cvtsepi64_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_cvtsepi16_epi8 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cvtsepi16_epi8(a); + #else + simde__m256i_private r_; + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = + (a_.i16[i] < INT8_MIN) + ? (INT8_MIN) + : ((a_.i16[i] > INT8_MAX) + ? (INT8_MAX) + : HEDLEY_STATIC_CAST(int8_t, a_.i16[i])); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtsepi16_epi8 + #define _mm512_cvtsepi16_epi8(a) simde_mm512_cvtsepi16_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_mask_cvtsepi16_epi8 (simde__m256i src, simde__mmask32 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_cvtsepi16_epi8(src, k, a); + #else + return simde_mm256_mask_mov_epi8(src, k, simde_mm512_cvtsepi16_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cvtsepi16_epi8 + #define _mm512_mask_cvtsepi16_epi8(src, k, a) simde_mm512_mask_cvtsepi16_epi8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_maskz_cvtsepi16_epi8 (simde__mmask32 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_cvtsepi16_epi8(k, a); + #else + return simde_mm256_maskz_mov_epi8(k, simde_mm512_cvtsepi16_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_cvtsepi16_epi8 + #define _mm512_maskz_cvtsepi16_epi8(k, a) simde_mm512_maskz_cvtsepi16_epi8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm512_cvtsepi32_epi8 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cvtsepi32_epi8(a); + #else + simde__m128i_private r_; + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i8[i] = + (a_.i32[i] < INT8_MIN) + ? (INT8_MIN) + : ((a_.i32[i] > INT8_MAX) + ? (INT8_MAX) + : HEDLEY_STATIC_CAST(int8_t, a_.i32[i])); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtsepi32_epi8 + #define _mm512_cvtsepi32_epi8(a) simde_mm512_cvtsepi32_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm512_mask_cvtsepi32_epi8 (simde__m128i src, simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cvtsepi32_epi8(src, k, a); + #else + simde__m128i_private r_; + simde__m128i_private src_ = simde__m128i_to_private(src); + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i8[i] = ((k>>i) &1 ) ? + ((a_.i32[i] < INT8_MIN) + ? (INT8_MIN) + : ((a_.i32[i] > INT8_MAX) + ? (INT8_MAX) + : HEDLEY_STATIC_CAST(int8_t, a_.i32[i]))) : src_.i8[i] ; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cvtsepi32_epi8 + #define _mm512_mask_cvtsepi32_epi8(src, k, a) simde_mm512_mask_cvtsepi32_epi8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_mask_cvtsepi32_storeu_epi8 (void* base_addr, simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + _mm512_mask_cvtsepi32_storeu_epi8(base_addr, k, a); + #else + simde__m256i_private r_ = simde__m256i_to_private(simde_mm256_loadu_epi8(base_addr)); + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i8[i] = ((k>>i) &1 ) ? + ((a_.i32[i] < INT8_MIN) + ? (INT8_MIN) + : ((a_.i32[i] > INT8_MAX) + ? (INT8_MAX) + : HEDLEY_STATIC_CAST(int8_t, a_.i32[i]))) : r_.i8[i]; + } + + simde_mm256_storeu_epi8(base_addr, simde__m256i_from_private(r_)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cvtsepi32_storeu_epi8 + #define _mm512_mask_cvtsepi32_storeu_epi8(base_addr, k, a) simde_mm512_mask_cvtsepi32_storeu_epi8(base_addr, k, a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm512_maskz_cvtsepi32_epi8 (simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_cvtsepi32_epi8(k, a); + #else + simde__m128i_private r_; + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i8[i] = ((k>>i) &1 ) ? + ((a_.i32[i] < INT8_MIN) + ? (INT8_MIN) + : ((a_.i32[i] > INT8_MAX) + ? (INT8_MAX) + : HEDLEY_STATIC_CAST(int8_t, a_.i32[i]))) : INT8_C(0) ; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_cvtsepi32_epi8 + #define _mm512_maskz_cvtsepi32_epi8(k, a) simde_mm512_maskz_cvtsepi32_epi8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_cvtsepi32_epi16 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cvtsepi32_epi16(a); + #else + simde__m256i_private r_; + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i16[i] = + (a_.i32[i] < INT16_MIN) + ? (INT16_MIN) + : ((a_.i32[i] > INT16_MAX) + ? (INT16_MAX) + : HEDLEY_STATIC_CAST(int16_t, a_.i32[i])); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtsepi32_epi16 + #define _mm512_cvtsepi32_epi16(a) simde_mm512_cvtsepi32_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_mask_cvtsepi32_epi16 (simde__m256i src, simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cvtsepi32_epi16(src, k, a); + #else + simde__m256i_private r_; + simde__m256i_private src_ = simde__m256i_to_private(src); + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i16[i] = ((k>>i) &1 ) ? + ((a_.i32[i] < INT16_MIN) + ? (INT16_MIN) + : ((a_.i32[i] > INT16_MAX) + ? (INT16_MAX) + : HEDLEY_STATIC_CAST(int16_t, a_.i32[i]))) : src_.i16[i]; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cvtsepi32_epi16 + #define _mm512_mask_cvtsepi32_epi16(src, k, a) simde_mm512_mask_cvtsepi32_epi16(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_mask_cvtsepi32_storeu_epi16 (void* base_addr, simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + _mm512_mask_cvtsepi32_storeu_epi16(base_addr, k, a); + #else + simde__m256i_private r_; + simde__m256i_private src_ = simde__m256i_to_private(simde_mm256_loadu_epi16(base_addr)); + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i16[i] = ((k>>i) &1 ) ? + ((a_.i32[i] < INT16_MIN) + ? (INT16_MIN) + : ((a_.i32[i] > INT16_MAX) + ? (INT16_MAX) + : HEDLEY_STATIC_CAST(int16_t, a_.i32[i]))) : src_.i16[i]; + } + + simde_mm256_storeu_epi16(base_addr, simde__m256i_from_private(r_)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cvtsepi32_storeu_epi16 + #define _mm512_mask_cvtsepi32_storeu_epi16(base_addr, k, a) simde_mm512_mask_cvtsepi32_storeu_epi16(base_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_maskz_cvtsepi32_epi16 (simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_cvtsepi32_epi16(k, a); + #else + simde__m256i_private r_; + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r_.i16[i] = ((k>>i) &1 ) ? + ((a_.i32[i] < INT16_MIN) + ? (INT16_MIN) + : ((a_.i32[i] > INT16_MAX) + ? (INT16_MAX) + : HEDLEY_STATIC_CAST(int16_t, a_.i32[i]))) : INT16_C(0); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_cvtsepi32_epi16 + #define _mm512_maskz_cvtsepi32_epi16(k, a) simde_mm512_maskz_cvtsepi32_epi16(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm512_cvtsepi64_epi8 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cvtsepi64_epi8(a); + #else + simde__m128i_private r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i8[i] = + (a_.i64[i] < INT8_MIN) + ? (INT8_MIN) + : ((a_.i64[i] > INT8_MAX) + ? (INT8_MAX) + : HEDLEY_STATIC_CAST(int8_t, a_.i64[i])); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtsepi64_epi8 + #define _mm512_cvtsepi64_epi8(a) simde_mm512_cvtsepi64_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm512_mask_cvtsepi64_epi8 (simde__m128i src, simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cvtsepi64_epi8(src, k, a); + #else + simde__m128i_private r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + simde__m128i_private src_ = simde__m128i_to_private(src); + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i8[i] = ((k>>i) &1 ) ? + ((a_.i64[i] < INT8_MIN) + ? (INT8_MIN) + : ((a_.i64[i] > INT8_MAX) + ? (INT8_MAX) + : HEDLEY_STATIC_CAST(int8_t, a_.i64[i]))) : src_.i8[i]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cvtsepi64_epi8 + #define _mm512_mask_cvtsepi64_epi8(src, k, a) simde_mm512_mask_cvtsepi64_epi8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm512_maskz_cvtsepi64_epi8 (simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_cvtsepi64_epi8(k, a); + #else + simde__m128i_private r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i8[i] = ((k>>i) &1 ) ? + ((a_.i64[i] < INT8_MIN) + ? (INT8_MIN) + : ((a_.i64[i] > INT8_MAX) + ? (INT8_MAX) + : HEDLEY_STATIC_CAST(int8_t, a_.i64[i]))) : INT8_C(0); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_cvtsepi64_epi8 + #define _mm512_maskz_cvtsepi64_epi8(k, a) simde_mm512_maskz_cvtsepi64_epi8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm512_cvtsepi64_epi16 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cvtsepi64_epi16(a); + #else + simde__m128i_private r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i16[i] = + (a_.i64[i] < INT16_MIN) + ? (INT16_MIN) + : ((a_.i64[i] > INT16_MAX) + ? (INT16_MAX) + : HEDLEY_STATIC_CAST(int16_t, a_.i64[i])); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtsepi64_epi16 + #define _mm512_cvtsepi64_epi16(a) simde_mm512_cvtsepi64_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm512_mask_cvtsepi64_epi16 (simde__m128i src, simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cvtsepi64_epi16(src, k, a); + #else + simde__m128i_private r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + simde__m128i_private src_ = simde__m128i_to_private(src); + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i16[i] = ((k>>i) & 1) ? + ((a_.i64[i] < INT16_MIN) + ? (INT16_MIN) + : ((a_.i64[i] > INT16_MAX) + ? (INT16_MAX) + : HEDLEY_STATIC_CAST(int16_t, a_.i64[i]))) : src_.i16[i]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cvtsepi64_epi16 + #define _mm512_mask_cvtsepi64_epi16(src, k, a) simde_mm512_mask_cvtsepi64_epi16(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm512_maskz_cvtsepi64_epi16 (simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_cvtsepi64_epi16(k, a); + #else + simde__m128i_private r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i16[i] = ((k>>i) & 1) ? + ((a_.i64[i] < INT16_MIN) + ? (INT16_MIN) + : ((a_.i64[i] > INT16_MAX) + ? (INT16_MAX) + : HEDLEY_STATIC_CAST(int16_t, a_.i64[i]))) : INT16_C(0); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_cvtsepi64_epi16 + #define _mm512_maskz_cvtsepi64_epi16(k, a) simde_mm512_maskz_cvtsepi64_epi16(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_cvtsepi64_epi32 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cvtsepi64_epi32(a); + #else + simde__m256i_private r_; + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i32[i] = + (a_.i64[i] < INT32_MIN) + ? (INT32_MIN) + : ((a_.i64[i] > INT32_MAX) + ? (INT32_MAX) + : HEDLEY_STATIC_CAST(int32_t, a_.i64[i])); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvtsepi64_epi32 + #define _mm512_cvtsepi64_epi32(a) simde_mm512_cvtsepi64_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_mask_cvtsepi64_epi32 (simde__m256i src, simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cvtsepi64_epi32(src, k, a); + #else + simde__m256i_private r_; + simde__m256i_private src_ = simde__m256i_to_private(src); + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i32[i] = ((k>>i) & 1) ? + ((a_.i64[i] < INT32_MIN) + ? (INT32_MIN) + : ((a_.i64[i] > INT32_MAX) + ? (INT32_MAX) + : HEDLEY_STATIC_CAST(int32_t, a_.i64[i]))) : src_.i32[i]; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cvtsepi64_epi32 + #define _mm512_mask_cvtsepi64_epi32(src, k, a) simde_mm512_mask_cvtsepi64_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_maskz_cvtsepi64_epi32 (simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_cvtsepi64_epi32(k, a); + #else + simde__m256i_private r_; + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r_.i32[i] = ((k>>i) & 1) ? + ((a_.i64[i] < INT32_MIN) + ? (INT32_MIN) + : ((a_.i64[i] > INT32_MAX) + ? (INT32_MAX) + : HEDLEY_STATIC_CAST(int32_t, a_.i64[i]))) : INT32_C(0); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_cvtsepi64_epi32 + #define _mm512_maskz_cvtsepi64_epi32(k, a) simde_mm512_maskz_cvtsepi64_epi32(k, a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_CVTS_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvtt.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvtt.h new file mode 100644 index 00000000..937f7fb7 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvtt.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2021 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_CVTT_H) +#define SIMDE_X86_AVX512_CVTT_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvttpd_epi64 (simde__m128d a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm_cvttpd_epi64(a); + #else + simde__m128i_private r_; + simde__m128d_private a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_ARCH_AMD64) + r_.n = + _mm_set_epi64x( + _mm_cvttsd_si64(_mm_unpackhi_pd(a_.n, a_.n)), + _mm_cvttsd_si64(a_.n) + ); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i64 = vcvtq_s64_f64(a_.neon_f64); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i64 = vec_signed(a_.altivec_f64); + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i64, a_.f64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = HEDLEY_STATIC_CAST(int64_t, a_.f64[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_cvttpd_epi64 + #define _mm_cvttpd_epi64(a) simde_mm_cvttpd_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_cvttpd_epi64(simde__m128i src, simde__mmask8 k, simde__m128d a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm_mask_cvttpd_epi64(src, k, a); + #else + return simde_mm_mask_mov_epi64(src, k, simde_mm_cvttpd_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_cvttpd_epi64 + #define _mm_mask_cvttpd_epi64(src, k, a) simde_mm_mask_cvttpd_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_cvttpd_epi64(simde__mmask8 k, simde__m128d a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm_maskz_cvttpd_epi64(k, a); + #else + return simde_mm_maskz_mov_epi64(k, simde_mm_cvttpd_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_cvttpd_epi64 + #define _mm_maskz_cvttpd_epi64(k, a) simde_mm_maskz_cvttpd_epi64(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_cvttps_epi32 (simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cvttps_epi32(a); + #else + simde__m512i_private r_; + simde__m512_private a_ = simde__m512_to_private(a); + + #if defined(simde_math_truncf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = SIMDE_CONVERT_FTOI(int32_t, simde_math_truncf(a_.f32[i])); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_cvttps_epi32 + #define _mm512_cvttps_epi32(a) simde_mm512_cvttps_epi32(a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_CVTT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvtus.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvtus.h new file mode 100644 index 00000000..ce423f6c --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/cvtus.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2023 Michael R. Crusoe + */ + +#if !defined(SIMDE_X86_AVX512_CVTUS_H) +#define SIMDE_X86_AVX512_CVTUS_H + +#include "types.h" +#include "mov.h" +#include "storeu.h" +#include "loadu.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_mask_cvtusepi32_storeu_epi8 (void* base_addr, simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + _mm512_mask_cvtusepi32_storeu_epi8(base_addr, k, a); + #else + simde__m256i_private r_ = simde__m256i_to_private(simde_mm256_loadu_epi8(base_addr)); + simde__m512i_private a_ = simde__m512i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + r_.i8[i] = ((k>>i) &1 ) ? + ((a_.u32[i] > UINT8_MAX) + ? (HEDLEY_STATIC_CAST(int8_t, UINT8_MAX)) + : HEDLEY_STATIC_CAST(int8_t, a_.u32[i])) : r_.i8[i]; + } + + simde_mm256_storeu_epi8(base_addr, simde__m256i_from_private(r_)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cvtusepi32_storeu_epi8 + #define _mm512_mask_cvtusepi32_storeu_epi8(base_addr, k, a) simde_mm512_mask_cvtusepi32_storeu_epi8((base_addr), (k), (a)) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_CVTUS_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dbsad.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dbsad.h new file mode 100644 index 00000000..c9a8e660 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dbsad.h @@ -0,0 +1,388 @@ +#if !defined(SIMDE_X86_AVX512_DBSAD_H) +#define SIMDE_X86_AVX512_DBSAD_H + +#include "types.h" +#include "mov.h" +#include "../avx2.h" +#include "shuffle.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_dbsad_epu8(a, b, imm8) _mm_dbsad_epu8((a), (b), (imm8)) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128i + simde_mm_dbsad_epu8_internal_ (simde__m128i a, simde__m128i b) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) + uint8_t a1 SIMDE_VECTOR(16) = + SIMDE_SHUFFLE_VECTOR_( + 8, 16, a_.u8, a_.u8, + 0, 1, 0, 1, + 4, 5, 4, 5, + 8, 9, 8, 9, + 12, 13, 12, 13); + uint8_t b1 SIMDE_VECTOR(16) = + SIMDE_SHUFFLE_VECTOR_( + 8, 16, b_.u8, b_.u8, + 0, 1, 1, 2, + 2, 3, 3, 4, + 8, 9, 9, 10, + 10, 11, 11, 12); + + __typeof__(r_.u8) abd1_mask = HEDLEY_REINTERPRET_CAST(__typeof__(abd1_mask), a1 < b1); + __typeof__(r_.u8) abd1 = (((b1 - a1) & abd1_mask) | ((a1 - b1) & ~abd1_mask)); + + r_.u16 = + __builtin_convertvector(__builtin_shufflevector(abd1, abd1, 0, 2, 4, 6, 8, 10, 12, 14), __typeof__(r_.u16)) + + __builtin_convertvector(__builtin_shufflevector(abd1, abd1, 1, 3, 5, 7, 9, 11, 13, 15), __typeof__(r_.u16)); + + uint8_t a2 SIMDE_VECTOR(16) = + SIMDE_SHUFFLE_VECTOR_( + 8, 16, a_.u8, a_.u8, + 2, 3, 2, 3, + 6, 7, 6, 7, + 10, 11, 10, 11, + 14, 15, 14, 15); + uint8_t b2 SIMDE_VECTOR(16) = + SIMDE_SHUFFLE_VECTOR_( + 8, 16, b_.u8, b_.u8, + 2, 3, 3, 4, + 4, 5, 5, 6, + 10, 11, 11, 12, + 12, 13, 13, 14); + + __typeof__(r_.u8) abd2_mask = HEDLEY_REINTERPRET_CAST(__typeof__(abd2_mask), a2 < b2); + __typeof__(r_.u8) abd2 = (((b2 - a2) & abd2_mask) | ((a2 - b2) & ~abd2_mask)); + + r_.u16 += + __builtin_convertvector(__builtin_shufflevector(abd2, abd2, 0, 2, 4, 6, 8, 10, 12, 14), __typeof__(r_.u16)) + + __builtin_convertvector(__builtin_shufflevector(abd2, abd2, 1, 3, 5, 7, 9, 11, 13, 15), __typeof__(r_.u16)); + #else + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = 0; + for (size_t j = 0 ; j < 4 ; j++) { + uint16_t A = HEDLEY_STATIC_CAST(uint16_t, a_.u8[((i << 1) & 12) + j]); + uint16_t B = HEDLEY_STATIC_CAST(uint16_t, b_.u8[((i & 3) | ((i << 1) & 8)) + j]); + r_.u16[i] += (A < B) ? (B - A) : (A - B); + } + } + #endif + + return simde__m128i_from_private(r_); + } + #define simde_mm_dbsad_epu8(a, b, imm8) simde_mm_dbsad_epu8_internal_((a), simde_mm_shuffle_epi32((b), (imm8))) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_dbsad_epu8 + #define _mm_dbsad_epu8(a, b, imm8) simde_mm_dbsad_epu8(a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_mask_dbsad_epu8(src, k, a, b, imm8) _mm_mask_dbsad_epu8((src), (k), (a), (b), (imm8)) +#else + #define simde_mm_mask_dbsad_epu8(src, k, a, b, imm8) simde_mm_mask_mov_epi16(src, k, simde_mm_dbsad_epu8(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_dbsad_epu8 + #define _mm_mask_dbsad_epu8(src, k, a, b, imm8) simde_mm_mask_dbsad_epu8(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_maskz_dbsad_epu8(k, a, b, imm8) _mm_maskz_dbsad_epu8((k), (a), (b), (imm8)) +#else + #define simde_mm_maskz_dbsad_epu8(k, a, b, imm8) simde_mm_maskz_mov_epi16(k, simde_mm_dbsad_epu8(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_dbsad_epu8 + #define _mm_maskz_dbsad_epu8(k, a, b, imm8) simde_mm_maskz_dbsad_epu8(k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_dbsad_epu8(a, b, imm8) _mm256_dbsad_epu8((a), (b), (imm8)) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm256_dbsad_epu8(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m256i_private \ + simde_mm256_dbsad_epu8_a_ = simde__m256i_to_private(a), \ + simde_mm256_dbsad_epu8_b_ = simde__m256i_to_private(b); \ + \ + simde_mm256_dbsad_epu8_a_.m128i[0] = simde_mm_dbsad_epu8(simde_mm256_dbsad_epu8_a_.m128i[0], simde_mm256_dbsad_epu8_b_.m128i[0], imm8); \ + simde_mm256_dbsad_epu8_a_.m128i[1] = simde_mm_dbsad_epu8(simde_mm256_dbsad_epu8_a_.m128i[1], simde_mm256_dbsad_epu8_b_.m128i[1], imm8); \ + \ + simde__m256i_from_private(simde_mm256_dbsad_epu8_a_); \ + })) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m256i + simde_mm256_dbsad_epu8_internal_ (simde__m256i a, simde__m256i b) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) + uint8_t a1 SIMDE_VECTOR(32) = + SIMDE_SHUFFLE_VECTOR_( + 8, 32, a_.u8, a_.u8, + 0, 1, 0, 1, + 4, 5, 4, 5, + 8, 9, 8, 9, + 12, 13, 12, 13, + 16, 17, 16, 17, + 20, 21, 20, 21, + 24, 25, 24, 25, + 28, 29, 28, 29); + uint8_t b1 SIMDE_VECTOR(32) = + SIMDE_SHUFFLE_VECTOR_( + 8, 16, b_.u8, b_.u8, + 0, 1, 1, 2, + 2, 3, 3, 4, + 8, 9, 9, 10, + 10, 11, 11, 12, + 16, 17, 17, 18, + 18, 19, 19, 20, + 24, 25, 25, 26, + 26, 27, 27, 28); + + __typeof__(r_.u8) abd1_mask = HEDLEY_REINTERPRET_CAST(__typeof__(abd1_mask), a1 < b1); + __typeof__(r_.u8) abd1 = (((b1 - a1) & abd1_mask) | ((a1 - b1) & ~abd1_mask)); + + r_.u16 = + __builtin_convertvector(__builtin_shufflevector(abd1, abd1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), __typeof__(r_.u16)) + + __builtin_convertvector(__builtin_shufflevector(abd1, abd1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31), __typeof__(r_.u16)); + + uint8_t a2 SIMDE_VECTOR(32) = + SIMDE_SHUFFLE_VECTOR_( + 8, 32, a_.u8, a_.u8, + 2, 3, 2, 3, + 6, 7, 6, 7, + 10, 11, 10, 11, + 14, 15, 14, 15, + 18, 19, 18, 19, + 22, 23, 22, 23, + 26, 27, 26, 27, + 30, 31, 30, 31); + uint8_t b2 SIMDE_VECTOR(32) = + SIMDE_SHUFFLE_VECTOR_( + 8, 16, b_.u8, b_.u8, + 2, 3, 3, 4, + 4, 5, 5, 6, + 10, 11, 11, 12, + 12, 13, 13, 14, + 18, 19, 19, 20, + 20, 21, 21, 22, + 26, 27, 27, 28, + 28, 29, 29, 30); + + __typeof__(r_.u8) abd2_mask = HEDLEY_REINTERPRET_CAST(__typeof__(abd2_mask), a2 < b2); + __typeof__(r_.u8) abd2 = (((b2 - a2) & abd2_mask) | ((a2 - b2) & ~abd2_mask)); + + r_.u16 += + __builtin_convertvector(__builtin_shufflevector(abd2, abd2, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30), __typeof__(r_.u16)) + + __builtin_convertvector(__builtin_shufflevector(abd2, abd2, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31), __typeof__(r_.u16)); + #else + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = 0; + for (size_t j = 0 ; j < 4 ; j++) { + uint16_t A = HEDLEY_STATIC_CAST(uint16_t, a_.u8[(((i << 1) & 12) | ((i & 8) << 1)) + j]); + uint16_t B = HEDLEY_STATIC_CAST(uint16_t, b_.u8[((i & 3) | ((i << 1) & 8) | ((i & 8) << 1)) + j]); + r_.u16[i] += (A < B) ? (B - A) : (A - B); + } + } + #endif + + return simde__m256i_from_private(r_); + } + #define simde_mm256_dbsad_epu8(a, b, imm8) simde_mm256_dbsad_epu8_internal_((a), simde_mm256_shuffle_epi32(b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_dbsad_epu8 + #define _mm256_dbsad_epu8(a, b, imm8) simde_mm256_dbsad_epu8(a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_dbsad_epu8(src, k, a, b, imm8) _mm256_mask_dbsad_epu8((src), (k), (a), (b), (imm8)) +#else + #define simde_mm256_mask_dbsad_epu8(src, k, a, b, imm8) simde_mm256_mask_mov_epi16(src, k, simde_mm256_dbsad_epu8(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_dbsad_epu8 + #define _mm256_mask_dbsad_epu8(src, k, a, b, imm8) simde_mm256_mask_dbsad_epu8(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_maskz_dbsad_epu8(k, a, b, imm8) _mm256_maskz_dbsad_epu8((k), (a), (b), (imm8)) +#else + #define simde_mm256_maskz_dbsad_epu8(k, a, b, imm8) simde_mm256_maskz_mov_epi16(k, simde_mm256_dbsad_epu8(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_dbsad_epu8 + #define _mm256_maskz_dbsad_epu8(k, a, b, imm8) simde_mm256_maskz_dbsad_epu8(k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512BW_NATIVE) + #define simde_mm512_dbsad_epu8(a, b, imm8) _mm512_dbsad_epu8((a), (b), (imm8)) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_dbsad_epu8(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512i_private \ + simde_mm512_dbsad_epu8_a_ = simde__m512i_to_private(a), \ + simde_mm512_dbsad_epu8_b_ = simde__m512i_to_private(b); \ + \ + simde_mm512_dbsad_epu8_a_.m256i[0] = simde_mm256_dbsad_epu8(simde_mm512_dbsad_epu8_a_.m256i[0], simde_mm512_dbsad_epu8_b_.m256i[0], imm8); \ + simde_mm512_dbsad_epu8_a_.m256i[1] = simde_mm256_dbsad_epu8(simde_mm512_dbsad_epu8_a_.m256i[1], simde_mm512_dbsad_epu8_b_.m256i[1], imm8); \ + \ + simde__m512i_from_private(simde_mm512_dbsad_epu8_a_); \ + })) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512i + simde_mm512_dbsad_epu8_internal_ (simde__m512i a, simde__m512i b) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) + uint8_t a1 SIMDE_VECTOR(64) = + SIMDE_SHUFFLE_VECTOR_( + 8, 64, a_.u8, a_.u8, + 0, 1, 0, 1, + 4, 5, 4, 5, + 8, 9, 8, 9, + 12, 13, 12, 13, + 16, 17, 16, 17, + 20, 21, 20, 21, + 24, 25, 24, 25, + 28, 29, 28, 29, + 32, 33, 32, 33, + 36, 37, 36, 37, + 40, 41, 40, 41, + 44, 45, 44, 45, + 48, 49, 48, 49, + 52, 53, 52, 53, + 56, 57, 56, 57, + 60, 61, 60, 61); + uint8_t b1 SIMDE_VECTOR(64) = + SIMDE_SHUFFLE_VECTOR_( + 8, 64, b_.u8, b_.u8, + 0, 1, 1, 2, + 2, 3, 3, 4, + 8, 9, 9, 10, + 10, 11, 11, 12, + 16, 17, 17, 18, + 18, 19, 19, 20, + 24, 25, 25, 26, + 26, 27, 27, 28, + 32, 33, 33, 34, + 34, 35, 35, 36, + 40, 41, 41, 42, + 42, 43, 43, 44, + 48, 49, 49, 50, + 50, 51, 51, 52, + 56, 57, 57, 58, + 58, 59, 59, 60); + + __typeof__(r_.u8) abd1_mask = HEDLEY_REINTERPRET_CAST(__typeof__(abd1_mask), a1 < b1); + __typeof__(r_.u8) abd1 = (((b1 - a1) & abd1_mask) | ((a1 - b1) & ~abd1_mask)); + + r_.u16 = + __builtin_convertvector(__builtin_shufflevector(abd1, abd1, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62), __typeof__(r_.u16)) + + __builtin_convertvector(__builtin_shufflevector(abd1, abd1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63), __typeof__(r_.u16)); + + uint8_t a2 SIMDE_VECTOR(64) = + SIMDE_SHUFFLE_VECTOR_( + 8, 64, a_.u8, a_.u8, + 2, 3, 2, 3, + 6, 7, 6, 7, + 10, 11, 10, 11, + 14, 15, 14, 15, + 18, 19, 18, 19, + 22, 23, 22, 23, + 26, 27, 26, 27, + 30, 31, 30, 31, + 34, 35, 34, 35, + 38, 39, 38, 39, + 42, 43, 42, 43, + 46, 47, 46, 47, + 50, 51, 50, 51, + 54, 55, 54, 55, + 58, 59, 58, 59, + 62, 63, 62, 63); + uint8_t b2 SIMDE_VECTOR(64) = + SIMDE_SHUFFLE_VECTOR_( + 8, 64, b_.u8, b_.u8, + 2, 3, 3, 4, + 4, 5, 5, 6, + 10, 11, 11, 12, + 12, 13, 13, 14, + 18, 19, 19, 20, + 20, 21, 21, 22, + 26, 27, 27, 28, + 28, 29, 29, 30, + 34, 35, 35, 36, + 36, 37, 37, 38, + 42, 43, 43, 44, + 44, 45, 45, 46, + 50, 51, 51, 52, + 52, 53, 53, 54, + 58, 59, 59, 60, + 60, 61, 61, 62); + + __typeof__(r_.u8) abd2_mask = HEDLEY_REINTERPRET_CAST(__typeof__(abd2_mask), a2 < b2); + __typeof__(r_.u8) abd2 = (((b2 - a2) & abd2_mask) | ((a2 - b2) & ~abd2_mask)); + + r_.u16 += + __builtin_convertvector(__builtin_shufflevector(abd2, abd2, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62), __typeof__(r_.u16)) + + __builtin_convertvector(__builtin_shufflevector(abd2, abd2, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63), __typeof__(r_.u16)); + #else + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = 0; + for (size_t j = 0 ; j < 4 ; j++) { + uint16_t A = HEDLEY_STATIC_CAST(uint16_t, a_.u8[(((i << 1) & 12) | ((i & 8) << 1) | ((i & 16) << 1)) + j]); + uint16_t B = HEDLEY_STATIC_CAST(uint16_t, b_.u8[((i & 3) | ((i << 1) & 8) | ((i & 8) << 1) | ((i & 16) << 1)) + j]); + r_.u16[i] += (A < B) ? (B - A) : (A - B); + } + } + #endif + + return simde__m512i_from_private(r_); + } + #define simde_mm512_dbsad_epu8(a, b, imm8) simde_mm512_dbsad_epu8_internal_((a), simde_mm512_castps_si512(simde_mm512_shuffle_ps(simde_mm512_castsi512_ps(b), simde_mm512_castsi512_ps(b), imm8))) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_dbsad_epu8 + #define _mm512_dbsad_epu8(a, b, imm8) simde_mm512_dbsad_epu8(a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512BW_NATIVE) + #define simde_mm512_mask_dbsad_epu8(src, k, a, b, imm8) _mm512_mask_dbsad_epu8((src), (k), (a), (b), (imm8)) +#else + #define simde_mm512_mask_dbsad_epu8(src, k, a, b, imm8) simde_mm512_mask_mov_epi16(src, k, simde_mm512_dbsad_epu8(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_dbsad_epu8 + #define _mm512_mask_dbsad_epu8(src, k, a, b, imm8) simde_mm512_mask_dbsad_epu8(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512BW_NATIVE) + #define simde_mm512_maskz_dbsad_epu8(k, a, b, imm8) _mm512_maskz_dbsad_epu8((k), (a), (b), (imm8)) +#else + #define simde_mm512_maskz_dbsad_epu8(k, a, b, imm8) simde_mm512_maskz_mov_epi16(k, simde_mm512_dbsad_epu8(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_dbsad_epu8 + #define _mm512_maskz_dbsad_epu8(k, a, b, imm8) simde_mm512_maskz_dbsad_epu8(k, a, b, imm8) +#endif + + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_DBSAD_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/div.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/div.h new file mode 100644 index 00000000..5e6349a2 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/div.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_DIV_H) +#define SIMDE_X86_AVX512_DIV_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_div_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_div_ps(a, b); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_div_ps(a_.m256[i], b_.m256[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = a_.f32 / b_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_div_ps(a_.m256[i], b_.m256[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_div_ps + #define _mm512_div_ps(a, b) simde_mm512_div_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_div_ps(simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_div_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_div_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_div_ps + #define _mm512_mask_div_ps(src, k, a, b) simde_mm512_mask_div_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_div_ps(simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_div_ps(k, a, b); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_div_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_div_ps + #define _mm512_maskz_div_ps(k, a, b) simde_mm512_maskz_div_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_div_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_div_pd(a, b); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_div_pd(a_.m256d[i], b_.m256d[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = a_.f64 / b_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_div_pd(a_.m256d[i], b_.m256d[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_div_pd + #define _mm512_div_pd(a, b) simde_mm512_div_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_div_pd(simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_div_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_div_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_div_pd + #define _mm512_mask_div_pd(src, k, a, b) simde_mm512_mask_div_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_div_pd(simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_div_pd(k, a, b); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_div_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_div_pd + #define _mm512_maskz_div_pd(k, a, b) simde_mm512_maskz_div_pd(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_DIV_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpbf16.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpbf16.h new file mode 100644 index 00000000..81e2aead --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpbf16.h @@ -0,0 +1,281 @@ +#if !defined(SIMDE_X86_AVX512_DPBF16_H) +#define SIMDE_X86_AVX512_DPBF16_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_dpbf16_ps (simde__m128 src, simde__m128bh a, simde__m128bh b) { + #if defined(SIMDE_X86_AVX512BF16_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_dpbf16_ps(src, a, b); + #else + simde__m128_private + src_ = simde__m128_to_private(src); + simde__m128bh_private + a_ = simde__m128bh_to_private(a), + b_ = simde__m128bh_to_private(b); + + #if ! ( defined(SIMDE_ARCH_X86) && defined(HEDLEY_GCC_VERSION) ) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && defined(SIMDE_CONVERT_VECTOR_) && defined(SIMDE_SHUFFLE_VECTOR_) + uint32_t x1 SIMDE_VECTOR(32); + uint32_t x2 SIMDE_VECTOR(32); + simde__m128_private + r1_[2], + r2_[2]; + + a_.u16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 16, + a_.u16, a_.u16, + 0, 2, 4, 6, + 1, 3, 5, 7 + ); + b_.u16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 16, + b_.u16, b_.u16, + 0, 2, 4, 6, + 1, 3, 5, 7 + ); + + SIMDE_CONVERT_VECTOR_(x1, a_.u16); + SIMDE_CONVERT_VECTOR_(x2, b_.u16); + + x1 <<= 16; + x2 <<= 16; + + simde_memcpy(&r1_, &x1, sizeof(x1)); + simde_memcpy(&r2_, &x2, sizeof(x2)); + + src_.f32 += + HEDLEY_REINTERPRET_CAST(__typeof__(a_.f32), r1_[0].u32) * HEDLEY_REINTERPRET_CAST(__typeof__(a_.f32), r2_[0].u32) + + HEDLEY_REINTERPRET_CAST(__typeof__(a_.f32), r1_[1].u32) * HEDLEY_REINTERPRET_CAST(__typeof__(a_.f32), r2_[1].u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u16) / sizeof(a_.u16[0])) ; i++) { + src_.f32[i / 2] += (simde_uint32_as_float32(HEDLEY_STATIC_CAST(uint32_t, a_.u16[i]) << 16) * simde_uint32_as_float32(HEDLEY_STATIC_CAST(uint32_t, b_.u16[i]) << 16)); + } + #endif + + return simde__m128_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512BF16_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_dpbf16_ps + #define _mm_dpbf16_ps(src, a, b) simde_mm_dpbf16_ps(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mask_dpbf16_ps (simde__m128 src, simde__mmask8 k, simde__m128bh a, simde__m128bh b) { + #if defined(SIMDE_X86_AVX512BF16_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_dpbf16_ps(src, k, a, b); + #else + return simde_mm_mask_mov_ps(src, k, simde_mm_dpbf16_ps(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BF16_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_dpbf16_ps + #define _mm_mask_dpbf16_ps(src, k, a, b) simde_mm_mask_dpbf16_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_maskz_dpbf16_ps (simde__mmask8 k, simde__m128 src, simde__m128bh a, simde__m128bh b) { + #if defined(SIMDE_X86_AVX512BF16_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_dpbf16_ps(k, src, a, b); + #else + return simde_mm_maskz_mov_ps(k, simde_mm_dpbf16_ps(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BF16_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_dpbf16_ps + #define _mm_maskz_dpbf16_ps(k, src, a, b) simde_mm_maskz_dpbf16_ps(k, src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_dpbf16_ps (simde__m256 src, simde__m256bh a, simde__m256bh b) { + #if defined(SIMDE_X86_AVX512BF16_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_dpbf16_ps(src, a, b); + #else + simde__m256_private + src_ = simde__m256_to_private(src); + simde__m256bh_private + a_ = simde__m256bh_to_private(a), + b_ = simde__m256bh_to_private(b); + + #if ! ( defined(SIMDE_ARCH_X86) && defined(HEDLEY_GCC_VERSION) ) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && defined(SIMDE_CONVERT_VECTOR_) && defined(SIMDE_SHUFFLE_VECTOR_) + uint32_t x1 SIMDE_VECTOR(64); + uint32_t x2 SIMDE_VECTOR(64); + simde__m256_private + r1_[2], + r2_[2]; + + a_.u16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 32, + a_.u16, a_.u16, + 0, 2, 4, 6, 8, 10, 12, 14, + 1, 3, 5, 7, 9, 11, 13, 15 + ); + b_.u16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 32, + b_.u16, b_.u16, + 0, 2, 4, 6, 8, 10, 12, 14, + 1, 3, 5, 7, 9, 11, 13, 15 + ); + + SIMDE_CONVERT_VECTOR_(x1, a_.u16); + SIMDE_CONVERT_VECTOR_(x2, b_.u16); + + x1 <<= 16; + x2 <<= 16; + + simde_memcpy(&r1_, &x1, sizeof(x1)); + simde_memcpy(&r2_, &x2, sizeof(x2)); + + src_.f32 += + HEDLEY_REINTERPRET_CAST(__typeof__(a_.f32), r1_[0].u32) * HEDLEY_REINTERPRET_CAST(__typeof__(a_.f32), r2_[0].u32) + + HEDLEY_REINTERPRET_CAST(__typeof__(a_.f32), r1_[1].u32) * HEDLEY_REINTERPRET_CAST(__typeof__(a_.f32), r2_[1].u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u16) / sizeof(a_.u16[0])) ; i++) { + src_.f32[i / 2] += (simde_uint32_as_float32(HEDLEY_STATIC_CAST(uint32_t, a_.u16[i]) << 16) * simde_uint32_as_float32(HEDLEY_STATIC_CAST(uint32_t, b_.u16[i]) << 16)); + } + #endif + + return simde__m256_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512BF16_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_dpbf16_ps + #define _mm256_dpbf16_ps(src, a, b) simde_mm256_dpbf16_ps(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask_dpbf16_ps (simde__m256 src, simde__mmask8 k, simde__m256bh a, simde__m256bh b) { + #if defined(SIMDE_X86_AVX512BF16_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_dpbf16_ps(src, k, a, b); + #else + return simde_mm256_mask_mov_ps(src, k, simde_mm256_dpbf16_ps(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BF16_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_dpbf16_ps + #define _mm256_mask_dpbf16_ps(src, k, a, b) simde_mm256_mask_dpbf16_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_maskz_dpbf16_ps (simde__mmask8 k, simde__m256 src, simde__m256bh a, simde__m256bh b) { + #if defined(SIMDE_X86_AVX512BF16_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_dpbf16_ps(k, src, a, b); + #else + return simde_mm256_maskz_mov_ps(k, simde_mm256_dpbf16_ps(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BF16_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_dpbf16_ps + #define _mm256_maskz_dpbf16_ps(k, src, a, b) simde_mm256_maskz_dpbf16_ps(k, src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_dpbf16_ps (simde__m512 src, simde__m512bh a, simde__m512bh b) { + #if defined(SIMDE_X86_AVX512BF16_NATIVE) + return _mm512_dpbf16_ps(src, a, b); + #else + simde__m512_private + src_ = simde__m512_to_private(src); + simde__m512bh_private + a_ = simde__m512bh_to_private(a), + b_ = simde__m512bh_to_private(b); + + #if ! ( defined(SIMDE_ARCH_X86) && defined(HEDLEY_GCC_VERSION) ) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && defined(SIMDE_CONVERT_VECTOR_) && defined(SIMDE_SHUFFLE_VECTOR_) + uint32_t x1 SIMDE_VECTOR(128); + uint32_t x2 SIMDE_VECTOR(128); + simde__m512_private + r1_[2], + r2_[2]; + + a_.u16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 64, + a_.u16, a_.u16, + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 + ); + b_.u16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 64, + b_.u16, b_.u16, + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 + ); + + SIMDE_CONVERT_VECTOR_(x1, a_.u16); + SIMDE_CONVERT_VECTOR_(x2, b_.u16); + + x1 <<= 16; + x2 <<= 16; + + simde_memcpy(&r1_, &x1, sizeof(x1)); + simde_memcpy(&r2_, &x2, sizeof(x2)); + + src_.f32 += + HEDLEY_REINTERPRET_CAST(__typeof__(a_.f32), r1_[0].u32) * HEDLEY_REINTERPRET_CAST(__typeof__(a_.f32), r2_[0].u32) + + HEDLEY_REINTERPRET_CAST(__typeof__(a_.f32), r1_[1].u32) * HEDLEY_REINTERPRET_CAST(__typeof__(a_.f32), r2_[1].u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u16) / sizeof(a_.u16[0])) ; i++) { + src_.f32[i / 2] += (simde_uint32_as_float32(HEDLEY_STATIC_CAST(uint32_t, a_.u16[i]) << 16) * simde_uint32_as_float32(HEDLEY_STATIC_CAST(uint32_t, b_.u16[i]) << 16)); + } + #endif + + return simde__m512_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512BF16_ENABLE_NATIVE_ALIASES) + #undef _mm512_dpbf16_ps + #define _mm512_dpbf16_ps(src, a, b) simde_mm512_dpbf16_ps(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_dpbf16_ps (simde__m512 src, simde__mmask16 k, simde__m512bh a, simde__m512bh b) { + #if defined(SIMDE_X86_AVX512BF16_NATIVE) + return _mm512_mask_dpbf16_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_dpbf16_ps(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BF16_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_dpbf16_ps + #define _mm512_mask_dpbf16_ps(src, k, a, b) simde_mm512_mask_dpbf16_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_dpbf16_ps (simde__mmask16 k, simde__m512 src, simde__m512bh a, simde__m512bh b) { + #if defined(SIMDE_X86_AVX512BF16_NATIVE) + return _mm512_maskz_dpbf16_ps(k, src, a, b); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_dpbf16_ps(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BF16_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_dpbf16_ps + #define _mm512_maskz_dpbf16_ps(k, src, a, b) simde_mm512_maskz_dpbf16_ps(k, src, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_DPBF16_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpbusd.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpbusd.h new file mode 100644 index 00000000..c45f3ca3 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpbusd.h @@ -0,0 +1,292 @@ +#if !defined(SIMDE_X86_AVX512_DPBUSD_H) +#define SIMDE_X86_AVX512_DPBUSD_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_dpbusd_epi32(simde__m128i src, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm_dpbusd_epi32(src, a, b); + #else + simde__m128i_private + src_ = simde__m128i_to_private(src), + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + #if defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_CONVERT_VECTOR_) + uint32_t x1_ SIMDE_VECTOR(64); + int32_t x2_ SIMDE_VECTOR(64); + simde__m128i_private + r1_[4], + r2_[4]; + + a_.u8 = + SIMDE_SHUFFLE_VECTOR_( + 8, 16, + a_.u8, a_.u8, + 0, 4, 8, 12, + 1, 5, 9, 13, + 2, 6, 10, 14, + 3, 7, 11, 15 + ); + b_.i8 = + SIMDE_SHUFFLE_VECTOR_( + 8, 16, + b_.i8, b_.i8, + 0, 4, 8, 12, + 1, 5, 9, 13, + 2, 6, 10, 14, + 3, 7, 11, 15 + ); + + SIMDE_CONVERT_VECTOR_(x1_, a_.u8); + SIMDE_CONVERT_VECTOR_(x2_, b_.i8); + + simde_memcpy(&r1_, &x1_, sizeof(x1_)); + simde_memcpy(&r2_, &x2_, sizeof(x2_)); + + src_.i32 += + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[0].u32) * r2_[0].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[1].u32) * r2_[1].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[2].u32) * r2_[2].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[3].u32) * r2_[3].i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0])) ; i++) { + src_.i32[i / 4] += HEDLEY_STATIC_CAST(uint16_t, a_.u8[i]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[i]); + } + #endif + + return simde__m128i_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm_dpbusd_epi32 + #define _mm_dpbusd_epi32(src, a, b) simde_mm_dpbusd_epi32(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_dpbusd_epi32(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm_mask_dpbusd_epi32(src, k, a, b); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_dpbusd_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_dpbusd_epi32 + #define _mm_mask_dpbusd_epi32(src, k, a, b) simde_mm_mask_dpbusd_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_dpbusd_epi32(simde__mmask8 k, simde__m128i src, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm_maskz_dpbusd_epi32(k, src, a, b); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_dpbusd_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_dpbusd_epi32 + #define _mm_maskz_dpbusd_epi32(k, src, a, b) simde_mm_maskz_dpbusd_epi32(k, src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_dpbusd_epi32(simde__m256i src, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm256_dpbusd_epi32(src, a, b); + #else + simde__m256i_private + src_ = simde__m256i_to_private(src), + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + src_.m128i[0] = simde_mm_dpbusd_epi32(src_.m128i[0], a_.m128i[0], b_.m128i[0]); + src_.m128i[1] = simde_mm_dpbusd_epi32(src_.m128i[1], a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_CONVERT_VECTOR_) + uint32_t x1_ SIMDE_VECTOR(128); + int32_t x2_ SIMDE_VECTOR(128); + simde__m256i_private + r1_[4], + r2_[4]; + + a_.u8 = + SIMDE_SHUFFLE_VECTOR_( + 8, 32, + a_.u8, a_.u8, + 0, 4, 8, 12, 16, 20, 24, 28, + 1, 5, 9, 13, 17, 21, 25, 29, + 2, 6, 10, 14, 18, 22, 26, 30, + 3, 7, 11, 15, 19, 23, 27, 31 + ); + b_.i8 = + SIMDE_SHUFFLE_VECTOR_( + 8, 32, + b_.i8, b_.i8, + 0, 4, 8, 12, 16, 20, 24, 28, + 1, 5, 9, 13, 17, 21, 25, 29, + 2, 6, 10, 14, 18, 22, 26, 30, + 3, 7, 11, 15, 19, 23, 27, 31 + ); + + SIMDE_CONVERT_VECTOR_(x1_, a_.u8); + SIMDE_CONVERT_VECTOR_(x2_, b_.i8); + + simde_memcpy(&r1_, &x1_, sizeof(x1_)); + simde_memcpy(&r2_, &x2_, sizeof(x2_)); + + src_.i32 += + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[0].u32) * r2_[0].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[1].u32) * r2_[1].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[2].u32) * r2_[2].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[3].u32) * r2_[3].i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0])) ; i++) { + src_.i32[i / 4] += HEDLEY_STATIC_CAST(uint16_t, a_.u8[i]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[i]); + } + #endif + + return simde__m256i_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_dpbusd_epi32 + #define _mm256_dpbusd_epi32(src, a, b) simde_mm256_dpbusd_epi32(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_dpbusd_epi32(simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm256_mask_dpbusd_epi32(src, k, a, b); + #else + return simde_mm256_mask_mov_epi32(src, k, simde_mm256_dpbusd_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_dpbusd_epi32 + #define _mm256_mask_dpbusd_epi32(src, k, a, b) simde_mm256_mask_dpbusd_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_dpbusd_epi32(simde__mmask8 k, simde__m256i src, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm256_maskz_dpbusd_epi32(k, src, a, b); + #else + return simde_mm256_maskz_mov_epi32(k, simde_mm256_dpbusd_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_dpbusd_epi32 + #define _mm256_maskz_dpbusd_epi32(k, src, a, b) simde_mm256_maskz_dpbusd_epi32(k, src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_dpbusd_epi32(simde__m512i src, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm512_dpbusd_epi32(src, a, b); + #else + simde__m512i_private + src_ = simde__m512i_to_private(src), + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + src_.m256i[0] = simde_mm256_dpbusd_epi32(src_.m256i[0], a_.m256i[0], b_.m256i[0]); + src_.m256i[1] = simde_mm256_dpbusd_epi32(src_.m256i[1], a_.m256i[1], b_.m256i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_CONVERT_VECTOR_) + uint32_t x1_ SIMDE_VECTOR(256); + int32_t x2_ SIMDE_VECTOR(256); + simde__m512i_private + r1_[4], + r2_[4]; + + a_.u8 = + SIMDE_SHUFFLE_VECTOR_( + 8, 64, + a_.u8, a_.u8, + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, + 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, + 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62, + 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63 + ); + b_.i8 = + SIMDE_SHUFFLE_VECTOR_( + 8, 64, + b_.i8, b_.i8, + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, + 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, + 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62, + 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63 + ); + + SIMDE_CONVERT_VECTOR_(x1_, a_.u8); + SIMDE_CONVERT_VECTOR_(x2_, b_.i8); + + simde_memcpy(&r1_, &x1_, sizeof(x1_)); + simde_memcpy(&r2_, &x2_, sizeof(x2_)); + + src_.i32 += + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[0].u32) * r2_[0].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[1].u32) * r2_[1].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[2].u32) * r2_[2].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[3].u32) * r2_[3].i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0])) ; i++) { + src_.i32[i / 4] += HEDLEY_STATIC_CAST(uint16_t, a_.u8[i]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[i]); + } + #endif + + return simde__m512i_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_dpbusd_epi32 + #define _mm512_dpbusd_epi32(src, a, b) simde_mm512_dpbusd_epi32(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_dpbusd_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm512_mask_dpbusd_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_dpbusd_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_dpbusd_epi32 + #define _mm512_mask_dpbusd_epi32(src, k, a, b) simde_mm512_mask_dpbusd_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_dpbusd_epi32(simde__mmask16 k, simde__m512i src, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm512_maskz_dpbusd_epi32(k, src, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_dpbusd_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_dpbusd_epi32 + #define _mm512_maskz_dpbusd_epi32(k, src, a, b) simde_mm512_maskz_dpbusd_epi32(k, src, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_DPBUSD_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpbusds.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpbusds.h new file mode 100644 index 00000000..0168fed2 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpbusds.h @@ -0,0 +1,344 @@ +#if !defined(SIMDE_X86_AVX512_DPBUSDS_H) +#define SIMDE_X86_AVX512_DPBUSDS_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_dpbusds_epi32(simde__m128i src, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm_dpbusds_epi32(src, a, b); + #else + simde__m128i_private + src_ = simde__m128i_to_private(src), + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_CONVERT_VECTOR_) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + uint32_t x1_ SIMDE_VECTOR(64); + int32_t x2_ SIMDE_VECTOR(64); + simde__m128i_private + r1_[4], + r2_[4]; + + a_.u8 = + SIMDE_SHUFFLE_VECTOR_( + 8, 16, + a_.u8, a_.u8, + 0, 4, 8, 12, + 1, 5, 9, 13, + 2, 6, 10, 14, + 3, 7, 11, 15 + ); + b_.i8 = + SIMDE_SHUFFLE_VECTOR_( + 8, 16, + b_.i8, b_.i8, + 0, 4, 8, 12, + 1, 5, 9, 13, + 2, 6, 10, 14, + 3, 7, 11, 15 + ); + + SIMDE_CONVERT_VECTOR_(x1_, a_.u8); + SIMDE_CONVERT_VECTOR_(x2_, b_.i8); + + simde_memcpy(&r1_, &x1_, sizeof(x1_)); + simde_memcpy(&r2_, &x2_, sizeof(x2_)); + + uint32_t au SIMDE_VECTOR(16) = + HEDLEY_REINTERPRET_CAST( + __typeof__(au), + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[0].u32) * r2_[0].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[1].u32) * r2_[1].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[2].u32) * r2_[2].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[3].u32) * r2_[3].i32) + ); + uint32_t bu SIMDE_VECTOR(16) = HEDLEY_REINTERPRET_CAST(__typeof__(bu), src_.i32); + uint32_t ru SIMDE_VECTOR(16) = au + bu; + + au = (au >> 31) + INT32_MAX; + + uint32_t m SIMDE_VECTOR(16) = HEDLEY_REINTERPRET_CAST(__typeof__(m), HEDLEY_REINTERPRET_CAST(__typeof__(src_.i32), (au ^ bu) | ~(bu ^ ru)) < 0); + src_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(src_.i32), (au & ~m) | (ru & m)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0]) / 4) ; i++) { + src_.i32[i] = + simde_math_adds_i32( + src_.i32[i], + HEDLEY_STATIC_CAST(uint16_t, a_.u8[(4 * i) ]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[(4 * i) ]) + + HEDLEY_STATIC_CAST(uint16_t, a_.u8[(4 * i) + 1]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[(4 * i) + 1]) + + HEDLEY_STATIC_CAST(uint16_t, a_.u8[(4 * i) + 2]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[(4 * i) + 2]) + + HEDLEY_STATIC_CAST(uint16_t, a_.u8[(4 * i) + 3]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[(4 * i) + 3]) + ); + } + #endif + + return simde__m128i_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm_dpbusds_epi32 + #define _mm_dpbusds_epi32(src, a, b) simde_mm_dpbusds_epi32(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_dpbusds_epi32(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm_mask_dpbusds_epi32(src, k, a, b); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_dpbusds_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_dpbusds_epi32 + #define _mm_mask_dpbusds_epi32(src, k, a, b) simde_mm_mask_dpbusds_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_dpbusds_epi32(simde__mmask8 k, simde__m128i src, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm_maskz_dpbusds_epi32(k, src, a, b); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_dpbusds_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_dpbusds_epi32 + #define _mm_maskz_dpbusds_epi32(k, src, a, b) simde_mm_maskz_dpbusds_epi32(k, src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_dpbusds_epi32(simde__m256i src, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm256_dpbusds_epi32(src, a, b); + #else + simde__m256i_private + src_ = simde__m256i_to_private(src), + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + src_.m128i[0] = simde_mm_dpbusds_epi32(src_.m128i[0], a_.m128i[0], b_.m128i[0]); + src_.m128i[1] = simde_mm_dpbusds_epi32(src_.m128i[1], a_.m128i[1], b_.m128i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_CONVERT_VECTOR_) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + uint32_t x1_ SIMDE_VECTOR(128); + int32_t x2_ SIMDE_VECTOR(128); + simde__m256i_private + r1_[4], + r2_[4]; + + a_.u8 = + SIMDE_SHUFFLE_VECTOR_( + 8, 32, + a_.u8, a_.u8, + 0, 4, 8, 12, 16, 20, 24, 28, + 1, 5, 9, 13, 17, 21, 25, 29, + 2, 6, 10, 14, 18, 22, 26, 30, + 3, 7, 11, 15, 19, 23, 27, 31 + ); + b_.i8 = + SIMDE_SHUFFLE_VECTOR_( + 8, 32, + b_.i8, b_.i8, + 0, 4, 8, 12, 16, 20, 24, 28, + 1, 5, 9, 13, 17, 21, 25, 29, + 2, 6, 10, 14, 18, 22, 26, 30, + 3, 7, 11, 15, 19, 23, 27, 31 + ); + + SIMDE_CONVERT_VECTOR_(x1_, a_.u8); + SIMDE_CONVERT_VECTOR_(x2_, b_.i8); + + simde_memcpy(&r1_, &x1_, sizeof(x1_)); + simde_memcpy(&r2_, &x2_, sizeof(x2_)); + + uint32_t au SIMDE_VECTOR(32) = + HEDLEY_REINTERPRET_CAST( + __typeof__(au), + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[0].u32) * r2_[0].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[1].u32) * r2_[1].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[2].u32) * r2_[2].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[3].u32) * r2_[3].i32) + ); + uint32_t bu SIMDE_VECTOR(32) = HEDLEY_REINTERPRET_CAST(__typeof__(bu), src_.i32); + uint32_t ru SIMDE_VECTOR(32) = au + bu; + + au = (au >> 31) + INT32_MAX; + + uint32_t m SIMDE_VECTOR(32) = HEDLEY_REINTERPRET_CAST(__typeof__(m), HEDLEY_REINTERPRET_CAST(__typeof__(src_.i32), (au ^ bu) | ~(bu ^ ru)) < 0); + src_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(src_.i32), (au & ~m) | (ru & m)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0]) / 4) ; i++) { + src_.i32[i] = + simde_math_adds_i32( + src_.i32[i], + HEDLEY_STATIC_CAST(uint16_t, a_.u8[(4 * i) ]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[(4 * i) ]) + + HEDLEY_STATIC_CAST(uint16_t, a_.u8[(4 * i) + 1]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[(4 * i) + 1]) + + HEDLEY_STATIC_CAST(uint16_t, a_.u8[(4 * i) + 2]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[(4 * i) + 2]) + + HEDLEY_STATIC_CAST(uint16_t, a_.u8[(4 * i) + 3]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[(4 * i) + 3]) + ); + } + #endif + + return simde__m256i_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_dpbusds_epi32 + #define _mm256_dpbusds_epi32(src, a, b) simde_mm256_dpbusds_epi32(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_dpbusds_epi32(simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm256_mask_dpbusds_epi32(src, k, a, b); + #else + return simde_mm256_mask_mov_epi32(src, k, simde_mm256_dpbusds_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_dpbusds_epi32 + #define _mm256_mask_dpbusds_epi32(src, k, a, b) simde_mm256_mask_dpbusds_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_dpbusds_epi32(simde__mmask8 k, simde__m256i src, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm256_maskz_dpbusds_epi32(k, src, a, b); + #else + return simde_mm256_maskz_mov_epi32(k, simde_mm256_dpbusds_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_dpbusds_epi32 + #define _mm256_maskz_dpbusds_epi32(k, src, a, b) simde_mm256_maskz_dpbusds_epi32(k, src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_dpbusds_epi32(simde__m512i src, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm512_dpbusds_epi32(src, a, b); + #else + simde__m512i_private + src_ = simde__m512i_to_private(src), + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + src_.m256i[0] = simde_mm256_dpbusds_epi32(src_.m256i[0], a_.m256i[0], b_.m256i[0]); + src_.m256i[1] = simde_mm256_dpbusds_epi32(src_.m256i[1], a_.m256i[1], b_.m256i[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_CONVERT_VECTOR_) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + uint32_t x1_ SIMDE_VECTOR(256); + int32_t x2_ SIMDE_VECTOR(256); + simde__m512i_private + r1_[4], + r2_[4]; + + a_.u8 = + SIMDE_SHUFFLE_VECTOR_( + 8, 64, + a_.u8, a_.u8, + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, + 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, + 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62, + 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63 + ); + b_.i8 = + SIMDE_SHUFFLE_VECTOR_( + 8, 64, + b_.i8, b_.i8, + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, + 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, + 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62, + 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63 + ); + + SIMDE_CONVERT_VECTOR_(x1_, a_.u8); + SIMDE_CONVERT_VECTOR_(x2_, b_.i8); + + simde_memcpy(&r1_, &x1_, sizeof(x1_)); + simde_memcpy(&r2_, &x2_, sizeof(x2_)); + + uint32_t au SIMDE_VECTOR(64) = + HEDLEY_REINTERPRET_CAST( + __typeof__(au), + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[0].u32) * r2_[0].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[1].u32) * r2_[1].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[2].u32) * r2_[2].i32) + + (HEDLEY_REINTERPRET_CAST(__typeof__(a_.i32), r1_[3].u32) * r2_[3].i32) + ); + uint32_t bu SIMDE_VECTOR(64) = HEDLEY_REINTERPRET_CAST(__typeof__(bu), src_.i32); + uint32_t ru SIMDE_VECTOR(64) = au + bu; + + au = (au >> 31) + INT32_MAX; + + uint32_t m SIMDE_VECTOR(64) = HEDLEY_REINTERPRET_CAST(__typeof__(m), HEDLEY_REINTERPRET_CAST(__typeof__(src_.i32), (au ^ bu) | ~(bu ^ ru)) < 0); + src_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(src_.i32), (au & ~m) | (ru & m)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0]) / 4) ; i++) { + src_.i32[i] = + simde_math_adds_i32( + src_.i32[i], + HEDLEY_STATIC_CAST(uint16_t, a_.u8[(4 * i) ]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[(4 * i) ]) + + HEDLEY_STATIC_CAST(uint16_t, a_.u8[(4 * i) + 1]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[(4 * i) + 1]) + + HEDLEY_STATIC_CAST(uint16_t, a_.u8[(4 * i) + 2]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[(4 * i) + 2]) + + HEDLEY_STATIC_CAST(uint16_t, a_.u8[(4 * i) + 3]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[(4 * i) + 3]) + ); + } + #endif + + return simde__m512i_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_dpbusds_epi32 + #define _mm512_dpbusds_epi32(src, a, b) simde_mm512_dpbusds_epi32(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_dpbusds_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm512_mask_dpbusds_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_dpbusds_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_dpbusds_epi32 + #define _mm512_mask_dpbusds_epi32(src, k, a, b) simde_mm512_mask_dpbusds_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_dpbusds_epi32(simde__mmask16 k, simde__m512i src, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm512_maskz_dpbusds_epi32(k, src, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_dpbusds_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_dpbusds_epi32 + #define _mm512_maskz_dpbusds_epi32(k, src, a, b) simde_mm512_maskz_dpbusds_epi32(k, src, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_DPBUSDS_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpwssd.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpwssd.h new file mode 100644 index 00000000..33b0ce55 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpwssd.h @@ -0,0 +1,269 @@ +#if !defined(SIMDE_X86_AVX512_DPWSSD_H) +#define SIMDE_X86_AVX512_DPWSSD_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_dpwssd_epi32(simde__m128i src, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm_dpwssd_epi32(src, a, b); + #else + simde__m128i_private + src_ = simde__m128i_to_private(src), + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_CONVERT_VECTOR_) + int32_t x1_ SIMDE_VECTOR(32); + int32_t x2_ SIMDE_VECTOR(32); + simde__m128i_private + r1_[2], + r2_[2]; + + a_.i16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 16, + a_.i16, a_.i16, + 0, 2, 4, 6, + 1, 3, 5, 7 + ); + b_.i16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 16, + b_.i16, b_.i16, + 0, 2, 4, 6, + 1, 3, 5, 7 + ); + + SIMDE_CONVERT_VECTOR_(x1_, a_.i16); + SIMDE_CONVERT_VECTOR_(x2_, b_.i16); + + simde_memcpy(&r1_, &x1_, sizeof(x1_)); + simde_memcpy(&r2_, &x2_, sizeof(x2_)); + + src_.i32 += + (r1_[0].i32 * r2_[0].i32) + + (r1_[1].i32 * r2_[1].i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u16) / sizeof(a_.i16[0])) ; i++) { + src_.i32[i / 2] += HEDLEY_STATIC_CAST(int32_t, a_.i16[i]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[i]); + } + #endif + + return simde__m128i_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm_dpwssd_epi32 + #define _mm_dpwssd_epi32(src, a, b) simde_mm_dpwssd_epi32(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_dpwssd_epi32(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm_mask_dpwssd_epi32(src, k, a, b); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_dpwssd_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_dpwssd_epi32 + #define _mm_mask_dpwssd_epi32(src, k, a, b) simde_mm_mask_dpwssd_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_dpwssd_epi32(simde__mmask8 k, simde__m128i src, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm_maskz_dpwssd_epi32(k, src, a, b); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_dpwssd_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_dpwssd_epi32 + #define _mm_maskz_dpwssd_epi32(k, src, a, b) simde_mm_maskz_dpwssd_epi32(k, src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_dpwssd_epi32(simde__m256i src, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm256_dpwssd_epi32(src, a, b); + #else + simde__m256i_private + src_ = simde__m256i_to_private(src), + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_CONVERT_VECTOR_) + int32_t x1_ SIMDE_VECTOR(64); + int32_t x2_ SIMDE_VECTOR(64); + simde__m256i_private + r1_[2], + r2_[2]; + + a_.i16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 32, + a_.i16, a_.i16, + 0, 2, 4, 6, 8, 10, 12, 14, + 1, 3, 5, 7, 9, 11, 13, 15 + ); + b_.i16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 32, + b_.i16, b_.i16, + 0, 2, 4, 6, 8, 10, 12, 14, + 1, 3, 5, 7, 9, 11, 13, 15 + ); + + SIMDE_CONVERT_VECTOR_(x1_, a_.i16); + SIMDE_CONVERT_VECTOR_(x2_, b_.i16); + + simde_memcpy(&r1_, &x1_, sizeof(x1_)); + simde_memcpy(&r2_, &x2_, sizeof(x2_)); + + src_.i32 += + (r1_[0].i32 * r2_[0].i32) + + (r1_[1].i32 * r2_[1].i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u16) / sizeof(a_.i16[0])) ; i++) { + src_.i32[i / 2] += HEDLEY_STATIC_CAST(int32_t, a_.i16[i]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[i]); + } + #endif + + return simde__m256i_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_dpwssd_epi32 + #define _mm256_dpwssd_epi32(src, a, b) simde_mm256_dpwssd_epi32(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_dpwssd_epi32(simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm256_mask_dpwssd_epi32(src, k, a, b); + #else + return simde_mm256_mask_mov_epi32(src, k, simde_mm256_dpwssd_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_dpwssd_epi32 + #define _mm256_mask_dpwssd_epi32(src, k, a, b) simde_mm256_mask_dpwssd_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_dpwssd_epi32(simde__mmask8 k, simde__m256i src, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm256_maskz_dpwssd_epi32(k, src, a, b); + #else + return simde_mm256_maskz_mov_epi32(k, simde_mm256_dpwssd_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_dpwssd_epi32 + #define _mm256_maskz_dpwssd_epi32(k, src, a, b) simde_mm256_maskz_dpwssd_epi32(k, src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_dpwssd_epi32(simde__m512i src, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm512_dpwssd_epi32(src, a, b); + #else + simde__m512i_private + src_ = simde__m512i_to_private(src), + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_CONVERT_VECTOR_) + int32_t x1_ SIMDE_VECTOR(128); + int32_t x2_ SIMDE_VECTOR(128); + simde__m512i_private + r1_[2], + r2_[2]; + + a_.i16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 64, + a_.i16, a_.i16, + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 + ); + b_.i16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 64, + b_.i16, b_.i16, + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 + ); + + SIMDE_CONVERT_VECTOR_(x1_, a_.i16); + SIMDE_CONVERT_VECTOR_(x2_, b_.i16); + + simde_memcpy(&r1_, &x1_, sizeof(x1_)); + simde_memcpy(&r2_, &x2_, sizeof(x2_)); + + src_.i32 += + (r1_[0].i32 * r2_[0].i32) + + (r1_[1].i32 * r2_[1].i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u16) / sizeof(a_.i16[0])) ; i++) { + src_.i32[i / 2] += HEDLEY_STATIC_CAST(int32_t, a_.i16[i]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[i]); + } + #endif + + return simde__m512i_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_dpwssd_epi32 + #define _mm512_dpwssd_epi32(src, a, b) simde_mm512_dpwssd_epi32(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_dpwssd_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm512_mask_dpwssd_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_dpwssd_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_dpwssd_epi32 + #define _mm512_mask_dpwssd_epi32(src, k, a, b) simde_mm512_mask_dpwssd_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_dpwssd_epi32(simde__mmask16 k, simde__m512i src, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm512_maskz_dpwssd_epi32(k, src, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_dpwssd_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_dpwssd_epi32 + #define _mm512_maskz_dpwssd_epi32(k, src, a, b) simde_mm512_maskz_dpwssd_epi32(k, src, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_DPWSSD_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpwssds.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpwssds.h new file mode 100644 index 00000000..ea720917 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/dpwssds.h @@ -0,0 +1,299 @@ +#if !defined(SIMDE_X86_AVX512_DPWSSDS_H) +#define SIMDE_X86_AVX512_DPWSSDS_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_dpwssds_epi32 (simde__m128i src, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_dpwssds_epi32(src, a, b); + #else + simde__m128i_private + src_ = simde__m128i_to_private(src), + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_CONVERT_VECTOR_) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + int32_t x1_ SIMDE_VECTOR(32); + int32_t x2_ SIMDE_VECTOR(32); + simde__m128i_private + r1_[2], + r2_[2]; + + a_.i16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 16, + a_.i16, a_.i16, + 0, 2, 4, 6, + 1, 3, 5, 7 + ); + b_.i16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 16, + b_.i16, b_.i16, + 0, 2, 4, 6, + 1, 3, 5, 7 + ); + + SIMDE_CONVERT_VECTOR_(x1_, a_.i16); + SIMDE_CONVERT_VECTOR_(x2_, b_.i16); + + simde_memcpy(&r1_, &x1_, sizeof(x1_)); + simde_memcpy(&r2_, &x2_, sizeof(x2_)); + + uint32_t au SIMDE_VECTOR(16) = HEDLEY_REINTERPRET_CAST(__typeof__(src_.u32), ((r1_[0].i32 * r2_[0].i32) + (r1_[1].i32 * r2_[1].i32))); + uint32_t bu SIMDE_VECTOR(16) = HEDLEY_REINTERPRET_CAST(__typeof__(src_.u32), src_.i32); + uint32_t ru SIMDE_VECTOR(16) = au + bu; + + au = (au >> 31) + INT32_MAX; + + uint32_t m SIMDE_VECTOR(16) = HEDLEY_REINTERPRET_CAST(__typeof__(m), HEDLEY_REINTERPRET_CAST(__typeof__(src_.i32), (au ^ bu) | ~(bu ^ ru)) < 0); + src_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(src_.i32), (au & ~m) | (ru & m)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0]) / 2) ; i++) { + src_.i32[i] = + simde_math_adds_i32( + src_.i32[i], + HEDLEY_STATIC_CAST(int32_t, a_.i16[(2 * i) ]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[(2 * i) ]) + + HEDLEY_STATIC_CAST(int32_t, a_.i16[(2 * i) + 1]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[(2 * i) + 1]) + ); + } + #endif + + return simde__m128i_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_dpwssds_epi32 + #define _mm_dpwssds_epi32(src, a, b) simde_mm_dpwssds_epi32(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_dpwssds_epi32 (simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_dpwssds_epi32(src, k, a, b); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_dpwssds_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_dpwssds_epi32 + #define _mm_mask_dpwssds_epi32(src, k, a, b) simde_mm_mask_dpwssds_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_dpwssds_epi32 (simde__mmask8 k, simde__m128i src, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_dpwssds_epi32(k, src, a, b); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_dpwssds_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_dpwssds_epi32 + #define _mm_maskz_dpwssds_epi32(k, src, a, b) simde_mm_maskz_dpwssds_epi32(k, src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_dpwssds_epi32 (simde__m256i src, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_dpwssds_epi32(src, a, b); + #else + simde__m256i_private + src_ = simde__m256i_to_private(src), + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_CONVERT_VECTOR_) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + int32_t x1_ SIMDE_VECTOR(64); + int32_t x2_ SIMDE_VECTOR(64); + simde__m256i_private + r1_[2], + r2_[2]; + + a_.i16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 32, + a_.i16, a_.i16, + 0, 2, 4, 6, 8, 10, 12, 14, + 1, 3, 5, 7, 9, 11, 13, 15 + ); + b_.i16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 32, + b_.i16, b_.i16, + 0, 2, 4, 6, 8, 10, 12, 14, + 1, 3, 5, 7, 9, 11, 13, 15 + ); + + SIMDE_CONVERT_VECTOR_(x1_, a_.i16); + SIMDE_CONVERT_VECTOR_(x2_, b_.i16); + + simde_memcpy(&r1_, &x1_, sizeof(x1_)); + simde_memcpy(&r2_, &x2_, sizeof(x2_)); + + uint32_t au SIMDE_VECTOR(32) = HEDLEY_REINTERPRET_CAST(__typeof__(src_.u32), ((r1_[0].i32 * r2_[0].i32) + (r1_[1].i32 * r2_[1].i32))); + uint32_t bu SIMDE_VECTOR(32) = HEDLEY_REINTERPRET_CAST(__typeof__(src_.u32), src_.i32); + uint32_t ru SIMDE_VECTOR(32) = au + bu; + + au = (au >> 31) + INT32_MAX; + + uint32_t m SIMDE_VECTOR(32) = HEDLEY_REINTERPRET_CAST(__typeof__(m), HEDLEY_REINTERPRET_CAST(__typeof__(src_.i32), (au ^ bu) | ~(bu ^ ru)) < 0); + src_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(src_.i32), (au & ~m) | (ru & m)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0]) / 2) ; i++) { + src_.i32[i] = + simde_math_adds_i32( + src_.i32[i], + HEDLEY_STATIC_CAST(int32_t, a_.i16[(2 * i) ]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[(2 * i) ]) + + HEDLEY_STATIC_CAST(int32_t, a_.i16[(2 * i) + 1]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[(2 * i) + 1]) + ); + } + #endif + + return simde__m256i_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_dpwssds_epi32 + #define _mm256_dpwssds_epi32(src, a, b) simde_mm256_dpwssds_epi32(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_dpwssds_epi32 (simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_dpwssds_epi32(src, k, a, b); + #else + return simde_mm256_mask_mov_epi32(src, k, simde_mm256_dpwssds_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_dpwssds_epi32 + #define _mm256_mask_dpwssds_epi32(src, k, a, b) simde_mm256_mask_dpwssds_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_dpwssds_epi32 (simde__mmask8 k, simde__m256i src, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_dpwssds_epi32(k, src, a, b); + #else + return simde_mm256_maskz_mov_epi32(k, simde_mm256_dpwssds_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_dpwssds_epi32 + #define _mm256_maskz_dpwssds_epi32(k, src, a, b) simde_mm256_maskz_dpwssds_epi32(k, src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_dpwssds_epi32 (simde__m512i src, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm512_dpwssds_epi32(src, a, b); + #else + simde__m512i_private + src_ = simde__m512i_to_private(src), + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_CONVERT_VECTOR_) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + int32_t x1_ SIMDE_VECTOR(128); + int32_t x2_ SIMDE_VECTOR(128); + simde__m512i_private + r1_[2], + r2_[2]; + + a_.i16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 64, + a_.i16, a_.i16, + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 + ); + b_.i16 = + SIMDE_SHUFFLE_VECTOR_( + 16, 64, + b_.i16, b_.i16, + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 + ); + + SIMDE_CONVERT_VECTOR_(x1_, a_.i16); + SIMDE_CONVERT_VECTOR_(x2_, b_.i16); + + simde_memcpy(&r1_, &x1_, sizeof(x1_)); + simde_memcpy(&r2_, &x2_, sizeof(x2_)); + + uint32_t au SIMDE_VECTOR(64) = HEDLEY_REINTERPRET_CAST(__typeof__(src_.u32), ((r1_[0].i32 * r2_[0].i32) + (r1_[1].i32 * r2_[1].i32))); + uint32_t bu SIMDE_VECTOR(64) = HEDLEY_REINTERPRET_CAST(__typeof__(src_.u32), src_.i32); + uint32_t ru SIMDE_VECTOR(64) = au + bu; + + au = (au >> 31) + INT32_MAX; + + uint32_t m SIMDE_VECTOR(64) = HEDLEY_REINTERPRET_CAST(__typeof__(m), HEDLEY_REINTERPRET_CAST(__typeof__(src_.i32), (au ^ bu) | ~(bu ^ ru)) < 0); + src_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(src_.i32), (au & ~m) | (ru & m)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0]) / 2) ; i++) { + src_.i32[i] = + simde_math_adds_i32( + src_.i32[i], + HEDLEY_STATIC_CAST(int32_t, a_.i16[(2 * i) ]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[(2 * i) ]) + + HEDLEY_STATIC_CAST(int32_t, a_.i16[(2 * i) + 1]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[(2 * i) + 1]) + ); + } + #endif + + return simde__m512i_from_private(src_); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_dpwssds_epi32 + #define _mm512_dpwssds_epi32(src, a, b) simde_mm512_dpwssds_epi32(src, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_dpwssds_epi32 (simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm512_mask_dpwssds_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_dpwssds_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_dpwssds_epi32 + #define _mm512_mask_dpwssds_epi32(src, k, a, b) simde_mm512_mask_dpwssds_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_dpwssds_epi32 (simde__mmask16 k, simde__m512i src, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VNNI_NATIVE) + return _mm512_maskz_dpwssds_epi32(k, src, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_dpwssds_epi32(src, a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VNNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_dpwssds_epi32 + #define _mm512_maskz_dpwssds_epi32(k, src, a, b) simde_mm512_maskz_dpwssds_epi32(k, src, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_DPWSSDS_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/expand.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/expand.h new file mode 100644 index 00000000..4afba87f --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/expand.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2021 Andrew Rodriguez + * 2021 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_EXPAND_H) +#define SIMDE_X86_AVX512_EXPAND_H + +#include "types.h" +#include "mov.h" +#include "mov_mask.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_expand_epi32(simde__m256i src, simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_expand_epi32(src, k, a); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + src_ = simde__m256i_to_private(src); + simde__m256i_private r_; + + size_t src_idx = 0; + for (size_t i = 0; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + if (k & (UINT64_C(1) << i)) { + r_.i32[i] = a_.i32[src_idx++]; + } else { + r_.i32[i] = src_.i32[i]; + } + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_expand_epi32 + #define _mm256_mask_expand_epi32(src, k, a) simde_mm256_mask_expand_epi32((src), (k), (a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_expand_epi32(simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_expand_epi32(k, a); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + r_; + + size_t src_idx = 0; + for (size_t i = 0; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + if (k & (UINT64_C(1) << i)) { + r_.i32[i] = a_.i32[src_idx++]; + } else { + r_.i32[i] = INT32_C(0); + } + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_expand_epi32 + #define _mm256_maskz_expand_epi32(k, a) simde_mm256_maskz_expand_epi32((k), (a)) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_EXPAND_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/extract.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/extract.h new file mode 100644 index 00000000..251715cf --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/extract.h @@ -0,0 +1,267 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Christopher Moore + */ + +#if !defined(SIMDE_X86_AVX512_EXTRACT_H) +#define SIMDE_X86_AVX512_EXTRACT_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm256_extractf32x4_ps (simde__m256 a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m256_private a_ = simde__m256_to_private(a); + + return a_.m128[imm8 & 1]; +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_extractf32x4_ps(a, imm8) _mm256_extractf32x4_ps(a, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_extractf32x4_ps + #define _mm256_extractf32x4_ps(a, imm8) simde_mm256_extractf32x4_ps((a), (imm8)) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm512_extractf32x4_ps (simde__m512 a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m512_private a_ = simde__m512_to_private(a); + + /* GCC 6 generates an ICE */ + #if defined(HEDLEY_GCC_VERSION) && !HEDLEY_GCC_VERSION_CHECK(7,0,0) + return a_.m128[imm8 & 3]; + #else + simde__m128_private r_; + const size_t offset = HEDLEY_STATIC_CAST(size_t, imm8 & 3) * (sizeof(r_.f32) / sizeof(r_.f32[0])); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[i + offset]; + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) + #define simde_mm512_extractf32x4_ps(a, imm8) _mm512_extractf32x4_ps(a, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_extractf32x4_ps + #define _mm512_extractf32x4_ps(a, imm8) simde_mm512_extractf32x4_ps((a), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) + #define simde_mm512_mask_extractf32x4_ps(src, k, a, imm8) _mm512_mask_extractf32x4_ps(src, k, a, imm8) +#else + #define simde_mm512_mask_extractf32x4_ps(src, k, a, imm8) simde_mm_mask_mov_ps((src), (k), simde_mm512_extractf32x4_ps((a), (imm8))) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_extractf32x4_ps + #define _mm512_mask_extractf32x4_ps(src, k, a, imm8) simde_mm512_mask_extractf32x4_ps((src), (k), (a), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) + #define simde_mm512_maskz_extractf32x4_ps(k, a, imm8) _mm512_maskz_extractf32x4_ps(k, a, imm8) +#else + #define simde_mm512_maskz_extractf32x4_ps(k, a, imm8) simde_mm_maskz_mov_ps((k), simde_mm512_extractf32x4_ps((a), (imm8))) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_extractf32x4_ps + #define _mm512_maskz_extractf32x4_ps(k, a, imm8) simde_mm512_maskz_extractf32x4_ps((k), (a), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm512_extractf32x8_ps (simde__m512 a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m512_private a_ = simde__m512_to_private(a); + + return a_.m256[imm8 & 1]; +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_extractf32x8_ps(a, imm8) _mm512_extractf32x8_ps(a, imm8) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_extractf32x8_ps + #define _mm512_extractf32x8_ps(a, imm8) simde_mm512_extractf32x8_ps(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm512_extractf64x4_pd (simde__m512d a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m512d_private a_ = simde__m512d_to_private(a); + + return a_.m256d[imm8 & 1]; +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) + #define simde_mm512_extractf64x4_pd(a, imm8) _mm512_extractf64x4_pd(a, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_extractf64x4_pd + #define _mm512_extractf64x4_pd(a, imm8) simde_mm512_extractf64x4_pd(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) + #define simde_mm512_mask_extractf64x4_pd(src, k, a, imm8) _mm512_mask_extractf64x4_pd(src, k, a, imm8) +#else + #define simde_mm512_mask_extractf64x4_pd(src, k, a, imm8) simde_mm256_mask_mov_pd(src, k, simde_mm512_extractf64x4_pd(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_extractf64x4_pd + #define _mm512_mask_extractf64x4_pd(src, k, a, imm8) simde_mm512_mask_extractf64x4_pd(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) + #define simde_mm512_maskz_extractf64x4_pd(k, a, imm8) _mm512_maskz_extractf64x4_pd(k, a, imm8) +#else + #define simde_mm512_maskz_extractf64x4_pd(k, a, imm8) simde_mm256_maskz_mov_pd(k, simde_mm512_extractf64x4_pd(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_extractf64x4_pd + #define _mm512_maskz_extractf64x4_pd(k, a, imm8) simde_mm512_maskz_extractf64x4_pd(k, a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm512_extracti32x4_epi32 (simde__m512i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m512i_private a_ = simde__m512i_to_private(a); + + return a_.m128i[imm8 & 3]; +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && !defined(SIMDE_BUG_CLANG_REV_299346) + #define simde_mm512_extracti32x4_epi32(a, imm8) _mm512_extracti32x4_epi32(a, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_extracti32x4_epi32 + #define _mm512_extracti32x4_epi32(a, imm8) simde_mm512_extracti32x4_epi32(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && !defined(SIMDE_BUG_CLANG_REV_299346) + #define simde_mm512_mask_extracti32x4_epi32(src, k, a, imm8) _mm512_mask_extracti32x4_epi32(src, k, a, imm8) +#else + #define simde_mm512_mask_extracti32x4_epi32(src, k, a, imm8) simde_mm_mask_mov_epi32(src, k, simde_mm512_extracti32x4_epi32(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_extracti32x4_epi32 + #define _mm512_mask_extracti32x4_epi32(src, k, a, imm8) simde_mm512_mask_extracti32x4_epi32(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && !defined(SIMDE_BUG_CLANG_REV_299346) + #define simde_mm512_maskz_extracti32x4_epi32(k, a, imm8) _mm512_maskz_extracti32x4_epi32(k, a, imm8) +#else + #define simde_mm512_maskz_extracti32x4_epi32(k, a, imm8) simde_mm_maskz_mov_epi32(k, simde_mm512_extracti32x4_epi32(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_extracti32x4_epi32 + #define _mm512_maskz_extracti32x4_epi32(k, a, imm8) simde_mm512_maskz_extracti32x4_epi32(k, a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_extracti32x8_epi32 (simde__m512i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m512i_private a_ = simde__m512i_to_private(a); + + return a_.m256i[imm8 & 1]; +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && !defined(SIMDE_BUG_CLANG_REV_299346) + #define simde_mm512_extracti32x8_epi32(a, imm8) _mm512_extracti32x8_epi32(a, imm8) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_extracti32x8_epi32 + #define _mm512_extracti32x8_epi32(a, imm8) simde_mm512_extracti32x8_epi32((a), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX51FDQ_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && !defined(SIMDE_BUG_CLANG_REV_299346) + #define simde_mm512_mask_extracti32x8_epi32(src, k, a, imm8) _mm512_mask_extracti32x8_epi32(src, k, a, imm8) +#else + #define simde_mm512_mask_extracti32x8_epi32(src, k, a, imm8) simde_mm256_mask_mov_epi32((src), (k), simde_mm512_extracti32x8_epi32((a), (imm8))) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_extracti32x8_epi32 + #define _mm512_mask_extracti32x8_epi32(src, k, a, imm8) simde_mm512_mask_extracti32x8_epi32((src), (k), (a), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && !defined(SIMDE_BUG_CLANG_REV_299346) + #define simde_mm512_maskz_extracti32x8_epi32(k, a, imm8) _mm512_maskz_extracti32x8_epi32(k, a, imm8) +#else + #define simde_mm512_maskz_extracti32x8_epi32(k, a, imm8) simde_mm256_maskz_mov_epi32((k), simde_mm512_extracti32x8_epi32((a), (imm8))) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_extracti32x8_epi32 + #define _mm512_maskz_extracti32x8_epi32(k, a, imm8) simde_mm512_maskz_extracti32x8_epi32((k), (a), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_extracti64x4_epi64 (simde__m512i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m512i_private a_ = simde__m512i_to_private(a); + + return a_.m256i[imm8 & 1]; +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && !defined(SIMDE_BUG_CLANG_REV_299346) + #define simde_mm512_extracti64x4_epi64(a, imm8) _mm512_extracti64x4_epi64(a, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_extracti64x4_epi64 + #define _mm512_extracti64x4_epi64(a, imm8) simde_mm512_extracti64x4_epi64((a), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && !defined(SIMDE_BUG_CLANG_REV_299346) + #define simde_mm512_mask_extracti64x4_epi64(src, k, a, imm8) _mm512_mask_extracti64x4_epi64(src, k, a, imm8) +#else + #define simde_mm512_mask_extracti64x4_epi64(src, k, a, imm8) simde_mm256_mask_mov_epi64((src), (k), simde_mm512_extracti64x4_epi64((a), (imm8))) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_extracti64x4_epi64 + #define _mm512_mask_extracti64x4_epi64(src, k, a, imm8) simde_mm512_mask_extracti64x4_epi64((src), (k), (a), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && !defined(SIMDE_BUG_CLANG_REV_299346) + #define simde_mm512_maskz_extracti64x4_epi64(k, a, imm8) _mm512_maskz_extracti64x4_epi64(k, a, imm8) +#else + #define simde_mm512_maskz_extracti64x4_epi64(k, a, imm8) simde_mm256_maskz_mov_epi64((k), simde_mm512_extracti64x4_epi64((a), (imm8))) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_extracti64x4_epi64 + #define _mm512_maskz_extracti64x4_epi64(k, a, imm8) simde_mm512_maskz_extracti64x4_epi64((k), (a), (imm8)) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_EXTRACT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fixupimm.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fixupimm.h new file mode 100644 index 00000000..2ea234bd --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fixupimm.h @@ -0,0 +1,900 @@ +#if !defined(SIMDE_X86_AVX512_FIXUPIMM_H) +#define SIMDE_X86_AVX512_FIXUPIMM_H + +#include "types.h" +#include "flushsubnormal.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_fixupimm_ps (simde__m128 a, simde__m128 b, simde__m128i c, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b), + s_ = simde__m128_to_private(simde_x_mm_flushsubnormal_ps(b)); + simde__m128i_private c_ = simde__m128i_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + int32_t select = 1; + switch (simde_math_fpclassifyf(s_.f32[i])) { + case SIMDE_MATH_FP_NORMAL: + select = (s_.f32[i] < SIMDE_FLOAT32_C(0.0)) ? 6 : (s_.f32[i] == SIMDE_FLOAT32_C(1.0)) ? 3 : 7; + break; + case SIMDE_MATH_FP_ZERO: + select = 2; + break; + case SIMDE_MATH_FP_NAN: + select = 0; + break; + case SIMDE_MATH_FP_INFINITE: + select = ((s_.f32[i] > SIMDE_FLOAT32_C(0.0)) ? 5 : 4); + break; + } + + switch (((c_.i32[i] >> (select << 2)) & 15)) { + case 0: + r_.f32[i] = a_.f32[i]; + break; + case 1: + r_.f32[i] = b_.f32[i]; + break; + case 2: + r_.f32[i] = SIMDE_MATH_NANF; + break; + case 3: + r_.f32[i] = -SIMDE_MATH_NANF; + break; + case 4: + r_.f32[i] = -SIMDE_MATH_INFINITYF; + break; + case 5: + r_.f32[i] = SIMDE_MATH_INFINITYF; + break; + case 6: + r_.f32[i] = s_.f32[i] < SIMDE_FLOAT32_C(0.0) ? -SIMDE_MATH_INFINITYF : SIMDE_MATH_INFINITYF; + break; + case 7: + r_.f32[i] = SIMDE_FLOAT32_C(-0.0); + break; + case 8: + r_.f32[i] = SIMDE_FLOAT32_C(0.0); + break; + case 9: + r_.f32[i] = SIMDE_FLOAT32_C(-1.0); + break; + case 10: + r_.f32[i] = SIMDE_FLOAT32_C(1.0); + break; + case 11: + r_.f32[i] = SIMDE_FLOAT32_C(0.5); + break; + case 12: + r_.f32[i] = SIMDE_FLOAT32_C(90.0); + break; + case 13: + r_.f32[i] = SIMDE_MATH_PIF / 2; + break; + case 14: + r_.f32[i] = SIMDE_MATH_FLT_MAX; + break; + case 15: + r_.f32[i] = -SIMDE_MATH_FLT_MAX; + break; + } + } + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_fixupimm_ps(a, b, c, imm8) _mm_fixupimm_ps(a, b, c, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_fixupimm_ps + #define _mm_fixupimm_ps(a, b, c, imm8) simde_mm_fixupimm_ps(a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_mask_fixupimm_ps(a, k, b, c, imm8) _mm_mask_fixupimm_ps(a, k, b, c, imm8) +#else + #define simde_mm_mask_fixupimm_ps(a, k, b, c, imm8) simde_mm_mask_mov_ps(a, k, simde_mm_fixupimm_ps(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_fixupimm_ps + #define _mm_mask_fixupimm_ps(a, k, b, c, imm8) simde_mm_mask_fixupimm_ps(a, k, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_maskz_fixupimm_ps(k, a, b, c, imm8) _mm_maskz_fixupimm_ps(k, a, b, c, imm8) +#else + #define simde_mm_maskz_fixupimm_ps(k, a, b, c, imm8) simde_mm_maskz_mov_ps(k, simde_mm_fixupimm_ps(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_fixupimm_ps + #define _mm_maskz_fixupimm_ps(k, a, b, c, imm8) simde_mm_maskz_fixupimm_ps(k, a, b, c, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_fixupimm_ps (simde__m256 a, simde__m256 b, simde__m256i c, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b), + s_ = simde__m256_to_private(simde_x_mm256_flushsubnormal_ps(b)); + simde__m256i_private c_ = simde__m256i_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + int32_t select = 1; + switch (simde_math_fpclassifyf(s_.f32[i])) { + case SIMDE_MATH_FP_NORMAL: + select = (s_.f32[i] < SIMDE_FLOAT32_C(0.0)) ? 6 : (s_.f32[i] == SIMDE_FLOAT32_C(1.0)) ? 3 : 7; + break; + case SIMDE_MATH_FP_ZERO: + select = 2; + break; + case SIMDE_MATH_FP_NAN: + select = 0; + break; + case SIMDE_MATH_FP_INFINITE: + select = ((s_.f32[i] > SIMDE_FLOAT32_C(0.0)) ? 5 : 4); + break; + } + + switch (((c_.i32[i] >> (select << 2)) & 15)) { + case 0: + r_.f32[i] = a_.f32[i]; + break; + case 1: + r_.f32[i] = b_.f32[i]; + break; + case 2: + r_.f32[i] = SIMDE_MATH_NANF; + break; + case 3: + r_.f32[i] = -SIMDE_MATH_NANF; + break; + case 4: + r_.f32[i] = -SIMDE_MATH_INFINITYF; + break; + case 5: + r_.f32[i] = SIMDE_MATH_INFINITYF; + break; + case 6: + r_.f32[i] = s_.f32[i] < SIMDE_FLOAT32_C(0.0) ? -SIMDE_MATH_INFINITYF : SIMDE_MATH_INFINITYF; + break; + case 7: + r_.f32[i] = SIMDE_FLOAT32_C(-0.0); + break; + case 8: + r_.f32[i] = SIMDE_FLOAT32_C(0.0); + break; + case 9: + r_.f32[i] = SIMDE_FLOAT32_C(-1.0); + break; + case 10: + r_.f32[i] = SIMDE_FLOAT32_C(1.0); + break; + case 11: + r_.f32[i] = SIMDE_FLOAT32_C(0.5); + break; + case 12: + r_.f32[i] = SIMDE_FLOAT32_C(90.0); + break; + case 13: + r_.f32[i] = SIMDE_MATH_PIF / 2; + break; + case 14: + r_.f32[i] = SIMDE_MATH_FLT_MAX; + break; + case 15: + r_.f32[i] = -SIMDE_MATH_FLT_MAX; + break; + } + } + + return simde__m256_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_fixupimm_ps(a, b, c, imm8) _mm256_fixupimm_ps(a, b, c, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_fixupimm_ps + #define _mm256_fixupimm_ps(a, b, c, imm8) simde_mm256_fixupimm_ps(a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_fixupimm_ps(a, k, b, c, imm8) _mm256_mask_fixupimm_ps(a, k, b, c, imm8) +#else + #define simde_mm256_mask_fixupimm_ps(a, k, b, c, imm8) simde_mm256_mask_mov_ps(a, k, simde_mm256_fixupimm_ps(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_fixupimm_ps + #define _mm256_mask_fixupimm_ps(a, k, b, c, imm8) simde_mm256_mask_fixupimm_ps(a, k, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_maskz_fixupimm_ps(k, a, b, c, imm8) _mm256_maskz_fixupimm_ps(k, a, b, c, imm8) +#else + #define simde_mm256_maskz_fixupimm_ps(k, a, b, c, imm8) simde_mm256_maskz_mov_ps(k, simde_mm256_fixupimm_ps(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_fixupimm_ps + #define _mm256_maskz_fixupimm_ps(k, a, b, c, imm8) simde_mm256_maskz_fixupimm_ps(k, a, b, c, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_fixupimm_ps (simde__m512 a, simde__m512 b, simde__m512i c, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b), + s_ = simde__m512_to_private(simde_x_mm512_flushsubnormal_ps(b)); + simde__m512i_private c_ = simde__m512i_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + int32_t select = 1; + switch (simde_math_fpclassifyf(s_.f32[i])) { + case SIMDE_MATH_FP_NORMAL: + select = (s_.f32[i] < SIMDE_FLOAT32_C(0.0)) ? 6 : (s_.f32[i] == SIMDE_FLOAT32_C(1.0)) ? 3 : 7; + break; + case SIMDE_MATH_FP_ZERO: + select = 2; + break; + case SIMDE_MATH_FP_NAN: + select = 0; + break; + case SIMDE_MATH_FP_INFINITE: + select = ((s_.f32[i] > SIMDE_FLOAT32_C(0.0)) ? 5 : 4); + break; + } + + switch (((c_.i32[i] >> (select << 2)) & 15)) { + case 0: + r_.f32[i] = a_.f32[i]; + break; + case 1: + r_.f32[i] = b_.f32[i]; + break; + case 2: + r_.f32[i] = SIMDE_MATH_NANF; + break; + case 3: + r_.f32[i] = -SIMDE_MATH_NANF; + break; + case 4: + r_.f32[i] = -SIMDE_MATH_INFINITYF; + break; + case 5: + r_.f32[i] = SIMDE_MATH_INFINITYF; + break; + case 6: + r_.f32[i] = s_.f32[i] < SIMDE_FLOAT32_C(0.0) ? -SIMDE_MATH_INFINITYF : SIMDE_MATH_INFINITYF; + break; + case 7: + r_.f32[i] = SIMDE_FLOAT32_C(-0.0); + break; + case 8: + r_.f32[i] = SIMDE_FLOAT32_C(0.0); + break; + case 9: + r_.f32[i] = SIMDE_FLOAT32_C(-1.0); + break; + case 10: + r_.f32[i] = SIMDE_FLOAT32_C(1.0); + break; + case 11: + r_.f32[i] = SIMDE_FLOAT32_C(0.5); + break; + case 12: + r_.f32[i] = SIMDE_FLOAT32_C(90.0); + break; + case 13: + r_.f32[i] = SIMDE_MATH_PIF / 2; + break; + case 14: + r_.f32[i] = SIMDE_MATH_FLT_MAX; + break; + case 15: + r_.f32[i] = -SIMDE_MATH_FLT_MAX; + break; + } + } + + return simde__m512_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_fixupimm_ps(a, b, c, imm8) _mm512_fixupimm_ps(a, b, c, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_fixupimm_ps + #define _mm512_fixupimm_ps(a, b, c, imm8) simde_mm512_fixupimm_ps(a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_fixupimm_ps(a, k, b, c, imm8) _mm512_mask_fixupimm_ps(a, k, b, c, imm8) +#else + #define simde_mm512_mask_fixupimm_ps(a, k, b, c, imm8) simde_mm512_mask_mov_ps(a, k, simde_mm512_fixupimm_ps(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_fixupimm_ps + #define _mm512_mask_fixupimm_ps(a, k, b, c, imm8) simde_mm512_mask_fixupimm_ps(a, k, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_maskz_fixupimm_ps(k, a, b, c, imm8) _mm512_maskz_fixupimm_ps(k, a, b, c, imm8) +#else + #define simde_mm512_maskz_fixupimm_ps(k, a, b, c, imm8) simde_mm512_maskz_mov_ps(k, simde_mm512_fixupimm_ps(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_fixupimm_ps + #define _mm512_maskz_fixupimm_ps(k, a, b, c, imm8) simde_mm512_maskz_fixupimm_ps(k, a, b, c, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_fixupimm_ss (simde__m128 a, simde__m128 b, simde__m128i c, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b), + s_ = simde__m128_to_private(simde_x_mm_flushsubnormal_ps(b)); + simde__m128i_private c_ = simde__m128i_to_private(c); + + int32_t select = 1; + switch (simde_math_fpclassifyf(s_.f32[0])) { + case SIMDE_MATH_FP_NORMAL: + select = (s_.f32[0] < SIMDE_FLOAT32_C(0.0)) ? 6 : (s_.f32[0] == SIMDE_FLOAT32_C(1.0)) ? 3 : 7; + break; + case SIMDE_MATH_FP_ZERO: + select = 2; + break; + case SIMDE_MATH_FP_NAN: + select = 0; + break; + case SIMDE_MATH_FP_INFINITE: + select = ((s_.f32[0] > SIMDE_FLOAT32_C(0.0)) ? 5 : 4); + break; + } + + switch (((c_.i32[0] >> (select << 2)) & 15)) { + case 0: + b_.f32[0] = a_.f32[0]; + break; + case 2: + b_.f32[0] = SIMDE_MATH_NANF; + break; + case 3: + b_.f32[0] = -SIMDE_MATH_NANF; + break; + case 4: + b_.f32[0] = -SIMDE_MATH_INFINITYF; + break; + case 5: + b_.f32[0] = SIMDE_MATH_INFINITYF; + break; + case 6: + b_.f32[0] = s_.f32[0] < SIMDE_FLOAT32_C(0.0) ? -SIMDE_MATH_INFINITYF : SIMDE_MATH_INFINITYF; + break; + case 7: + b_.f32[0] = SIMDE_FLOAT32_C(-0.0); + break; + case 8: + b_.f32[0] = SIMDE_FLOAT32_C(0.0); + break; + case 9: + b_.f32[0] = SIMDE_FLOAT32_C(-1.0); + break; + case 10: + b_.f32[0] = SIMDE_FLOAT32_C(1.0); + break; + case 11: + b_.f32[0] = SIMDE_FLOAT32_C(0.5); + break; + case 12: + b_.f32[0] = SIMDE_FLOAT32_C(90.0); + break; + case 13: + b_.f32[0] = SIMDE_MATH_PIF / 2; + break; + case 14: + b_.f32[0] = SIMDE_MATH_FLT_MAX; + break; + case 15: + b_.f32[0] = -SIMDE_MATH_FLT_MAX; + break; + } + + return simde__m128_from_private(b_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_fixupimm_ss(a, b, c, imm8) _mm_fixupimm_ss(a, b, c, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_fixupimm_ss + #define _mm_fixupimm_ss(a, b, c, imm8) simde_mm_fixupimm_ss(a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_mask_fixupimm_ss(a, k, b, c, imm8) _mm_mask_fixupimm_ss(a, k, b, c, imm8) +#else + #define simde_mm_mask_fixupimm_ss(a, k, b, c, imm8) simde_mm_mask_mov_ps(a, ((k) | 14), simde_mm_fixupimm_ss(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_fixupimm_ss + #define _mm_mask_fixupimm_ss(a, k, b, c, imm8) simde_mm_mask_fixupimm_ss(a, k, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_maskz_fixupimm_ss(k, a, b, c, imm8) _mm_maskz_fixupimm_ss(k, a, b, c, imm8) +#else + #define simde_mm_maskz_fixupimm_ss(k, a, b, c, imm8) simde_mm_maskz_mov_ps(((k) | 14), simde_mm_fixupimm_ss(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_fixupimm_ss + #define _mm_maskz_fixupimm_ss(k, a, b, c, imm8) simde_mm_maskz_fixupimm_ss(k, a, b, c, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_fixupimm_pd (simde__m128d a, simde__m128d b, simde__m128i c, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b), + s_ = simde__m128d_to_private(simde_x_mm_flushsubnormal_pd(b)); + simde__m128i_private c_ = simde__m128i_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + int32_t select = 1; + switch (simde_math_fpclassify(s_.f64[i])) { + case SIMDE_MATH_FP_NORMAL: + select = (s_.f64[i] < SIMDE_FLOAT64_C(0.0)) ? 6 : (s_.f64[i] == SIMDE_FLOAT64_C(1.0)) ? 3 : 7; + break; + case SIMDE_MATH_FP_ZERO: + select = 2; + break; + case SIMDE_MATH_FP_NAN: + select = 0; + break; + case SIMDE_MATH_FP_INFINITE: + select = ((s_.f64[i] > SIMDE_FLOAT64_C(0.0)) ? 5 : 4); + break; + } + + switch (((c_.i64[i] >> (select << 2)) & 15)) { + case 0: + r_.f64[i] = a_.f64[i]; + break; + case 1: + r_.f64[i] = b_.f64[i]; + break; + case 2: + r_.f64[i] = SIMDE_MATH_NAN; + break; + case 3: + r_.f64[i] = -SIMDE_MATH_NAN; + break; + case 4: + r_.f64[i] = -SIMDE_MATH_INFINITY; + break; + case 5: + r_.f64[i] = SIMDE_MATH_INFINITY; + break; + case 6: + r_.f64[i] = s_.f64[i] < SIMDE_FLOAT64_C(0.0) ? -SIMDE_MATH_INFINITY : SIMDE_MATH_INFINITY; + break; + case 7: + r_.f64[i] = SIMDE_FLOAT64_C(-0.0); + break; + case 8: + r_.f64[i] = SIMDE_FLOAT64_C(0.0); + break; + case 9: + r_.f64[i] = SIMDE_FLOAT64_C(-1.0); + break; + case 10: + r_.f64[i] = SIMDE_FLOAT64_C(1.0); + break; + case 11: + r_.f64[i] = SIMDE_FLOAT64_C(0.5); + break; + case 12: + r_.f64[i] = SIMDE_FLOAT64_C(90.0); + break; + case 13: + r_.f64[i] = SIMDE_MATH_PI / 2; + break; + case 14: + r_.f64[i] = SIMDE_MATH_DBL_MAX; + break; + case 15: + r_.f64[i] = -SIMDE_MATH_DBL_MAX; + break; + } + } + + return simde__m128d_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_fixupimm_pd(a, b, c, imm8) _mm_fixupimm_pd(a, b, c, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_fixupimm_pd + #define _mm_fixupimm_pd(a, b, c, imm8) simde_mm_fixupimm_pd(a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_mask_fixupimm_pd(a, k, b, c, imm8) _mm_mask_fixupimm_pd(a, k, b, c, imm8) +#else + #define simde_mm_mask_fixupimm_pd(a, k, b, c, imm8) simde_mm_mask_mov_pd(a, k, simde_mm_fixupimm_pd(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_fixupimm_pd + #define _mm_mask_fixupimm_pd(a, k, b, c, imm8) simde_mm_mask_fixupimm_pd(a, k, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_maskz_fixupimm_pd(k, a, b, c, imm8) _mm_maskz_fixupimm_pd(k, a, b, c, imm8) +#else + #define simde_mm_maskz_fixupimm_pd(k, a, b, c, imm8) simde_mm_maskz_mov_pd(k, simde_mm_fixupimm_pd(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_fixupimm_pd + #define _mm_maskz_fixupimm_pd(k, a, b, c, imm8) simde_mm_maskz_fixupimm_pd(k, a, b, c, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_fixupimm_pd (simde__m256d a, simde__m256d b, simde__m256i c, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b), + s_ = simde__m256d_to_private(simde_x_mm256_flushsubnormal_pd(b)); + simde__m256i_private c_ = simde__m256i_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + int32_t select = 1; + switch (simde_math_fpclassify(s_.f64[i])) { + case SIMDE_MATH_FP_NORMAL: + select = (s_.f64[i] < SIMDE_FLOAT64_C(0.0)) ? 6 : (s_.f64[i] == SIMDE_FLOAT64_C(1.0)) ? 3 : 7; + break; + case SIMDE_MATH_FP_ZERO: + select = 2; + break; + case SIMDE_MATH_FP_NAN: + select = 0; + break; + case SIMDE_MATH_FP_INFINITE: + select = ((s_.f64[i] > SIMDE_FLOAT64_C(0.0)) ? 5 : 4); + break; + } + + switch (((c_.i64[i] >> (select << 2)) & 15)) { + case 0: + r_.f64[i] = a_.f64[i]; + break; + case 1: + r_.f64[i] = b_.f64[i]; + break; + case 2: + r_.f64[i] = SIMDE_MATH_NAN; + break; + case 3: + r_.f64[i] = -SIMDE_MATH_NAN; + break; + case 4: + r_.f64[i] = -SIMDE_MATH_INFINITY; + break; + case 5: + r_.f64[i] = SIMDE_MATH_INFINITY; + break; + case 6: + r_.f64[i] = s_.f64[i] < SIMDE_FLOAT64_C(0.0) ? -SIMDE_MATH_INFINITY : SIMDE_MATH_INFINITY; + break; + case 7: + r_.f64[i] = SIMDE_FLOAT64_C(-0.0); + break; + case 8: + r_.f64[i] = SIMDE_FLOAT64_C(0.0); + break; + case 9: + r_.f64[i] = SIMDE_FLOAT64_C(-1.0); + break; + case 10: + r_.f64[i] = SIMDE_FLOAT64_C(1.0); + break; + case 11: + r_.f64[i] = SIMDE_FLOAT64_C(0.5); + break; + case 12: + r_.f64[i] = SIMDE_FLOAT64_C(90.0); + break; + case 13: + r_.f64[i] = SIMDE_MATH_PI / 2; + break; + case 14: + r_.f64[i] = SIMDE_MATH_DBL_MAX; + break; + case 15: + r_.f64[i] = -SIMDE_MATH_DBL_MAX; + break; + } + } + + return simde__m256d_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_fixupimm_pd(a, b, c, imm8) _mm256_fixupimm_pd(a, b, c, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_fixupimm_pd + #define _mm256_fixupimm_pd(a, b, c, imm8) simde_mm256_fixupimm_pd(a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_fixupimm_pd(a, k, b, c, imm8) _mm256_mask_fixupimm_pd(a, k, b, c, imm8) +#else + #define simde_mm256_mask_fixupimm_pd(a, k, b, c, imm8) simde_mm256_mask_mov_pd(a, k, simde_mm256_fixupimm_pd(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_fixupimm_pd + #define _mm256_mask_fixupimm_pd(a, k, b, c, imm8) simde_mm256_mask_fixupimm_pd(a, k, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_maskz_fixupimm_pd(k, a, b, c, imm8) _mm256_maskz_fixupimm_pd(k, a, b, c, imm8) +#else + #define simde_mm256_maskz_fixupimm_pd(k, a, b, c, imm8) simde_mm256_maskz_mov_pd(k, simde_mm256_fixupimm_pd(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_fixupimm_pd + #define _mm256_maskz_fixupimm_pd(k, a, b, c, imm8) simde_mm256_maskz_fixupimm_pd(k, a, b, c, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_fixupimm_pd (simde__m512d a, simde__m512d b, simde__m512i c, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b), + s_ = simde__m512d_to_private(simde_x_mm512_flushsubnormal_pd(b)); + simde__m512i_private c_ = simde__m512i_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + int32_t select = 1; + switch (simde_math_fpclassify(s_.f64[i])) { + case SIMDE_MATH_FP_NORMAL: + select = (s_.f64[i] < SIMDE_FLOAT64_C(0.0)) ? 6 : (s_.f64[i] == SIMDE_FLOAT64_C(1.0)) ? 3 : 7; + break; + case SIMDE_MATH_FP_ZERO: + select = 2; + break; + case SIMDE_MATH_FP_NAN: + select = 0; + break; + case SIMDE_MATH_FP_INFINITE: + select = ((s_.f64[i] > SIMDE_FLOAT64_C(0.0)) ? 5 : 4); + break; + } + + switch (((c_.i64[i] >> (select << 2)) & 15)) { + case 0: + r_.f64[i] = a_.f64[i]; + break; + case 1: + r_.f64[i] = b_.f64[i]; + break; + case 2: + r_.f64[i] = SIMDE_MATH_NAN; + break; + case 3: + r_.f64[i] = -SIMDE_MATH_NAN; + break; + case 4: + r_.f64[i] = -SIMDE_MATH_INFINITY; + break; + case 5: + r_.f64[i] = SIMDE_MATH_INFINITY; + break; + case 6: + r_.f64[i] = s_.f64[i] < SIMDE_FLOAT64_C(0.0) ? -SIMDE_MATH_INFINITY : SIMDE_MATH_INFINITY; + break; + case 7: + r_.f64[i] = SIMDE_FLOAT64_C(-0.0); + break; + case 8: + r_.f64[i] = SIMDE_FLOAT64_C(0.0); + break; + case 9: + r_.f64[i] = SIMDE_FLOAT64_C(-1.0); + break; + case 10: + r_.f64[i] = SIMDE_FLOAT64_C(1.0); + break; + case 11: + r_.f64[i] = SIMDE_FLOAT64_C(0.5); + break; + case 12: + r_.f64[i] = SIMDE_FLOAT64_C(90.0); + break; + case 13: + r_.f64[i] = SIMDE_MATH_PI / 2; + break; + case 14: + r_.f64[i] = SIMDE_MATH_DBL_MAX; + break; + case 15: + r_.f64[i] = -SIMDE_MATH_DBL_MAX; + break; + } + } + + return simde__m512d_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_fixupimm_pd(a, b, c, imm8) _mm512_fixupimm_pd(a, b, c, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_fixupimm_pd + #define _mm512_fixupimm_pd(a, b, c, imm8) simde_mm512_fixupimm_pd(a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_fixupimm_pd(a, k, b, c, imm8) _mm512_mask_fixupimm_pd(a, k, b, c, imm8) +#else + #define simde_mm512_mask_fixupimm_pd(a, k, b, c, imm8) simde_mm512_mask_mov_pd(a, k, simde_mm512_fixupimm_pd(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_fixupimm_pd + #define _mm512_mask_fixupimm_pd(a, k, b, c, imm8) simde_mm512_mask_fixupimm_pd(a, k, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_maskz_fixupimm_pd(k, a, b, c, imm8) _mm512_maskz_fixupimm_pd(k, a, b, c, imm8) +#else + #define simde_mm512_maskz_fixupimm_pd(k, a, b, c, imm8) simde_mm512_maskz_mov_pd(k, simde_mm512_fixupimm_pd(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_fixupimm_pd + #define _mm512_maskz_fixupimm_pd(k, a, b, c, imm8) simde_mm512_maskz_fixupimm_pd(k, a, b, c, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_fixupimm_sd (simde__m128d a, simde__m128d b, simde__m128i c, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b), + s_ = simde__m128d_to_private(simde_x_mm_flushsubnormal_pd(b)); + simde__m128i_private c_ = simde__m128i_to_private(c); + + int32_t select = 1; + switch (simde_math_fpclassify(s_.f64[0])) { + case SIMDE_MATH_FP_NORMAL: + select = (s_.f64[0] < SIMDE_FLOAT64_C(0.0)) ? 6 : (s_.f64[0] == SIMDE_FLOAT64_C(1.0)) ? 3 : 7; + break; + case SIMDE_MATH_FP_ZERO: + select = 2; + break; + case SIMDE_MATH_FP_NAN: + select = 0; + break; + case SIMDE_MATH_FP_INFINITE: + select = ((s_.f64[0] > SIMDE_FLOAT64_C(0.0)) ? 5 : 4); + break; + } + + switch (((c_.i64[0] >> (select << 2)) & 15)) { + case 0: + b_.f64[0] = a_.f64[0]; + break; + case 1: + b_.f64[0] = b_.f64[0]; + break; + case 2: + b_.f64[0] = SIMDE_MATH_NAN; + break; + case 3: + b_.f64[0] = -SIMDE_MATH_NAN; + break; + case 4: + b_.f64[0] = -SIMDE_MATH_INFINITY; + break; + case 5: + b_.f64[0] = SIMDE_MATH_INFINITY; + break; + case 6: + b_.f64[0] = s_.f64[0] < SIMDE_FLOAT64_C(0.0) ? -SIMDE_MATH_INFINITY : SIMDE_MATH_INFINITY; + break; + case 7: + b_.f64[0] = SIMDE_FLOAT64_C(-0.0); + break; + case 8: + b_.f64[0] = SIMDE_FLOAT64_C(0.0); + break; + case 9: + b_.f64[0] = SIMDE_FLOAT64_C(-1.0); + break; + case 10: + b_.f64[0] = SIMDE_FLOAT64_C(1.0); + break; + case 11: + b_.f64[0] = SIMDE_FLOAT64_C(0.5); + break; + case 12: + b_.f64[0] = SIMDE_FLOAT64_C(90.0); + break; + case 13: + b_.f64[0] = SIMDE_MATH_PI / 2; + break; + case 14: + b_.f64[0] = SIMDE_MATH_DBL_MAX; + break; + case 15: + b_.f64[0] = -SIMDE_MATH_DBL_MAX; + break; + } + + return simde__m128d_from_private(b_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_fixupimm_sd(a, b, c, imm8) _mm_fixupimm_sd(a, b, c, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_fixupimm_sd + #define _mm_fixupimm_sd(a, b, c, imm8) simde_mm_fixupimm_sd(a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_mask_fixupimm_sd(a, k, b, c, imm8) _mm_mask_fixupimm_sd(a, k, b, c, imm8) +#else + #define simde_mm_mask_fixupimm_sd(a, k, b, c, imm8) simde_mm_mask_mov_pd(a, ((k) | 2), simde_mm_fixupimm_sd(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_fixupimm_sd + #define _mm_mask_fixupimm_sd(a, k, b, c, imm8) simde_mm_mask_fixupimm_sd(a, k, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_maskz_fixupimm_sd(k, a, b, c, imm8) _mm_maskz_fixupimm_sd(k, a, b, c, imm8) +#else + #define simde_mm_maskz_fixupimm_sd(k, a, b, c, imm8) simde_mm_maskz_mov_pd(((k) | 2), simde_mm_fixupimm_sd(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_fixupimm_sd + #define _mm_maskz_fixupimm_sd(k, a, b, c, imm8) simde_mm_maskz_fixupimm_sd(k, a, b, c, imm8) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_FIXUPIMM_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fixupimm_round.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fixupimm_round.h new file mode 100644 index 00000000..636b82a8 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fixupimm_round.h @@ -0,0 +1,687 @@ +#if !defined(SIMDE_X86_AVX512_FIXUPIMM_ROUND_H) +#define SIMDE_X86_AVX512_FIXUPIMM_ROUND_H + +#include "types.h" +#include "fixupimm.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_fixupimm_round_ps(a, b, c, imm8, sae) _mm512_fixupimm_round_ps(a, b, c, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_fixupimm_round_ps(a, b, c, imm8, sae) simde_mm512_fixupimm_ps(a, b, c, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_fixupimm_round_ps(a, b, c, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512 simde_mm512_fixupimm_round_ps_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_fixupimm_round_ps_envp; \ + int simde_mm512_fixupimm_round_ps_x = feholdexcept(&simde_mm512_fixupimm_round_ps_envp); \ + simde_mm512_fixupimm_round_ps_r = simde_mm512_fixupimm_ps(a, b, c, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_fixupimm_round_ps_x == 0)) \ + fesetenv(&simde_mm512_fixupimm_round_ps_envp); \ + } \ + else { \ + simde_mm512_fixupimm_round_ps_r = simde_mm512_fixupimm_ps(a, b, c, imm8); \ + } \ + \ + simde_mm512_fixupimm_round_ps_r; \ + })) + #else + #define simde_mm512_fixupimm_round_ps(a, b, c, imm8, sae) simde_mm512_fixupimm_ps(a, b, c, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512 + simde_mm512_fixupimm_round_ps (simde__m512 a, simde__m512 b, simde__m512i c, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m512 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_fixupimm_ps(a, b, c, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_fixupimm_ps(a, b, c, imm8); + #endif + } + else { + r = simde_mm512_fixupimm_ps(a, b, c, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_fixupimm_round_ps + #define _mm512_fixupimm_round_ps(a, b, c, imm8, sae) simde_mm512_fixupimm_round_ps(a, b, c, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_fixupimm_round_ps(a, k, b, c, imm8, sae) _mm512_mask_fixupimm_round_ps(a, k, b, c, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_mask_fixupimm_round_ps(a, k, b, c, imm8, sae) simde_mm512_mask_fixupimm_ps(a, k, b, c, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_mask_fixupimm_round_ps(a, k, b, c, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512 simde_mm512_mask_fixupimm_round_ps_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_mask_fixupimm_round_ps_envp; \ + int simde_mm512_mask_fixupimm_round_ps_x = feholdexcept(&simde_mm512_mask_fixupimm_round_ps_envp); \ + simde_mm512_mask_fixupimm_round_ps_r = simde_mm512_mask_fixupimm_ps(a, k, b, c, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_mask_fixupimm_round_ps_x == 0)) \ + fesetenv(&simde_mm512_mask_fixupimm_round_ps_envp); \ + } \ + else { \ + simde_mm512_mask_fixupimm_round_ps_r = simde_mm512_mask_fixupimm_ps(a, k, b, c, imm8); \ + } \ + \ + simde_mm512_mask_fixupimm_round_ps_r; \ + })) + #else + #define simde_mm512_mask_fixupimm_round_ps(a, k, b, c, imm8, sae) simde_mm512_mask_fixupimm_ps(a, k, b, c, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512 + simde_mm512_mask_fixupimm_round_ps (simde__m512 a, simde__mmask16 k, simde__m512 b, simde__m512i c, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m512 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_mask_fixupimm_ps(a, k, b, c, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_mask_fixupimm_ps(a, k, b, c, imm8); + #endif + } + else { + r = simde_mm512_mask_fixupimm_ps(a, k, b, c, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_fixupimm_round_ps + #define _mm512_mask_fixupimm_round_ps(a, k, b, c, imm8, sae) simde_mm512_mask_fixupimm_round_ps(a, k, b, c, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_maskz_fixupimm_round_ps(k, a, b, c, imm8, sae) _mm512_maskz_fixupimm_round_ps(k, a, b, c, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_maskz_fixupimm_round_ps(k, a, b, c, imm8, sae) simde_mm512_maskz_fixupimm_ps(k, a, b, c, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_maskz_fixupimm_round_ps(k, a, b, c, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512 simde_mm512_maskz_fixupimm_round_ps_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_maskz_fixupimm_round_ps_envp; \ + int simde_mm512_maskz_fixupimm_round_ps_x = feholdexcept(&simde_mm512_maskz_fixupimm_round_ps_envp); \ + simde_mm512_maskz_fixupimm_round_ps_r = simde_mm512_maskz_fixupimm_ps(k, a, b, c, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_maskz_fixupimm_round_ps_x == 0)) \ + fesetenv(&simde_mm512_maskz_fixupimm_round_ps_envp); \ + } \ + else { \ + simde_mm512_maskz_fixupimm_round_ps_r = simde_mm512_maskz_fixupimm_ps(k, a, b, c, imm8); \ + } \ + \ + simde_mm512_maskz_fixupimm_round_ps_r; \ + })) + #else + #define simde_mm512_maskz_fixupimm_round_ps(k, a, b, c, imm8, sae) simde_mm512_maskz_fixupimm_ps(k, a, b, c, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512 + simde_mm512_maskz_fixupimm_round_ps (simde__mmask16 k, simde__m512 a, simde__m512 b, simde__m512i c, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m512 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_maskz_fixupimm_ps(k, a, b, c, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_maskz_fixupimm_ps(k, a, b, c, imm8); + #endif + } + else { + r = simde_mm512_maskz_fixupimm_ps(k, a, b, c, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_fixupimm_round_ps + #define _mm512_maskz_fixupimm_round_ps(k, a, b, c, imm8, sae) simde_mm512_maskz_fixupimm_round_ps(k, a, b, c, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_fixupimm_round_pd(a, b, c, imm8, sae) _mm512_fixupimm_round_pd(a, b, c, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_fixupimm_round_pd(a, b, c, imm8, sae) simde_mm512_fixupimm_pd(a, b, c, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_fixupimm_round_pd(a, b, c, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d simde_mm512_fixupimm_round_pd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_fixupimm_round_pd_envp; \ + int simde_mm512_fixupimm_round_pd_x = feholdexcept(&simde_mm512_fixupimm_round_pd_envp); \ + simde_mm512_fixupimm_round_pd_r = simde_mm512_fixupimm_pd(a, b, c, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_fixupimm_round_pd_x == 0)) \ + fesetenv(&simde_mm512_fixupimm_round_pd_envp); \ + } \ + else { \ + simde_mm512_fixupimm_round_pd_r = simde_mm512_fixupimm_pd(a, b, c, imm8); \ + } \ + \ + simde_mm512_fixupimm_round_pd_r; \ + })) + #else + #define simde_mm512_fixupimm_round_pd(a, b, c, imm8, sae) simde_mm512_fixupimm_pd(a, b, c, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512d + simde_mm512_fixupimm_round_pd (simde__m512d a, simde__m512d b, simde__m512i c, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m512d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_fixupimm_pd(a, b, c, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_fixupimm_pd(a, b, c, imm8); + #endif + } + else { + r = simde_mm512_fixupimm_pd(a, b, c, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_fixupimm_round_pd + #define _mm512_fixupimm_round_pd(a, b, c, imm8, sae) simde_mm512_fixupimm_round_pd(a, b, c, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_fixupimm_round_pd(a, k, b, c, imm8, sae) _mm512_mask_fixupimm_round_pd(a, k, b, c, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_mask_fixupimm_round_pd(a, k, b, c, imm8, sae) simde_mm512_mask_fixupimm_pd(a, k, b, c, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_mask_fixupimm_round_pd(a, k, b, c, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d simde_mm512_mask_fixupimm_round_pd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_mask_fixupimm_round_pd_envp; \ + int simde_mm512_mask_fixupimm_round_pd_x = feholdexcept(&simde_mm512_mask_fixupimm_round_pd_envp); \ + simde_mm512_mask_fixupimm_round_pd_r = simde_mm512_mask_fixupimm_pd(a, k, b, c, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_mask_fixupimm_round_pd_x == 0)) \ + fesetenv(&simde_mm512_mask_fixupimm_round_pd_envp); \ + } \ + else { \ + simde_mm512_mask_fixupimm_round_pd_r = simde_mm512_mask_fixupimm_pd(a, k, b, c, imm8); \ + } \ + \ + simde_mm512_mask_fixupimm_round_pd_r; \ + })) + #else + #define simde_mm512_mask_fixupimm_round_pd(a, k, b, c, imm8, sae) simde_mm512_mask_fixupimm_pd(a, k, b, c, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512d + simde_mm512_mask_fixupimm_round_pd (simde__m512d a, simde__mmask8 k, simde__m512d b, simde__m512i c, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m512d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_mask_fixupimm_pd(a, k, b, c, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_mask_fixupimm_pd(a, k, b, c, imm8); + #endif + } + else { + r = simde_mm512_mask_fixupimm_pd(a, k, b, c, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_fixupimm_round_pd + #define _mm512_mask_fixupimm_round_pd(a, k, b, c, imm8, sae) simde_mm512_mask_fixupimm_round_pd(a, k, b, c, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_maskz_fixupimm_round_pd(k, a, b, c, imm8, sae) _mm512_maskz_fixupimm_round_pd(k, a, b, c, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_maskz_fixupimm_round_pd(k, a, b, c, imm8, sae) simde_mm512_maskz_fixupimm_pd(k, a, b, c, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_maskz_fixupimm_round_pd(k, a, b, c, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d simde_mm512_maskz_fixupimm_round_pd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_maskz_fixupimm_round_pd_envp; \ + int simde_mm512_maskz_fixupimm_round_pd_x = feholdexcept(&simde_mm512_maskz_fixupimm_round_pd_envp); \ + simde_mm512_maskz_fixupimm_round_pd_r = simde_mm512_maskz_fixupimm_pd(k, a, b, c, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_maskz_fixupimm_round_pd_x == 0)) \ + fesetenv(&simde_mm512_maskz_fixupimm_round_pd_envp); \ + } \ + else { \ + simde_mm512_maskz_fixupimm_round_pd_r = simde_mm512_maskz_fixupimm_pd(k, a, b, c, imm8); \ + } \ + \ + simde_mm512_maskz_fixupimm_round_pd_r; \ + })) + #else + #define simde_mm512_maskz_fixupimm_round_pd(k, a, b, c, imm8, sae) simde_mm512_maskz_fixupimm_pd(k, a, b, c, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512d + simde_mm512_maskz_fixupimm_round_pd (simde__mmask8 k, simde__m512d a, simde__m512d b, simde__m512i c, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m512d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_maskz_fixupimm_pd(k, a, b, c, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_maskz_fixupimm_pd(k, a, b, c, imm8); + #endif + } + else { + r = simde_mm512_maskz_fixupimm_pd(k, a, b, c, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_fixupimm_round_pd + #define _mm512_maskz_fixupimm_round_pd(k, a, b, c, imm8, sae) simde_mm512_maskz_fixupimm_round_pd(k, a, b, c, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_fixupimm_round_ss(a, b, c, imm8, sae) _mm_fixupimm_round_ss(a, b, c, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_fixupimm_round_ss(a, b, c, imm8, sae) simde_mm_fixupimm_ss(a, b, c, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_fixupimm_round_ss(a, b, c, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128 simde_mm_fixupimm_round_ss_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_fixupimm_round_ss_envp; \ + int simde_mm_fixupimm_round_ss_x = feholdexcept(&simde_mm_fixupimm_round_ss_envp); \ + simde_mm_fixupimm_round_ss_r = simde_mm_fixupimm_ss(a, b, c, imm8); \ + if (HEDLEY_LIKELY(simde_mm_fixupimm_round_ss_x == 0)) \ + fesetenv(&simde_mm_fixupimm_round_ss_envp); \ + } \ + else { \ + simde_mm_fixupimm_round_ss_r = simde_mm_fixupimm_ss(a, b, c, imm8); \ + } \ + \ + simde_mm_fixupimm_round_ss_r; \ + })) + #else + #define simde_mm_fixupimm_round_ss(a, b, c, imm8, sae) simde_mm_fixupimm_ss(a, b, c, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_fixupimm_round_ss (simde__m128 a, simde__m128 b, simde__m128i c, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_fixupimm_ss(a, b, c, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_fixupimm_ss(a, b, c, imm8); + #endif + } + else { + r = simde_mm_fixupimm_ss(a, b, c, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_fixupimm_round_ss + #define _mm_fixupimm_round_ss(a, b, c, imm8, sae) simde_mm_fixupimm_round_ss(a, b, c, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_mask_fixupimm_round_ss(a, k, b, c, imm8, sae) _mm_mask_fixupimm_round_ss(a, k, b, c, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_mask_fixupimm_round_ss(a, k, b, c, imm8, sae) simde_mm_mask_fixupimm_ss(a, k, b, c, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_mask_fixupimm_round_ss(a, k, b, c, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128 simde_mm_mask_fixupimm_round_ss_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_mask_fixupimm_round_ss_envp; \ + int simde_mm_mask_fixupimm_round_ss_x = feholdexcept(&simde_mm_mask_fixupimm_round_ss_envp); \ + simde_mm_mask_fixupimm_round_ss_r = simde_mm_mask_fixupimm_ss(a, k, b, c, imm8); \ + if (HEDLEY_LIKELY(simde_mm_mask_fixupimm_round_ss_x == 0)) \ + fesetenv(&simde_mm_mask_fixupimm_round_ss_envp); \ + } \ + else { \ + simde_mm_mask_fixupimm_round_ss_r = simde_mm_mask_fixupimm_ss(a, k, b, c, imm8); \ + } \ + \ + simde_mm_mask_fixupimm_round_ss_r; \ + })) + #else + #define simde_mm_mask_fixupimm_round_ss(a, k, b, c, imm8, sae) simde_mm_mask_fixupimm_ss(a, k, b, c, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_mask_fixupimm_round_ss (simde__m128 a, simde__mmask8 k, simde__m128 b, simde__m128i c, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_mask_fixupimm_ss(a, k, b, c, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_mask_fixupimm_ss(a, k, b, c, imm8); + #endif + } + else { + r = simde_mm_mask_fixupimm_ss(a, k, b, c, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_fixupimm_round_ss + #define _mm_mask_fixupimm_round_ss(a, k, b, c, imm8, sae) simde_mm_mask_fixupimm_round_ss(a, k, b, c, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_maskz_fixupimm_round_ss(k, a, b, c, imm8, sae) _mm_maskz_fixupimm_round_ss(k, a, b, c, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_maskz_fixupimm_round_ss(k, a, b, c, imm8, sae) simde_mm_maskz_fixupimm_ss(k, a, b, c, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_maskz_fixupimm_round_ss(k, a, b, c, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128 simde_mm_maskz_fixupimm_round_ss_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_maskz_fixupimm_round_ss_envp; \ + int simde_mm_maskz_fixupimm_round_ss_x = feholdexcept(&simde_mm_maskz_fixupimm_round_ss_envp); \ + simde_mm_maskz_fixupimm_round_ss_r = simde_mm_maskz_fixupimm_ss(k, a, b, c, imm8); \ + if (HEDLEY_LIKELY(simde_mm_maskz_fixupimm_round_ss_x == 0)) \ + fesetenv(&simde_mm_maskz_fixupimm_round_ss_envp); \ + } \ + else { \ + simde_mm_maskz_fixupimm_round_ss_r = simde_mm_maskz_fixupimm_ss(k, a, b, c, imm8); \ + } \ + \ + simde_mm_maskz_fixupimm_round_ss_r; \ + })) + #else + #define simde_mm_maskz_fixupimm_round_ss(k, a, b, c, imm8, sae) simde_mm_maskz_fixupimm_ss(k, a, b, c, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_maskz_fixupimm_round_ss (simde__mmask8 k, simde__m128 a, simde__m128 b, simde__m128i c, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_maskz_fixupimm_ss(k, a, b, c, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_maskz_fixupimm_ss(k, a, b, c, imm8); + #endif + } + else { + r = simde_mm_maskz_fixupimm_ss(k, a, b, c, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_fixupimm_round_ss + #define _mm_maskz_fixupimm_round_ss(k, a, b, c, imm8, sae) simde_mm_maskz_fixupimm_round_ss(k, a, b, c, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_fixupimm_round_sd(a, b, c, imm8, sae) _mm_fixupimm_round_sd(a, b, c, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_fixupimm_round_sd(a, b, c, imm8, sae) simde_mm_fixupimm_sd(a, b, c, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_fixupimm_round_sd(a, b, c, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128d simde_mm_fixupimm_round_sd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_fixupimm_round_sd_envp; \ + int simde_mm_fixupimm_round_sd_x = feholdexcept(&simde_mm_fixupimm_round_sd_envp); \ + simde_mm_fixupimm_round_sd_r = simde_mm_fixupimm_sd(a, b, c, imm8); \ + if (HEDLEY_LIKELY(simde_mm_fixupimm_round_sd_x == 0)) \ + fesetenv(&simde_mm_fixupimm_round_sd_envp); \ + } \ + else { \ + simde_mm_fixupimm_round_sd_r = simde_mm_fixupimm_sd(a, b, c, imm8); \ + } \ + \ + simde_mm_fixupimm_round_sd_r; \ + })) + #else + #define simde_mm_fixupimm_round_sd(a, b, c, imm8, sae) simde_mm_fixupimm_sd(a, b, c, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_fixupimm_round_sd (simde__m128d a, simde__m128d b, simde__m128i c, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_fixupimm_sd(a, b, c, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_fixupimm_sd(a, b, c, imm8); + #endif + } + else { + r = simde_mm_fixupimm_sd(a, b, c, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_fixupimm_round_sd + #define _mm_fixupimm_round_sd(a, b, c, imm8, sae) simde_mm_fixupimm_round_sd(a, b, c, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_mask_fixupimm_round_sd(a, k, b, c, imm8, sae) _mm_mask_fixupimm_round_sd(a, k, b, c, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_mask_fixupimm_round_sd(a, k, b, c, imm8, sae) simde_mm_mask_fixupimm_sd(a, k, b, c, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_mask_fixupimm_round_sd(a, k, b, c, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128d simde_mm_mask_fixupimm_round_sd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_mask_fixupimm_round_sd_envp; \ + int simde_mm_mask_fixupimm_round_sd_x = feholdexcept(&simde_mm_mask_fixupimm_round_sd_envp); \ + simde_mm_mask_fixupimm_round_sd_r = simde_mm_mask_fixupimm_sd(a, k, b, c, imm8); \ + if (HEDLEY_LIKELY(simde_mm_mask_fixupimm_round_sd_x == 0)) \ + fesetenv(&simde_mm_mask_fixupimm_round_sd_envp); \ + } \ + else { \ + simde_mm_mask_fixupimm_round_sd_r = simde_mm_mask_fixupimm_sd(a, k, b, c, imm8); \ + } \ + \ + simde_mm_mask_fixupimm_round_sd_r; \ + })) + #else + #define simde_mm_mask_fixupimm_round_sd(a, k, b, c, imm8, sae) simde_mm_mask_fixupimm_sd(a, k, b, c, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_mask_fixupimm_round_sd (simde__m128d a, simde__mmask8 k, simde__m128d b, simde__m128i c, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_mask_fixupimm_sd(a, k, b, c, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_mask_fixupimm_sd(a, k, b, c, imm8); + #endif + } + else { + r = simde_mm_mask_fixupimm_sd(a, k, b, c, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_fixupimm_round_sd + #define _mm_mask_fixupimm_round_sd(a, k, b, c, imm8, sae) simde_mm_mask_fixupimm_round_sd(a, k, b, c, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_maskz_fixupimm_round_sd(k, a, b, c, imm8, sae) _mm_maskz_fixupimm_round_sd(k, a, b, c, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_maskz_fixupimm_round_sd(k, a, b, c, imm8, sae) simde_mm_maskz_fixupimm_sd(k, a, b, c, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_maskz_fixupimm_round_sd(k, a, b, c, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128d simde_mm_maskz_fixupimm_round_sd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_maskz_fixupimm_round_sd_envp; \ + int simde_mm_maskz_fixupimm_round_sd_x = feholdexcept(&simde_mm_maskz_fixupimm_round_sd_envp); \ + simde_mm_maskz_fixupimm_round_sd_r = simde_mm_maskz_fixupimm_sd(k, a, b, c, imm8); \ + if (HEDLEY_LIKELY(simde_mm_maskz_fixupimm_round_sd_x == 0)) \ + fesetenv(&simde_mm_maskz_fixupimm_round_sd_envp); \ + } \ + else { \ + simde_mm_maskz_fixupimm_round_sd_r = simde_mm_maskz_fixupimm_sd(k, a, b, c, imm8); \ + } \ + \ + simde_mm_maskz_fixupimm_round_sd_r; \ + })) + #else + #define simde_mm_maskz_fixupimm_round_sd(k, a, b, c, imm8, sae) simde_mm_maskz_fixupimm_sd(k, a, b, c, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_maskz_fixupimm_round_sd (simde__mmask8 k, simde__m128d a, simde__m128d b, simde__m128i c, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_maskz_fixupimm_sd(k, a, b, c, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_maskz_fixupimm_sd(k, a, b, c, imm8); + #endif + } + else { + r = simde_mm_maskz_fixupimm_sd(k, a, b, c, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_fixupimm_round_sd + #define _mm_maskz_fixupimm_round_sd(k, a, b, c, imm8, sae) simde_mm_maskz_fixupimm_round_sd(k, a, b, c, imm8, sae) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_FIXUPIMM_ROUND_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/flushsubnormal.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/flushsubnormal.h new file mode 100644 index 00000000..6830e7c6 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/flushsubnormal.h @@ -0,0 +1,91 @@ +#if !defined(SIMDE_X86_AVX512_FLUSHSUBNORMAL_H) +#define SIMDE_X86_AVX512_FLUSHSUBNORMAL_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_x_mm_flushsubnormal_ps (simde__m128 a) { + simde__m128_private a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + a_.f32[i] = simde_math_issubnormalf(a_.f32[i]) ? 0 : a_.f32[i]; + } + + return simde__m128_from_private(a_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_x_mm256_flushsubnormal_ps (simde__m256 a) { + simde__m256_private a_ = simde__m256_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + a_.f32[i] = simde_math_issubnormalf(a_.f32[i]) ? 0 : a_.f32[i]; + } + + return simde__m256_from_private(a_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_x_mm512_flushsubnormal_ps (simde__m512 a) { + simde__m512_private a_ = simde__m512_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + a_.f32[i] = simde_math_issubnormalf(a_.f32[i]) ? 0 : a_.f32[i]; + } + + return simde__m512_from_private(a_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_x_mm_flushsubnormal_pd (simde__m128d a) { + simde__m128d_private a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + a_.f64[i] = simde_math_issubnormal(a_.f64[i]) ? 0 : a_.f64[i]; + } + + return simde__m128d_from_private(a_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_x_mm256_flushsubnormal_pd (simde__m256d a) { + simde__m256d_private a_ = simde__m256d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + a_.f64[i] = simde_math_issubnormal(a_.f64[i]) ? 0 : a_.f64[i]; + } + + return simde__m256d_from_private(a_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_x_mm512_flushsubnormal_pd (simde__m512d a) { + simde__m512d_private a_ = simde__m512d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + a_.f64[i] = simde_math_issubnormal(a_.f64[i]) ? 0 : a_.f64[i]; + } + + return simde__m512d_from_private(a_); +} + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_FLUSHSUBNORMAL_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fmadd.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fmadd.h new file mode 100644 index 00000000..0a89b4cd --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fmadd.h @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + */ + +#if !defined(SIMDE_X86_AVX512_FMADD_H) +#define SIMDE_X86_AVX512_FMADD_H + +#include "types.h" +#include "mov.h" +#include "../fma.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_fmadd_ps (simde__m512 a, simde__m512 b, simde__m512 c) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_fmadd_ps(a, b, c); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b), + c_ = simde__m512_to_private(c); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_fmadd_ps(a_.m256[i], b_.m256[i], c_.m256[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = (a_.f32 * b_.f32) + c_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = (a_.f32[i] * b_.f32[i]) + c_.f32[i]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_fmadd_ps + #define _mm512_fmadd_ps(a, b, c) simde_mm512_fmadd_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_fmadd_ps(simde__m512 a, simde__mmask16 k, simde__m512 b, simde__m512 c) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_fmadd_ps(a, k, b, c); + #else + return simde_mm512_mask_mov_ps(a, k, simde_mm512_fmadd_ps(a, b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_fmadd_ps + #define _mm512_mask_fmadd_ps(a, k, b, c) simde_mm512_mask_fmadd_ps(a, k, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_fmadd_ps(simde__mmask16 k, simde__m512 a, simde__m512 b, simde__m512 c) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_fmadd_ps(k, a, b, c); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_fmadd_ps(a, b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_fmadd_ps + #define _mm512_maskz_fmadd_ps(k, a, b, c) simde_mm512_maskz_fmadd_ps(k, a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_fmadd_pd (simde__m512d a, simde__m512d b, simde__m512d c) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_fmadd_pd(a, b, c); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b), + c_ = simde__m512d_to_private(c); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_fmadd_pd(a_.m256d[i], b_.m256d[i], c_.m256d[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = (a_.f64 * b_.f64) + c_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = (a_.f64[i] * b_.f64[i]) + c_.f64[i]; + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_fmadd_pd + #define _mm512_fmadd_pd(a, b, c) simde_mm512_fmadd_pd(a, b, c) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_FMADD_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fmsub.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fmsub.h new file mode 100644 index 00000000..4f52d407 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fmsub.h @@ -0,0 +1,276 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 kitegi + */ + +#if !defined(SIMDE_X86_AVX512_FMSUB_H) +#define SIMDE_X86_AVX512_FMSUB_H + +#include "types.h" +#include "mov.h" +#include "../fma.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mask3_fmsub_pd (simde__m256d a, simde__m256d b, simde__m256d c, simde__mmask8 k) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask3_fmsub_pd(a, b, c, k); + #else + return simde_mm256_mask_mov_pd(c, k, simde_mm256_fmsub_pd(a, b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask3_fmsub_pd + #define _mm256_mask3_fmsub_pd(a, b, c, k) simde_mm256_mask3_fmsub_pd(a, b, c, k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mask_fmsub_pd (simde__m256d a, simde__mmask8 k, simde__m256d b, simde__m256d c) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_fmsub_pd(a, k, b, c); + #else + return simde_mm256_mask_mov_pd(a, k, simde_mm256_fmsub_pd(a, b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_fmsub_pd + #define _mm256_mask_fmsub_pd(a, k, b, c) simde_mm256_mask_fmsub_pd(a, k, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_maskz_fmsub_pd (simde__mmask8 k, simde__m256d a, simde__m256d b, simde__m256d c) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_fmsub_pd(k, a, b, c); + #else + return simde_mm256_maskz_mov_pd(k, simde_mm256_fmsub_pd(a, b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_fmsub_pd + #define _mm256_maskz_fmsub_pd(k, a, b, c) simde_mm256_maskz_fmsub_pd(k, a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mask3_fmsub_pd (simde__m128d a, simde__m128d b, simde__m128d c, simde__mmask8 k) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask3_fmsub_pd(a, b, c, k); + #else + return simde_mm_mask_mov_pd(c, k, simde_mm_fmsub_pd(a, b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask3_fmsub_pd + #define _mm_mask3_fmsub_pd(a, b, c, k) simde_mm_mask3_fmsub_pd(a, b, c, k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mask_fmsub_pd (simde__m128d a, simde__mmask8 k, simde__m128d b, simde__m128d c) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_fmsub_pd(a, k, b, c); + #else + return simde_mm_mask_mov_pd(a, k, simde_mm_fmsub_pd(a, b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_fmsub_pd + #define _mm_mask_fmsub_pd(a, k, b, c) simde_mm_mask_fmsub_pd(a, k, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_maskz_fmsub_pd (simde__mmask8 k, simde__m128d a, simde__m128d b, simde__m128d c) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_fmsub_pd(k, a, b, c); + #else + return simde_mm_maskz_mov_pd(k, simde_mm_fmsub_pd(a, b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_fmsub_pd + #define _mm_maskz_fmsub_pd(k, a, b, c) simde_mm_maskz_fmsub_pd(k, a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask3_fmsub_ps (simde__m256 a, simde__m256 b, simde__m256 c, simde__mmask8 k) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask3_fmsub_ps(a, b, c, k); + #else + return simde_mm256_mask_mov_ps(c, k, simde_mm256_fmsub_ps(a, b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask3_fmsub_ps + #define _mm256_mask3_fmsub_ps(a, b, c, k) simde_mm256_mask3_fmsub_ps(a, b, c, k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask_fmsub_ps (simde__m256 a, simde__mmask8 k, simde__m256 b, simde__m256 c) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_fmsub_ps(a, k, b, c); + #else + return simde_mm256_mask_mov_ps(a, k, simde_mm256_fmsub_ps(a, b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_fmsub_ps + #define _mm256_mask_fmsub_ps(a, k, b, c) simde_mm256_mask_fmsub_ps(a, k, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_maskz_fmsub_ps (simde__mmask8 k, simde__m256 a, simde__m256 b, simde__m256 c) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_fmsub_ps(k, a, b, c); + #else + return simde_mm256_maskz_mov_ps(k, simde_mm256_fmsub_ps(a, b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_fmsub_ps + #define _mm256_maskz_fmsub_ps(k, a, b, c) simde_mm256_maskz_fmsub_ps(k, a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mask3_fmsub_ps (simde__m128 a, simde__m128 b, simde__m128 c, simde__mmask8 k) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask3_fmsub_ps(a, b, c, k); + #else + return simde_mm_mask_mov_ps(c, k, simde_mm_fmsub_ps(a, b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask3_fmsub_ps + #define _mm_mask3_fmsub_ps(a, b, c, k) simde_mm_mask3_fmsub_ps(a, b, c, k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mask_fmsub_ps (simde__m128 a, simde__mmask8 k, simde__m128 b, simde__m128 c) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_fmsub_ps(a, k, b, c); + #else + return simde_mm_mask_mov_ps(a, k, simde_mm_fmsub_ps(a, b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_fmsub_ps + #define _mm_mask_fmsub_ps(a, k, b, c) simde_mm_mask_fmsub_ps(a, k, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_maskz_fmsub_ps (simde__mmask8 k, simde__m128 a, simde__m128 b, simde__m128 c) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_fmsub_ps(k, a, b, c); + #else + return simde_mm_maskz_mov_ps(k, simde_mm_fmsub_ps(a, b, c)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_fmsub_ps + #define _mm_maskz_fmsub_ps(k, a, b, c) simde_mm_maskz_fmsub_ps(k, a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_fmsub_ps (simde__m512 a, simde__m512 b, simde__m512 c) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_fmsub_ps(a, b, c); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b), + c_ = simde__m512_to_private(c); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_fmsub_ps(a_.m256[i], b_.m256[i], c_.m256[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = (a_.f32 * b_.f32) - c_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = (a_.f32[i] * b_.f32[i]) - c_.f32[i]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_fmsub_ps + #define _mm512_fmsub_ps(a, b, c) simde_mm512_fmsub_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_fmsub_pd (simde__m512d a, simde__m512d b, simde__m512d c) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_fmsub_pd(a, b, c); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b), + c_ = simde__m512d_to_private(c); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_fmsub_pd(a_.m256d[i], b_.m256d[i], c_.m256d[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = (a_.f64 * b_.f64) - c_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = (a_.f64[i] * b_.f64[i]) - c_.f64[i]; + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_fmsub_pd + #define _mm512_fmsub_pd(a, b, c) simde_mm512_fmsub_pd(a, b, c) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_FMSUB_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fnmadd.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fnmadd.h new file mode 100644 index 00000000..6779dbd5 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fnmadd.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 kitegi + */ + +#if !defined(SIMDE_X86_AVX512_FNMADD_H) +#define SIMDE_X86_AVX512_FNMADD_H + +#include "types.h" +#include "mov.h" +#include "../fma.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_fnmadd_ps (simde__m512 a, simde__m512 b, simde__m512 c) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_fnmadd_ps(a, b, c); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b), + c_ = simde__m512_to_private(c); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_fnmadd_ps(a_.m256[i], b_.m256[i], c_.m256[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = -(a_.f32 * b_.f32) + c_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = -(a_.f32[i] * b_.f32[i]) + c_.f32[i]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_fnmadd_ps + #define _mm512_fnmadd_ps(a, b, c) simde_mm512_fnmadd_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_fnmadd_pd (simde__m512d a, simde__m512d b, simde__m512d c) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_fnmadd_pd(a, b, c); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b), + c_ = simde__m512d_to_private(c); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_fnmadd_pd(a_.m256d[i], b_.m256d[i], c_.m256d[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = -(a_.f64 * b_.f64) + c_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = -(a_.f64[i] * b_.f64[i]) + c_.f64[i]; + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_fnmadd_pd + #define _mm512_fnmadd_pd(a, b, c) simde_mm512_fnmadd_pd(a, b, c) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_FNMADD_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fnmsub.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fnmsub.h new file mode 100644 index 00000000..8d969de4 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fnmsub.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 kitegi + */ + +#if !defined(SIMDE_X86_AVX512_FNMSUB_H) +#define SIMDE_X86_AVX512_FNMSUB_H + +#include "types.h" +#include "mov.h" +#include "../fma.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_fnmsub_ps (simde__m512 a, simde__m512 b, simde__m512 c) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_fnmsub_ps(a, b, c); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b), + c_ = simde__m512_to_private(c); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_fnmsub_ps(a_.m256[i], b_.m256[i], c_.m256[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = -(a_.f32 * b_.f32) - c_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = -(a_.f32[i] * b_.f32[i]) - c_.f32[i]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_fnmsub_ps + #define _mm512_fnmsub_ps(a, b, c) simde_mm512_fnmsub_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_fnmsub_pd (simde__m512d a, simde__m512d b, simde__m512d c) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_fnmsub_pd(a, b, c); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b), + c_ = simde__m512d_to_private(c); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_fnmsub_pd(a_.m256d[i], b_.m256d[i], c_.m256d[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = -(a_.f64 * b_.f64) - c_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = -(a_.f64[i] * b_.f64[i]) - c_.f64[i]; + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_fnmsub_pd + #define _mm512_fnmsub_pd(a, b, c) simde_mm512_fnmsub_pd(a, b, c) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_FNMSUB_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fpclass.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fpclass.h new file mode 100644 index 00000000..1765570d --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/fpclass.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2023 Michael R. Crusoe + */ + +#if !defined(SIMDE_X86_AVX512_FPCLASS_H) +#define SIMDE_X86_AVX512_FPCLASS_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_fpclass_ps_mask(simde__m256 a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 0x88) { + simde__mmask8 r = 0; + simde__m256_private a_ = simde__m256_to_private(a); + + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + r |= simde_math_fpclassf(a_.f32[i], imm8) ? (UINT8_C(1) << i) : 0; + } + return r; +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) +# define simde_mm256_fpclass_ps_mask(a, imm8) _mm256_fpclass_ps_mask((a), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) +# undef _mm256_fpclass_ps_mask +# define _mm256_fpclass_ps_mask(a, imm8) simde_mm256_fpclass_ps_mask((a), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_fpclass_ph_mask(simde__m512h a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 0x88) { + simde__mmask32 r = 0; + simde__m512h_private a_ = simde__m512h_to_private(a); + + for (size_t i = 0 ; i < (sizeof(a_.f16) / sizeof(a_.f16[0])) ; i++) { + r |= simde_fpclasshf(a_.f16[i], imm8) ? (UINT8_C(1) << i) : 0; + } + return r; +} +#if defined(SIMDE_X86_AVX512FP16_NATIVE) +# define simde_mm512_fpclass_ph_mask(a, imm8) _mm512_fpclass_ph_mask((a), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) +# undef _mm512_fpclass_ph_mask +# define _mm512_fpclass_ph_mask(a, imm8) simde_mm512_fpclass_ph_mask((a), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_fpclass_pd_mask(simde__m512d a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 0x88) { + simde__mmask8 r = 0; + simde__m512d_private a_ = simde__m512d_to_private(a); + + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + r |= simde_math_fpclass(a_.f64[i], imm8) ? (UINT8_C(1) << i) : 0; + } + return r; +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) +# define simde_mm512_fpclass_pd_mask(a, imm8) _mm512_fpclass_pd_mask((a), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) +# undef _mm512_fpclass_pd_mask +# define _mm512_fpclass_pd_mask(a, imm8) simde_mm512_fpclass_pd_mask((a), (imm8)) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_FPCLASS_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/gather.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/gather.h new file mode 100644 index 00000000..8dec2ee0 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/gather.h @@ -0,0 +1,312 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2023 Michael R. Crusoe + */ + +#if !defined(SIMDE_X86_AVX512_GATHER_H) +#define SIMDE_X86_AVX512_GATHER_H + +#include "types.h" +#include "../avx2.h" +#include "extract.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_i32gather_ps(simde__m512i vindex, const void* base_addr, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m512i_private vindex_ = simde__m512i_to_private(vindex); + simde__m512_private r_ = simde__m512_to_private(simde_mm512_setzero_ps()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i32) / sizeof(vindex_.i32[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i32[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float32 dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.f32[i] = dst; + } + + return simde__m512_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(__clang__) || SIMDE_DETECT_CLANG_VERSION_CHECK(10,0,0)) + #define simde_mm512_i32gather_ps(vindex, base_addr, scale) _mm512_i32gather_ps((vindex), (base_addr), (scale)) +#elif defined(SIMDE_X86_AVX2_NATIVE) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_i32gather_ps(vindex, base_addr, scale) SIMDE_STATEMENT_EXPR_(({\ + simde__m512_private simde_mm512_i32gather_ps_r_; \ + simde__m512i_private simde_mm512_i32gather_ps_vindex_ = simde__m512i_to_private((vindex)); \ + simde_mm512_i32gather_ps_r_.m256[0] = _mm256_i32gather_ps( \ + HEDLEY_STATIC_CAST(float const*, (base_addr)), simde_mm512_i32gather_ps_vindex_.m256i[0], (scale)); \ + simde_mm512_i32gather_ps_r_.m256[1] = _mm256_i32gather_ps( \ + HEDLEY_STATIC_CAST(float const*, (base_addr)), simde_mm512_i32gather_ps_vindex_.m256i[1], (scale)); \ + simde__m512_from_private(simde_mm512_i32gather_ps_r_); \ + })) +#elif defined(SIMDE_X86_AVX2_NATIVE) && !defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_i32gather_ps(vindex, base_addr, scale) \ + simde_x_mm512_set_m256( \ + _mm256_i32gather_ps(HEDLEY_STATIC_CAST(float const*, (base_addr)), \ + simde_mm512_extracti32x8_epi32((vindex), 1), (scale)), \ + _mm256_i32gather_ps(HEDLEY_STATIC_CAST(float const*, (base_addr)), \ + simde_mm512_extracti32x8_epi32((vindex), 0), (scale)) ) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_i32gather_ps + #define _mm512_i32gather_ps(vindex, base_addr, scale) simde_mm512_i32gather_ps((vindex), (base_addr), (scale)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm512_i64gather_epi32(simde__m512i vindex, const void* base_addr, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m512i_private vindex_; + simde__m256i_private r_; + vindex_ = simde__m512i_to_private(vindex); + r_ = simde__m256i_to_private(simde_mm256_setzero_si256()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int32_t dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.i32[i] = dst; + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_i64gather_epi32(vindex, base_addr, scale) _mm512_i64gather_epi32((vindex), (base_addr), (scale)) +#elif defined(SIMDE_X86_AVX2_NATIVE) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_i64gather_epi32(vindex, base_addr, scale) SIMDE_STATEMENT_EXPR_(({\ + simde__m256i_private simde_mm512_i64gather_epi32_r_; \ + simde__m512i_private simde_mm512_i64gather_epi32_vindex_ = simde__m512i_to_private((vindex)); \ + simde_mm512_i64gather_epi32_r_.m128i[0] = _mm256_i64gather_epi32( \ + HEDLEY_STATIC_CAST(int const*, (base_addr)), simde_mm512_i64gather_epi32_vindex_.m256i[0], (scale)); \ + simde_mm512_i64gather_epi32_r_.m128i[1] = _mm256_i64gather_epi32( \ + HEDLEY_STATIC_CAST(int const*, (base_addr)), simde_mm512_i64gather_epi32_vindex_.m256i[1], (scale)); \ + simde__m256i_from_private(simde_mm512_i64gather_epi32_r_); \ + })) +#elif defined(SIMDE_X86_AVX2_NATIVE) && !defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_i64gather_epi32(vindex, base_addr, scale) \ + _mm256_insertf128_si256( \ + _mm256_castsi128_si256( \ + _mm256_i64gather_epi32(HEDLEY_STATIC_CAST(int const*, (base_addr)), \ + simde_mm512_extracti64x4_epi64((vindex), 0), (scale))), \ + _mm256_i64gather_epi32(HEDLEY_STATIC_CAST(int const*, (base_addr)), \ + simde_mm512_extracti64x4_epi64((vindex), 1), (scale)), \ + 1) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_i64gather_epi32 + #define _mm512_i64gather_epi32(vindex, base_addr, scale) simde_mm512_i64gather_epi32((vindex), (base_addr), (scale)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_i64gather_epi32(src, k, vindex, base_addr, scale) _mm512_mask_i64gather_epi32((src), (k), (vindex), (base_addr), (scale)) +#else + #define simde_mm512_mask_i64gather_epi32(src, k, vindex, base_addr, scale) simde_mm256_mask_mov_epi32(src, k, simde_mm512_i64gather_epi32((vindex), (base_addr), (scale))) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_i64gather_epi32 + #define _mm512_mask_i64gather_epi32(src, k, vindex, base_addr, scale) simde_mm512_mask_i64gather_epi32((src), (k), (vindex), (base_addr), (scale)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_i64gather_epi64(simde__m512i vindex, const void* base_addr, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m512i_private + vindex_ = simde__m512i_to_private(vindex), + r_ = simde__m512i_to_private(simde_mm512_setzero_si512()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + int64_t dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.i64[i] = dst; + } + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_i64gather_epi64(vindex, base_addr, scale) _mm512_i64gather_epi64((vindex), (base_addr), (scale)) +#elif defined(SIMDE_X86_AVX2_NATIVE) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_i64gather_epi64(vindex, base_addr, scale) SIMDE_STATEMENT_EXPR_(({\ + simde__m512i_private simde_mm512_i64gather_epi64_r_, \ + simde_mm512_i64gather_epi64_vindex_ = simde__m512i_to_private((vindex)); \ + simde_mm512_i64gather_epi64_r_.m256i[0] = _mm256_i64gather_epi64( \ + HEDLEY_STATIC_CAST(long long const*, (base_addr)), simde_mm512_i64gather_epi64_vindex_.m256i[0], (scale)); \ + simde_mm512_i64gather_epi64_r_.m256i[1] = _mm256_i64gather_epi64( \ + HEDLEY_STATIC_CAST(long long const*, (base_addr)), simde_mm512_i64gather_epi64_vindex_.m256i[1], (scale)); \ + simde__m512i_from_private(simde_mm512_i64gather_epi64_r_); \ + })) +#elif defined(SIMDE_X86_AVX2_NATIVE) && !defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_i64gather_epi64(vindex, base_addr, scale) \ + simde_x_mm512_set_m256i( \ + _mm256_i64gather_epi64(HEDLEY_STATIC_CAST(long long const*, (base_addr)), \ + simde_mm512_extracti32x8_epi32((vindex), 1), (scale)), \ + _mm256_i64gather_epi64(HEDLEY_STATIC_CAST(long long const*, (base_addr)), \ + simde_mm512_extracti32x8_epi32((vindex), 0), (scale)) ) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_i64gather_epi64 + #define _mm512_i64gather_epi64(vindex, base_addr, scale) simde_mm512_i64gather_epi64(vindex, (base_addr), (scale)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_i64gather_epi64(src, k, vindex, base_addr, scale) _mm512_mask_i64gather_epi64((src), (k), (vindex), (base_addr), (scale)) +#else + #define simde_mm512_mask_i64gather_epi64(src, k, vindex, base_addr, scale) simde_mm512_mask_mov_epi64((src), (k), simde_mm512_i64gather_epi64((vindex), (base_addr), (scale))) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_i64gather_epi64 + #define _mm512_mask_i64gather_epi64(src, k, vindex, base_addr, scale) simde_mm512_mask_i64gather_epi64((src), (k), (vindex), (base_addr), (scale)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_i64gather_pd(simde__m512i vindex, const void* base_addr, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m512i_private vindex_; + simde__m512d_private r_; + vindex_ = simde__m512i_to_private(vindex); + r_ = simde__m512d_to_private(simde_mm512_setzero_pd()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float64 dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.f64[i] = dst; + } + + return simde__m512d_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_i64gather_pd(vindex, base_addr, scale) _mm512_i64gather_pd((vindex), (base_addr), (scale)) +#elif defined(SIMDE_X86_AVX2_NATIVE) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_i64gather_pd(vindex, base_addr, scale) SIMDE_STATEMENT_EXPR_(({\ + simde__m512d_private simde_mm512_i64gather_pd_r_; \ + simde__m512i_private simde_mm512_i64gather_pd_vindex_ = simde__m512i_to_private((vindex)); \ + simde_mm512_i64gather_pd_r_.m256d[0] = _mm256_i64gather_pd( \ + HEDLEY_STATIC_CAST(double const*, (base_addr)), simde_mm512_i64gather_pd_vindex_.m256i[0], (scale)); \ + simde_mm512_i64gather_pd_r_.m256d[1] = _mm256_i64gather_pd( \ + HEDLEY_STATIC_CAST(double const*, (base_addr)), simde_mm512_i64gather_pd_vindex_.m256i[1], (scale)); \ + simde__m512d_from_private(simde_mm512_i64gather_pd_r_); \ + })) +#elif defined(SIMDE_X86_AVX2_NATIVE) && !defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_i64gather_pd(vindex, base_addr, scale) \ + simde_x_mm512_set_m256d( \ + _mm256_i64gather_pd(HEDLEY_STATIC_CAST(double const*, (base_addr)), \ + simde_mm512_extracti64x4_epi64((vindex), 1), (scale)), \ + _mm256_i64gather_pd(HEDLEY_STATIC_CAST(double const*, (base_addr)), \ + simde_mm512_extracti64x4_epi64((vindex), 0), (scale)) ) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_i64gather_pd + #define _mm512_i64gather_pd(vindex, base_addr, scale) simde_mm512_i64gather_pd((vindex), (base_addr), (scale)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_i64gather_pd(src, k, vindex, base_addr, scale) _mm512_mask_i64gather_pd((src), (k), (vindex), (base_addr), (scale)) +#else + #define simde_mm512_mask_i64gather_pd(src, k, vindex, base_addr, scale) simde_mm512_mask_mov_pd((src), (k), simde_mm512_i64gather_pd((vindex), (base_addr), (scale))) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_i64gather_pd + #define _mm512_mask_i64gather_pd(src, k, vindex, base_addr, scale) simde_mm512_mask_i64gather_pd((src), (k), (vindex), (base_addr), (scale)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm512_i64gather_ps(simde__m512i vindex, const void* base_addr, const int32_t scale) + SIMDE_REQUIRE_CONSTANT(scale) + HEDLEY_REQUIRE_MSG((scale && scale <= 8 && !(scale & (scale - 1))), "`scale' must be a power of two less than or equal to 8") { + simde__m512i_private vindex_; + simde__m256_private r_; + vindex_ = simde__m512i_to_private(vindex); + r_ = simde__m256_to_private(simde_mm256_setzero_ps()); + const uint8_t* addr = HEDLEY_REINTERPRET_CAST(const uint8_t*, base_addr); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(vindex_.i64) / sizeof(vindex_.i64[0])) ; i++) { + const uint8_t* src = addr + (HEDLEY_STATIC_CAST(size_t , vindex_.i64[i]) * HEDLEY_STATIC_CAST(size_t , scale)); + simde_float32 dst; + simde_memcpy(&dst, src, sizeof(dst)); + r_.f32[i] = dst; + } + + return simde__m256_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_i64gather_ps(vindex, base_addr, scale) _mm512_i64gather_ps((vindex), (base_addr), (scale)) +#elif defined(SIMDE_X86_AVX2_NATIVE) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_i64gather_ps(vindex, base_addr, scale) SIMDE_STATEMENT_EXPR_(({\ + simde__m256_private simde_mm512_i64gather_ps_r_; \ + simde__m512i_private simde_mm512_i64gather_ps_vindex_ = simde__m512i_to_private((vindex)); \ + simde_mm512_i64gather_ps_r_.m128[0] = _mm256_i64gather_ps( \ + HEDLEY_STATIC_CAST(float const*, (base_addr)), simde_mm512_i64gather_ps_vindex_.m256i[0], (scale)); \ + simde_mm512_i64gather_ps_r_.m128[1] = _mm256_i64gather_ps( \ + HEDLEY_STATIC_CAST(float const*, (base_addr)), simde_mm512_i64gather_ps_vindex_.m256i[1], (scale)); \ + simde__m256_from_private(simde_mm512_i64gather_ps_r_); \ + })) +#elif defined(SIMDE_X86_AVX2_NATIVE) && !defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_i64gather_ps(vindex, base_addr, scale) \ + _mm256_insertf128_ps( \ + _mm256_castps128_ps256( \ + _mm256_i64gather_ps(HEDLEY_STATIC_CAST(float const*, (base_addr)), \ + simde_mm512_extracti64x4_epi64((vindex), 0), (scale))), \ + _mm256_i64gather_ps(HEDLEY_STATIC_CAST(float const*, (base_addr)), \ + simde_mm512_extracti64x4_epi64((vindex), 1), (scale)), \ + 1) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_i64gather_ps + #define _mm512_i64gather_ps(vindex, base_addr, scale) simde_mm512_i64gather_ps((vindex), (base_addr), (scale)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_i64gather_ps(src, k, vindex, base_addr, scale) _mm512_mask_i64gather_ps((src), (k), (vindex), (base_addr), (scale)) +#else + #define simde_mm512_mask_i64gather_ps(src, k, vindex, base_addr, scale) simde_mm256_mask_mov_ps((src), (k), simde_mm512_i64gather_ps((vindex), (base_addr), (scale))) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_i64gather_ps + #define _mm512_mask_i64gather_ps(src, k, vindex, base_addr, scale) simde_mm512_mask_i64gather_ps((src), (k), (vindex), (base_addr), (scale)) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_GATHER_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/insert.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/insert.h new file mode 100644 index 00000000..67120d31 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/insert.h @@ -0,0 +1,490 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Christopher Moore + */ + +#if !defined(SIMDE_X86_AVX512_INSERT_H) +#define SIMDE_X86_AVX512_INSERT_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_insertf32x4 (simde__m512 a, simde__m128 b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + simde__m512 r; + switch(imm8) { + case 0: r = _mm512_insertf32x4(a, b, 0); break; + case 1: r = _mm512_insertf32x4(a, b, 1); break; + case 2: r = _mm512_insertf32x4(a, b, 2); break; + case 3: r = _mm512_insertf32x4(a, b, 3); break; + default: HEDLEY_UNREACHABLE(); r = simde_mm512_setzero_ps(); break; + } + return r; + #else + simde__m512_private a_ = simde__m512_to_private(a); + + a_.m128[imm8 & 3] = b; + + return simde__m512_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_insertf32x4 + #define _mm512_insertf32x4(a, b, imm8) simde_mm512_insertf32x4(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_insertf32x4 (simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m128 b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m512 r; + + #if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(8,0,0)) + SIMDE_CONSTIFY_4_(_mm512_mask_insertf32x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_ps ()), imm8, src, k, a, b); + return r; + #else + SIMDE_CONSTIFY_4_(simde_mm512_insertf32x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_ps ()), imm8, a, b); + return simde_mm512_mask_mov_ps(src, k, r); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_insertf32x4 + #define _mm512_mask_insertf32x4(src, k, a, b, imm8) simde_mm512_mask_insertf32x4(src, k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_insertf32x4 (simde__mmask16 k, simde__m512 a, simde__m128 b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m512 r; + + #if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(8,0,0)) + SIMDE_CONSTIFY_4_(_mm512_maskz_insertf32x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_ps ()), imm8, k, a, b); + return r; + #else + SIMDE_CONSTIFY_4_(simde_mm512_insertf32x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_ps ()), imm8, a, b); + return simde_mm512_maskz_mov_ps(k, r); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_insertf32x4 + #define _mm512_maskz_insertf32x4(k, a, b, imm8) simde_mm512_maskz_insertf32x4(k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_insertf64x4 (simde__m512d a, simde__m256d b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m512d_private a_ = simde__m512d_to_private(a); + + a_.m256d[imm8 & 1] = b; + + return simde__m512d_from_private(a_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_insertf64x4(a, b, imm8) _mm512_insertf64x4(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_insertf64x4 + #define _mm512_insertf64x4(a, b, imm8) simde_mm512_insertf64x4(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_insertf64x4 (simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m256d b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m512d r; + + #if defined(SIMDE_X86_AVX512F_NATIVE) + SIMDE_CONSTIFY_2_(_mm512_mask_insertf64x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_pd ()), imm8, src, k, a, b); + return r; + #else + SIMDE_CONSTIFY_2_(simde_mm512_insertf64x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_pd ()), imm8, a, b); + return simde_mm512_mask_mov_pd(src, k, r); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_insertf64x4 + #define _mm512_mask_insertf64x4(src, k, a, b, imm8) simde_mm512_mask_insertf64x4(src, k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_insertf64x4 (simde__mmask8 k, simde__m512d a, simde__m256d b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m512d r; + + #if defined(SIMDE_X86_AVX512F_NATIVE) + SIMDE_CONSTIFY_2_(_mm512_maskz_insertf64x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_pd ()), imm8, k, a, b); + return r; + #else + SIMDE_CONSTIFY_2_(simde_mm512_insertf64x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_pd ()), imm8, a, b); + return simde_mm512_maskz_mov_pd(k, r); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_insertf64x4 + #define _mm512_maskz_insertf64x4(k, a, b, imm8) simde_mm512_maskz_insertf64x4(k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_inserti32x4 (simde__m512i a, simde__m128i b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m512i_private a_ = simde__m512i_to_private(a); + + a_.m128i[imm8 & 3] = b; + + return simde__m512i_from_private(a_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_inserti32x4(a, b, imm8) _mm512_inserti32x4(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_inserti32x4 + #define _mm512_inserti32x4(a, b, imm8) simde_mm512_inserti32x4(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_inserti32x4 (simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m128i b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m512i r; + + #if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(8,0,0)) + SIMDE_CONSTIFY_4_(_mm512_mask_inserti32x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_si512 ()), imm8, src, k, a, b); + return r; + #else + SIMDE_CONSTIFY_4_(simde_mm512_inserti32x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_si512 ()), imm8, a, b); + return simde_mm512_mask_mov_epi32(src, k, r); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_inserti32x4 + #define _mm512_mask_inserti32x4(src, k, a, b, imm8) simde_mm512_mask_inserti32x4(src, k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_inserti32x4 (simde__mmask16 k, simde__m512i a, simde__m128i b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m512i r; + + #if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(8,0,0)) + SIMDE_CONSTIFY_4_(_mm512_maskz_inserti32x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_si512 ()), imm8, k, a, b); + return r; + #else + SIMDE_CONSTIFY_4_(simde_mm512_inserti32x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_si512 ()), imm8, a, b); + return simde_mm512_maskz_mov_epi32(k, r); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_inserti32x4 + #define _mm512_maskz_inserti32x4(k, a, b, imm8) simde_mm512_maskz_inserti32x4(k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_inserti64x4 (simde__m512i a, simde__m256i b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m512i_private a_ = simde__m512i_to_private(a); + + a_.m256i[imm8 & 1] = b; + + return simde__m512i_from_private(a_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_inserti64x4(a, b, imm8) _mm512_inserti64x4(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_inserti64x4 + #define _mm512_inserti64x4(a, b, imm8) simde_mm512_inserti64x4(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_inserti64x4 (simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m256i b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 2) { + simde__m512i r; + + #if defined(SIMDE_X86_AVX512F_NATIVE) + SIMDE_CONSTIFY_2_(_mm512_mask_inserti64x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_si512 ()), imm8, src, k, a, b); + return r; + #else + SIMDE_CONSTIFY_2_(simde_mm512_inserti64x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_si512 ()), imm8, a, b); + return simde_mm512_mask_mov_epi64(src, k, r); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_inserti64x4 + #define _mm512_mask_inserti64x4(src, k, a, b, imm8) simde_mm512_mask_inserti64x4(src, k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_inserti64x4 (simde__mmask8 k, simde__m512i a, simde__m256i b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 2) { + simde__m512i r; + + #if defined(SIMDE_X86_AVX512F_NATIVE) + SIMDE_CONSTIFY_2_(_mm512_maskz_inserti64x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_si512 ()), imm8, k, a, b); + return r; + #else + SIMDE_CONSTIFY_2_(simde_mm512_inserti64x4, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_si512 ()), imm8, a, b); + return simde_mm512_maskz_mov_epi64(k, r); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_inserti64x4 + #define _mm512_maskz_inserti64x4(k, a, b, imm8) simde_mm512_maskz_inserti64x4(k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_insertf32x8 (simde__m512 a, simde__m256 b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m512_private a_ = simde__m512_to_private(a); + + a_.m256[imm8 & 1] = b; + + return simde__m512_from_private(a_); +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_insertf32x8(a, b, imm8) _mm512_insertf32x8(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_insertf32x8 + #define _mm512_insertf32x8(a, b, imm8) simde_mm512_insertf32x8(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_insertf32x8(simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m256 b, const int imm8) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + simde__m512 r; + SIMDE_CONSTIFY_2_(_mm512_mask_insertf32x8, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_ps ()), imm8, src, k, a, b); + return r; + #else + simde__m512 r; + SIMDE_CONSTIFY_2_(simde_mm512_insertf32x8, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_ps ()), imm8, a, b); + return simde_mm512_mask_mov_ps(src, k, r); + #endif + } +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_insertf32x8 + #define _mm512_mask_insertf32x8(src, k, a, b, imm8) simde_mm512_mask_insertf32x8(src, k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_insertf32x8(simde__mmask16 k, simde__m512 a, simde__m256 b, const int imm8) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + simde__m512 r; + SIMDE_CONSTIFY_2_(_mm512_maskz_insertf32x8, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_ps ()), imm8, k, a, b); + return r; + #else + simde__m512 r; + SIMDE_CONSTIFY_2_(simde_mm512_insertf32x8, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_ps ()), imm8, a, b); + return simde_mm512_maskz_mov_ps(k, r); + #endif + } +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_insertf32x8 + #define _mm512_maskz_insertf32x8(k, a, b, imm8) simde_mm512_maskz_insertf32x8(k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_insertf64x2 (simde__m512d a, simde__m128d b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m512d_private a_ = simde__m512d_to_private(a); + + a_.m128d[imm8 & 3] = b; + + return simde__m512d_from_private(a_); +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_insertf64x2(a, b, imm8) _mm512_insertf64x2(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_insertf64x2 + #define _mm512_insertf64x2(a, b, imm8) simde_mm512_insertf64x2(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_insertf64x2(simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m128d b, const int imm8) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + simde__m512d r; + SIMDE_CONSTIFY_4_(_mm512_mask_insertf64x2, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_pd ()), imm8, src, k, a, b); + return r; + #else + simde__m512d r; + SIMDE_CONSTIFY_4_(simde_mm512_insertf64x2, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_pd ()), imm8, a, b); + return simde_mm512_mask_mov_pd(src, k, r); + #endif + } +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_insertf64x2 + #define _mm512_mask_insertf64x2(src, k, a, b, imm8) simde_mm512_mask_insertf64x2(src, k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_insertf64x2(simde__mmask8 k, simde__m512d a, simde__m128d b, const int imm8) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + simde__m512d r; + SIMDE_CONSTIFY_4_(_mm512_maskz_insertf64x2, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_pd ()), imm8, k, a, b); + return r; + #else + simde__m512d r; + SIMDE_CONSTIFY_4_(simde_mm512_insertf64x2, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_pd ()), imm8, a, b); + return simde_mm512_maskz_mov_pd(k, r); + #endif + } +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_insertf64x2 + #define _mm512_maskz_insertf64x2(k, a, b, imm8) simde_mm512_maskz_insertf64x2(k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_inserti32x8 (simde__m512i a, simde__m256i b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m512i_private a_ = simde__m512i_to_private(a); + + a_.m256i[imm8 & 1] = b; + + return simde__m512i_from_private(a_); +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_inserti32x8(a, b, imm8) _mm512_inserti32x8(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_inserti32x8 + #define _mm512_inserti32x8(a, b, imm8) simde_mm512_inserti32x8(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_inserti32x8(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m256i b, const int imm8) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + simde__m512i r; + SIMDE_CONSTIFY_2_(_mm512_mask_inserti32x8, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_epi32 ()), imm8, src, k, a, b); + return r; + #else + simde__m512i r; + SIMDE_CONSTIFY_2_(simde_mm512_inserti32x8, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_epi32 ()), imm8, a, b); + return simde_mm512_mask_mov_epi32(src, k, r); + #endif + } +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_inserti32x8 + #define _mm512_mask_inserti32x8(src, k, a, b, imm8) simde_mm512_mask_inserti32x8(src, k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_inserti32x8(simde__mmask16 k, simde__m512i a, simde__m256i b, const int imm8) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + simde__m512i r; + SIMDE_CONSTIFY_2_(_mm512_maskz_inserti32x8, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_epi32 ()), imm8, k, a, b); + return r; + #else + simde__m512i r; + SIMDE_CONSTIFY_2_(simde_mm512_inserti32x8, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_epi32 ()), imm8, a, b); + return simde_mm512_maskz_mov_epi32(k, r); + #endif + } +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_inserti32x8 + #define _mm512_maskz_inserti32x8(k, a, b, imm8) simde_mm512_maskz_inserti32x8(k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_inserti64x2 (simde__m512i a, simde__m128i b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m512i_private a_ = simde__m512i_to_private(a); + + a_.m128i[imm8 & 3] = b; + + return simde__m512i_from_private(a_); +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_inserti64x2(a, b, imm8) _mm512_inserti64x2(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_inserti64x2 + #define _mm512_inserti64x2(a, b, imm8) simde_mm512_inserti64x2(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_inserti64x2(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m128i b, const int imm8) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + simde__m512i r; + SIMDE_CONSTIFY_4_(_mm512_mask_inserti64x2, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_si512 ()), imm8, src, k, a, b); + return r; + #else + simde__m512i r; + SIMDE_CONSTIFY_4_(simde_mm512_inserti64x2, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_si512 ()), imm8, a, b); + return simde_mm512_mask_mov_epi64(src, k, r); + #endif + } +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_inserti64x2 + #define _mm512_mask_inserti64x2(src, k, a, b, imm8) simde_mm512_mask_inserti64x2(src, k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_inserti64x2(simde__mmask8 k, simde__m512i a, simde__m128i b, const int imm8) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + simde__m512i r; + SIMDE_CONSTIFY_4_(_mm512_maskz_inserti64x2, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_si512 ()), imm8, k, a, b); + return r; + #else + simde__m512i r; + SIMDE_CONSTIFY_4_(simde_mm512_inserti64x2, r, (HEDLEY_UNREACHABLE(), simde_mm512_setzero_si512 ()), imm8, a, b); + return simde_mm512_maskz_mov_epi64(k, r); + #endif + } +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_inserti64x2 + #define _mm512_maskz_inserti64x2(k, a, b, imm8) simde_mm512_maskz_inserti64x2(k, a, b, imm8) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_INSERT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/kand.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/kand.h new file mode 100644 index 00000000..78641000 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/kand.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2023 Michael R. Crusoe + */ + +#if !defined(SIMDE_X86_AVX512_KAND_H) +#define SIMDE_X86_AVX512_KAND_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_kand (simde__mmask16 a, simde__mmask16 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_kand(a, b); + #else + return a & b; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_kand + #define _mm512_kand(a, b) simde_mm512_kand((a), (b)) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_KAND_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/knot.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/knot.h new file mode 100644 index 00000000..3b4696e8 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/knot.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2023 Michael R. Crusoe + */ + +#if !defined(SIMDE_X86_AVX512_KNOT_H) +#define SIMDE_X86_AVX512_KNOT_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_knot_mask8 (simde__mmask8 a) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) \ + && (!defined(__clang__) || SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + return _knot_mask8(a); + #else + return ~a; + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _knot_mask8 + #define _knot_mask8(a) simde_knot_mask8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_knot_mask16 (simde__mmask16 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) \ + && (!defined(__clang__) || SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + return _knot_mask16(a); + #else + return ~a; + #endif +} +#define simde_mm512_knot(a) simde_knot_mask16(a) +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _knot_mask16 + #undef _mm512_knot + #define _knot_mask16(a) simde_knot_mask16(a) + #define _mm512_knot(a) simde_knot_mask16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_knot_mask32 (simde__mmask32 a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) \ + && (!defined(__clang__) || SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + return _knot_mask32(a); + #else + return ~a; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _knot_mask32 + #define _knot_mask32(a) simde_knot_mask32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_knot_mask64 (simde__mmask64 a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) \ + && (!defined(__clang__) || SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + return _knot_mask64(a); + #else + return ~a; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _knot_mask64 + #define _knot_mask64(a) simde_knot_mask64(a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_KNOT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/kshift.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/kshift.h new file mode 100644 index 00000000..4dfe855d --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/kshift.h @@ -0,0 +1,152 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Christopher Moore + */ + +#if !defined(SIMDE_X86_AVX512_KSHIFT_H) +#define SIMDE_X86_AVX512_KSHIFT_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_kshiftli_mask16 (simde__mmask16 a, unsigned int count) + SIMDE_REQUIRE_CONSTANT_RANGE(count, 0, 255) { + return HEDLEY_STATIC_CAST(simde__mmask16, (count <= 15) ? (a << count) : 0); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && (!defined(SIMDE_DETECT_CLANG_VERSION) && SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) + #define simde_kshiftli_mask16(a, count) _kshiftli_mask16(a, count) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _kshiftli_mask16 + #define _kshiftli_mask16(a, count) simde_kshiftli_mask16(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_kshiftli_mask32 (simde__mmask32 a, unsigned int count) + SIMDE_REQUIRE_CONSTANT_RANGE(count, 0, 255) { + return (count <= 31) ? (a << count) : 0; +} +#if defined(SIMDE_X86_AVX512BW_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && (!defined(SIMDE_DETECT_CLANG_VERSION) && SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) + #define simde_kshiftli_mask32(a, count) _kshiftli_mask32(a, count) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _kshiftli_mask32 + #define _kshiftli_mask32(a, count) simde_kshiftli_mask32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_kshiftli_mask64 (simde__mmask64 a, unsigned int count) + SIMDE_REQUIRE_CONSTANT_RANGE(count, 0, 255) { + return (count <= 63) ? (a << count) : 0; +} +#if defined(SIMDE_X86_AVX512BW_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && (!defined(SIMDE_DETECT_CLANG_VERSION) && SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) + #define simde_kshiftli_mask64(a, count) _kshiftli_mask64(a, count) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _kshiftli_mask64 + #define _kshiftli_mask64(a, count) simde_kshiftli_mask64(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_kshiftli_mask8 (simde__mmask8 a, unsigned int count) + SIMDE_REQUIRE_CONSTANT_RANGE(count, 0, 255) { + return HEDLEY_STATIC_CAST(simde__mmask8, (count <= 7) ? (a << count) : 0); +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && (!defined(SIMDE_DETECT_CLANG_VERSION) && SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) + #define simde_kshiftli_mask8(a, count) _kshiftli_mask8(a, count) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _kshiftli_mask8 + #define _kshiftli_mask8(a, count) simde_kshiftli_mask8(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_kshiftri_mask16 (simde__mmask16 a, unsigned int count) + SIMDE_REQUIRE_CONSTANT_RANGE(count, 0, 255) { + return HEDLEY_STATIC_CAST(simde__mmask16, (count <= 15) ? (a >> count) : 0); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && (!defined(SIMDE_DETECT_CLANG_VERSION) && SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) + #define simde_kshiftri_mask16(a, count) _kshiftri_mask16(a, count) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _kshiftri_mask16 + #define _kshiftri_mask16(a, count) simde_kshiftri_mask16(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_kshiftri_mask32 (simde__mmask32 a, unsigned int count) + SIMDE_REQUIRE_CONSTANT_RANGE(count, 0, 255) { + return (count <= 31) ? (a >> count) : 0; +} +#if defined(SIMDE_X86_AVX512BW_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && (!defined(SIMDE_DETECT_CLANG_VERSION) && SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) + #define simde_kshiftri_mask32(a, count) _kshiftri_mask32(a, count) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _kshiftri_mask32 + #define _kshiftri_mask32(a, count) simde_kshiftri_mask32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_kshiftri_mask64 (simde__mmask64 a, unsigned int count) + SIMDE_REQUIRE_CONSTANT_RANGE(count, 0, 255) { + return (count <= 63) ? (a >> count) : 0; +} +#if defined(SIMDE_X86_AVX512BW_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && (!defined(SIMDE_DETECT_CLANG_VERSION) && SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) + #define simde_kshiftri_mask64(a, count) _kshiftri_mask64(a, count) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _kshiftri_mask64 + #define _kshiftri_mask64(a, count) simde_kshiftri_mask64(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_kshiftri_mask8 (simde__mmask8 a, unsigned int count) + SIMDE_REQUIRE_CONSTANT_RANGE(count, 0, 255) { + return HEDLEY_STATIC_CAST(simde__mmask8, (count <= 7) ? (a >> count) : 0); +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(7,0,0)) && (!defined(SIMDE_DETECT_CLANG_VERSION) && SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) + #define simde_kshiftri_mask8(a, count) _kshiftri_mask8(a, count) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _kshiftri_mask8 + #define _kshiftri_mask8(a, count) simde_kshiftri_mask8(a, count) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_KSHIFT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/kxor.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/kxor.h new file mode 100644 index 00000000..45f5d04d --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/kxor.h @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2023 Michael R. Crusoe + */ + +#if !defined(SIMDE_X86_AVX512_KXOR_H) +#define SIMDE_X86_AVX512_KXOR_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_kxor_mask8 (simde__mmask8 a, simde__mmask8 b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) \ + && (!defined(__clang__) || SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + return _kxor_mask8(a, b); + #else + return a^b; + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _kxor_mask8 + #define _kxor_mask8(a, b) simde_kxor_mask8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_kxor_mask16 (simde__mmask16 a, simde__mmask16 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) \ + && (!defined(__clang__) || SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + return _kxor_mask16(a, b); + #else + return a^b; + #endif +} +#define simde_mm512_kxor(a, b) simde_kxor_mask16(a, b) +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _kxor_mask16 + #undef _mm512_kxor + #define _kxor_mask16(a, b) simde_kxor_mask16(a, b) + #define _mm512_kxor(a, b) simde_kxor_mask16(a, b) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_kxor_mask32 (simde__mmask32 a, simde__mmask32 b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) \ + && (!defined(__clang__) || SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + return _kxor_mask32(a, b); + #else + return a^b; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _kxor_mask32 + #define _kxor_mask32(a, b) simde_kxor_mask32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_kxor_mask64 (simde__mmask64 a, simde__mmask64 b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) \ + && (!defined(__clang__) || SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + return _kxor_mask64(a, b); + #else + return a^b; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _kxor_mask64 + #define _kxor_mask64(a, b) simde_kxor_mask64(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_KXOR_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/load.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/load.h new file mode 100644 index 00000000..6a4af937 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/load.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_LOAD_H) +#define SIMDE_X86_AVX512_LOAD_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_load_pd (void const * mem_addr) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_load_pd(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m512d)); + #else + simde__m512d r; + simde_memcpy(&r, SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m512d), sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_load_pd + #define _mm512_load_pd(a) simde_mm512_load_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_load_ps (void const * mem_addr) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_load_ps(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m512)); + #else + simde__m512 r; + simde_memcpy(&r, SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m512), sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_load_ps + #define _mm512_load_ps(a) simde_mm512_load_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512h +simde_mm512_load_ph (void const * mem_addr) { + #if defined(SIMDE_X86_AVX512FP16_NATIVE) + return _mm512_load_ph(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m512h)); + #else + simde__m512h r; + simde_memcpy(&r, SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m512h), sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) + #undef _mm512_load_ph + #define _mm512_load_ph(a) simde_mm512_load_ph(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_load_si512 (void const * mem_addr) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_load_si512(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m512i)); + #else + simde__m512i r; + simde_memcpy(&r, SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m512i), sizeof(r)); + return r; + #endif +} +#define simde_mm512_load_epi8(mem_addr) simde_mm512_load_si512(mem_addr) +#define simde_mm512_load_epi16(mem_addr) simde_mm512_load_si512(mem_addr) +#define simde_mm512_load_epi32(mem_addr) simde_mm512_load_si512(mem_addr) +#define simde_mm512_load_epi64(mem_addr) simde_mm512_load_si512(mem_addr) +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_load_epi8 + #undef _mm512_load_epi16 + #undef _mm512_load_epi32 + #undef _mm512_load_epi64 + #undef _mm512_load_si512 + #define _mm512_load_si512(a) simde_mm512_load_si512(a) + #define _mm512_load_epi8(a) simde_mm512_load_si512(a) + #define _mm512_load_epi16(a) simde_mm512_load_si512(a) + #define _mm512_load_epi32(a) simde_mm512_load_si512(a) + #define _mm512_load_epi64(a) simde_mm512_load_si512(a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_LOAD_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/loadu.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/loadu.h new file mode 100644 index 00000000..4a31966b --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/loadu.h @@ -0,0 +1,297 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_LOADU_H) +#define SIMDE_X86_AVX512_LOADU_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_loadu_ps (void const * mem_addr) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + #if defined(SIMDE_BUG_CLANG_REV_298042) + return _mm512_loadu_ps(SIMDE_ALIGN_CAST(const float *, mem_addr)); + #else + return _mm512_loadu_ps(mem_addr); + #endif + #else + simde__m512 r; + simde_memcpy(&r, mem_addr, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_loadu_ps + #define _mm512_loadu_ps(a) simde_mm512_loadu_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_loadu_pd (void const * mem_addr) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + #if defined(SIMDE_BUG_CLANG_REV_298042) + return _mm512_loadu_pd(SIMDE_ALIGN_CAST(const double *, mem_addr)); + #else + return _mm512_loadu_pd(mem_addr); + #endif + #else + simde__m512d r; + simde_memcpy(&r, mem_addr, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_loadu_pd + #define _mm512_loadu_pd(a) simde_mm512_loadu_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512h +simde_mm512_loadu_ph (void const * mem_addr) { + #if defined(SIMDE_X86_AVX512FP16_NATIVE) + return _mm512_loadu_ph(mem_addr); + #else + simde__m512h r; + simde_memcpy(&r, mem_addr, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) + #undef _mm512_loadu_ph + #define _mm512_loadu_ph(a) simde_mm512_loadu_ph(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_loadu_si512 (void const * mem_addr) { + simde__m512i r; + #if HEDLEY_GNUC_HAS_ATTRIBUTE(may_alias,3,3,0) + HEDLEY_DIAGNOSTIC_PUSH + SIMDE_DIAGNOSTIC_DISABLE_PACKED_ + struct simde_mm512_loadu_si512_s { + __typeof__(r) v; + } __attribute__((__packed__, __may_alias__)); + r = HEDLEY_REINTERPRET_CAST(const struct simde_mm512_loadu_si512_s *, mem_addr)->v; + HEDLEY_DIAGNOSTIC_POP + #else + simde_memcpy(&r, mem_addr, sizeof(r)); + #endif + + return r; +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(10,0,0)) && (!defined(__clang__) || SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) + #define simde_mm512_loadu_si512(mem_addr) _mm512_loadu_si512(mem_addr) + #define simde_mm512_loadu_epi32(mem_addr) _mm512_loadu_epi32(mem_addr) + #define simde_mm512_loadu_epi64(mem_addr) _mm512_loadu_epi64(mem_addr) +#else + #define simde_mm512_loadu_epi32(mem_addr) simde_mm512_loadu_si512(mem_addr) + #define simde_mm512_loadu_epi64(mem_addr) simde_mm512_loadu_si512(mem_addr) +#endif +#if defined(SIMDE_X86_AVX512BW_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(11,0,0)) && (!defined(__clang__) || SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0)) + #define simde_mm512_loadu_epi8(mem_addr) _mm512_loadu_epi8(mem_addr) + #define simde_mm512_loadu_epi16(mem_addr) _mm512_loadu_epi16(mem_addr) +#else + #define simde_mm512_loadu_epi8(mem_addr) simde_mm512_loadu_si512(mem_addr) + #define simde_mm512_loadu_epi16(mem_addr) simde_mm512_loadu_si512(mem_addr) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_loadu_epi8 + #undef _mm512_loadu_epi16 + #define _mm512_loadu_epi8(a) simde_mm512_loadu_epi8(a) + #define _mm512_loadu_epi16(a) simde_mm512_loadu_epi16(a) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_loadu_epi32 + #undef _mm512_loadu_epi64 + #undef _mm512_loadu_si512 + #define _mm512_loadu_si512(a) simde_mm512_loadu_si512(a) + #define _mm512_loadu_epi32(a) simde_mm512_loadu_epi32(a) + #define _mm512_loadu_epi64(a) simde_mm512_loadu_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_loadu_epi16 (simde__mmask16 k, void const * mem_addr) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_loadu_epi16(k, HEDLEY_REINTERPRET_CAST(void const*, mem_addr)); + #else + return simde_mm256_maskz_mov_epi16(k, simde_mm256_loadu_epi16(mem_addr)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_loadu_epi16 + #define _mm256_maskz_loadu_epi16(k, mem_addr) simde_mm256_maskz_loadu_epi16(k, mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_maskz_loadu_ps (simde__mmask8 k, void const * mem_addr) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_loadu_ps(k, HEDLEY_REINTERPRET_CAST(void const*, mem_addr)); + #else + return simde_mm256_maskz_mov_ps(k, simde_mm256_loadu_ps(HEDLEY_REINTERPRET_CAST(const float*, mem_addr))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_loadu_ps + #define _mm256_maskz_loadu_ps(k, mem_addr) simde_mm256_maskz_loadu_ps(k, mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_loadu_epi16 (simde__m512i src, simde__mmask32 k, void const * mem_addr) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_loadu_epi16(src, k, HEDLEY_REINTERPRET_CAST(void const*, mem_addr)); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_loadu_epi16(mem_addr)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_loadu_epi16 + #define _mm512_mask_loadu_epi16(src, k, mem_addr) simde_mm512_mask_loadu_epi16(src, k, mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_loadu_epi16 (simde__mmask32 k, void const * mem_addr) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_loadu_epi16(k, HEDLEY_REINTERPRET_CAST(void const*, mem_addr)); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_loadu_epi16(mem_addr)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_loadu_epi16 + #define _mm512_maskz_loadu_epi16(k, mem_addr) simde_mm512_maskz_loadu_epi16(k, mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_loadu_epi32 (simde__m512i src, simde__mmask16 k, void const * mem_addr) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_loadu_epi32(src, k, HEDLEY_REINTERPRET_CAST(void const*, mem_addr)); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_loadu_epi32(mem_addr)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_loadu_epi32 + #define _mm512_mask_loadu_epi32(src, k, mem_addr) simde_mm512_mask_loadu_epi32(src, k, mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_loadu_epi64 (simde__m512i src, simde__mmask8 k, void const * mem_addr) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_loadu_epi64(src, k, HEDLEY_REINTERPRET_CAST(void const*, mem_addr)); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_loadu_epi64(mem_addr)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_loadu_epi64 + #define _mm512_mask_loadu_epi64(src, k, mem_addr) simde_mm512_mask_loadu_epi64(src, k, mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_loadu_epi64 (simde__mmask8 k, void const * mem_addr) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_loadu_epi64(k, HEDLEY_REINTERPRET_CAST(void const*, mem_addr)); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_loadu_epi64(mem_addr)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_loadu_epi64 + #define _mm512_maskz_loadu_epi64(k, mem_addr) simde_mm512_maskz_loadu_epi64((k), (mem_addr)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_loadu_pd (simde__m512d src, simde__mmask8 k, void const * mem_addr) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_loadu_pd(src, k, HEDLEY_REINTERPRET_CAST(void const*, mem_addr)); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_loadu_pd(mem_addr)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_loadu_pd + #define _mm512_mask_loadu_pd(src, k, mem_addr) simde_mm512_mask_loadu_pd(src, k, mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_loadu_ps (simde__m512 src, simde__mmask16 k, void const * mem_addr) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_loadu_ps(src, k, HEDLEY_REINTERPRET_CAST(void const*, mem_addr)); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_loadu_ps(mem_addr)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_loadu_ps + #define _mm512_mask_loadu_ps(src, k, mem_addr) simde_mm512_mask_loadu_ps(src, k, mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_loadu_ps (simde__mmask16 k, void const * mem_addr) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_loadu_ps(k, HEDLEY_REINTERPRET_CAST(void const*, mem_addr)); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_loadu_ps(mem_addr)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_loadu_ps + #define _mm512_maskz_loadu_ps(k, mem_addr) simde_mm512_maskz_loadu_ps(k, mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_loadu_pd (simde__mmask8 k, void const * mem_addr) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_loadu_pd(k, HEDLEY_REINTERPRET_CAST(void const*, mem_addr)); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_loadu_pd(mem_addr)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_loadu_pd + #define _mm512_maskz_loadu_pd(k, mem_addr) simde_mm512_maskz_loadu_pd(k, mem_addr) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_LOADU_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/lzcnt.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/lzcnt.h new file mode 100644 index 00000000..41a0eecb --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/lzcnt.h @@ -0,0 +1,220 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_LZCNT_H) +#define SIMDE_X86_AVX512_LZCNT_H + +#include "types.h" +#include "mov.h" +#if HEDLEY_MSVC_VERSION_CHECK(14,0,0) +#include +#pragma intrinsic(_BitScanReverse) + #if defined(_M_AMD64) || defined(_M_ARM64) + #pragma intrinsic(_BitScanReverse64) + #endif +#endif + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#if \ + ( HEDLEY_HAS_BUILTIN(__builtin_clz) || \ + HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + HEDLEY_ARM_VERSION_CHECK(4,1,0) ) && \ + defined(__INT_MAX__) && defined(__LONG_MAX__) && defined(__LONG_LONG_MAX__) && \ + defined(__INT32_MAX__) && defined(__INT64_MAX__) + #if __INT_MAX__ == __INT32_MAX__ + #define simde_x_clz32(v) __builtin_clz(HEDLEY_STATIC_CAST(unsigned int, (v))) + #elif __LONG_MAX__ == __INT32_MAX__ + #define simde_x_clz32(v) __builtin_clzl(HEDLEY_STATIC_CAST(unsigned long, (v))) + #elif __LONG_LONG_MAX__ == __INT32_MAX__ + #define simde_x_clz32(v) __builtin_clzll(HEDLEY_STATIC_CAST(unsigned long long, (v))) + #endif + + #if __INT_MAX__ == __INT64_MAX__ + #define simde_x_clz64(v) __builtin_clz(HEDLEY_STATIC_CAST(unsigned int, (v))) + #elif __LONG_MAX__ == __INT64_MAX__ + #define simde_x_clz64(v) __builtin_clzl(HEDLEY_STATIC_CAST(unsigned long, (v))) + #elif __LONG_LONG_MAX__ == __INT64_MAX__ + #define simde_x_clz64(v) __builtin_clzll(HEDLEY_STATIC_CAST(unsigned long long, (v))) + #endif +#elif HEDLEY_MSVC_VERSION_CHECK(14,0,0) + static int simde_x_clz32(uint32_t x) { + unsigned long r; + _BitScanReverse(&r, x); + return 31 - HEDLEY_STATIC_CAST(int, r); + } + #define simde_x_clz32 simde_x_clz32 + + static int simde_x_clz64(uint64_t x) { + unsigned long r; + + #if defined(_M_AMD64) || defined(_M_ARM64) + _BitScanReverse64(&r, x); + return 63 - HEDLEY_STATIC_CAST(int, r); + #else + uint32_t high = HEDLEY_STATIC_CAST(uint32_t, x >> 32); + if (high != 0) + return _BitScanReverse(&r, HEDLEY_STATIC_CAST(unsigned long, high)); + else + return _BitScanReverse(&r, HEDLEY_STATIC_CAST(unsigned long, x & ~UINT32_C(0))) + 32; + #endif + } + #define simde_x_clz64 simde_x_clz64 +#endif + +#if !defined(simde_x_clz32) || !defined(simde_x_clz64) + static uint8_t simde_x_avx512cd_lz_lookup(const uint8_t value) { + static const uint8_t lut[256] = { + 7, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + return lut[value]; + }; + + #if !defined(simde_x_clz32) + static int simde_x_clz32(uint32_t x) { + size_t s = sizeof(x) * 8; + uint32_t r; + + while ((s -= 8) != 0) { + r = x >> s; + if (r != 0) + return simde_x_avx512cd_lz_lookup(HEDLEY_STATIC_CAST(uint8_t, r)) + + (((sizeof(x) - 1) * 8) - s); + } + + if (x == 0) + return (int) ((sizeof(x) * 8) - 1); + else + return simde_x_avx512cd_lz_lookup(HEDLEY_STATIC_CAST(uint8_t, x)) + + ((sizeof(x) - 1) * 8); + } + #endif + + #if !defined(simde_x_clz64) + static int simde_x_clz64(uint64_t x) { + size_t s = sizeof(x) * 8; + uint64_t r; + + while ((s -= 8) != 0) { + r = x >> s; + if (r != 0) + return simde_x_avx512cd_lz_lookup(HEDLEY_STATIC_CAST(uint8_t, r)) + + (((sizeof(x) - 1) * 8) - s); + } + + if (x == 0) + return (int) ((sizeof(x) * 8) - 1); + else + return simde_x_avx512cd_lz_lookup(HEDLEY_STATIC_CAST(uint8_t, x)) + + ((sizeof(x) - 1) * 8); + } + #endif +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_lzcnt_epi32(simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm_lzcnt_epi32(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + /* https://stackoverflow.com/a/58827596/501126 */ + a = _mm_andnot_si128(_mm_srli_epi32(a, 8), a); + a = _mm_castps_si128(_mm_cvtepi32_ps(a)); + a = _mm_srli_epi32(a, 23); + a = _mm_subs_epu16(_mm_set1_epi32(158), a); + a = _mm_min_epi16(a, _mm_set1_epi32(32)); + return a; + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u32 = vec_cntlz(a_.altivec_u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0; i < (sizeof(r_.i32) / sizeof(r_.i32[0])); i++) { + r_.i32[i] = (HEDLEY_UNLIKELY(a_.i32[i] == 0) ? HEDLEY_STATIC_CAST(int32_t, sizeof(int32_t) * CHAR_BIT) : HEDLEY_STATIC_CAST(int32_t, simde_x_clz32(HEDLEY_STATIC_CAST(uint32_t, a_.i32[i])))); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512CD_ENABLE_NATIVE_ALIASES) + #undef _mm_lzcnt_epi32 + #define _mm_lzcnt_epi32(a) simde_mm_lzcnt_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_lzcnt_epi32(simde__m128i src, simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm_mask_lzcnt_epi32(src, k, a); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_lzcnt_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_lzcnt_epi32 + #define _mm_mask_lzcnt_epi32(src, k, a) simde_mm_mask_lzcnt_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_lzcnt_epi32(simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512CD_NATIVE) + return _mm_maskz_lzcnt_epi32(k, a); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_lzcnt_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_lzcnt_epi32 + #define _mm_maskz_lzcnt_epi32(k, a) simde_mm_maskz_lzcnt_epi32(k, a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_LZCNT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/madd.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/madd.h new file mode 100644 index 00000000..547d71ce --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/madd.h @@ -0,0 +1,157 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Ashleigh Newman-Jones + */ + +#if !defined(SIMDE_X86_AVX512_MADD_H) +#define SIMDE_X86_AVX512_MADD_H + +#include "types.h" +#include "mov.h" +#include "../avx2.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_madd_epi16 (simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_madd_epi16(src, k, a, b); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_madd_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_madd_epi16 + #define _mm_mask_madd_epi16(src, k, a, b) simde_mm_mask_madd_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_madd_epi16 (simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_madd_epi16(k, a, b); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_madd_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_madd_epi16 + #define _mm_maskz_madd_epi16(k, a, b) simde_mm_maskz_madd_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_madd_epi16 (simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_madd_epi16(src, k, a, b); + #else + return simde_mm256_mask_mov_epi32(src, k, simde_mm256_madd_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_madd_epi16 + #define _mm256_mask_madd_epi16(src, k, a, b) simde_mm256_mask_madd_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_madd_epi16 (simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_madd_epi16(k, a, b); + #else + return simde_mm256_maskz_mov_epi32(k, simde_mm256_madd_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_madd_epi16 + #define _mm256_maskz_madd_epi16(k, a, b) simde_mm256_maskz_madd_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_madd_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_madd_epi16(a, b); + #else + simde__m512i_private r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if 0 && SIMDE_NATURAL_VECTOR_SIZE_LE(256) || defined(SIMDE_BUG_CLANG_BAD_MADD) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_madd_epi16(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_) / sizeof(r_.i16[0])) ; i += 2) { + r_.i32[i / 2] = + (HEDLEY_STATIC_CAST(int32_t, a_.i16[ i ]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[ i ])) + + (HEDLEY_STATIC_CAST(int32_t, a_.i16[i + 1]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[i + 1])); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_madd_epi16 + #define _mm512_madd_epi16(a, b) simde_mm512_madd_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_madd_epi16 (simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_madd_epi16(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_madd_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_madd_epi16 + #define _mm512_mask_madd_epi16(src, k, a, b) simde_mm512_mask_madd_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_madd_epi16 (simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_madd_epi16(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_madd_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_madd_epi16 + #define _mm512_maskz_madd_epi16(k, a, b) simde_mm512_maskz_madd_epi16(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_MADD_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/maddubs.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/maddubs.h new file mode 100644 index 00000000..43b5594c --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/maddubs.h @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Ashleigh Newman-Jones + */ + +#if !defined(SIMDE_X86_AVX512_MADDUBS_H) +#define SIMDE_X86_AVX512_MADDUBS_H + +#include "types.h" +#include "mov.h" +#include "../avx2.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_maddubs_epi16 (simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_maddubs_epi16(src, k, a, b); + #else + return simde_mm_mask_mov_epi16(src, k, simde_mm_maddubs_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_maddubs_epi16 + #define _mm_mask_maddubs_epi16(src, k, a, b) simde_mm_mask_maddubs_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_maddubs_epi16 (simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE ) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_maddubs_epi16(k, a, b); + #else + return simde_mm_maskz_mov_epi16(k, simde_mm_maddubs_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_maddubs_epi16 + #define _mm_maskz_maddubs_epi16(k, a, b) simde_mm_maskz_maddubs_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_maddubs_epi16 (simde__m256i src, simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_maddubs_epi16(src, k, a, b); + #else + return simde_mm256_mask_mov_epi16(src, k, simde_mm256_maddubs_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_maddubs_epi16 + #define _mm256_mask_maddubs_epi16(src, k, a, b) simde_mm256_mask_maddubs_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_maddubs_epi16 (simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_maddubs_epi16(k, a, b); + #else + return simde_mm256_maskz_mov_epi16(k, simde_mm256_maddubs_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_maddubs_epi16 + #define _mm256_maskz_maddubs_epi16(k, a, b) simde_mm256_maskz_maddubs_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maddubs_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maddubs_epi16(a, b); + #else + simde__m512i_private r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) || defined(SIMDE_BUG_CLANG_BAD_MADD) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_maddubs_epi16(a_.m256i[i], b_.m256i[i]); + } + #else + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + const int idx = HEDLEY_STATIC_CAST(int, i) << 1; + int32_t ts = + (HEDLEY_STATIC_CAST(int16_t, a_.u8[ idx ]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[ idx ])) + + (HEDLEY_STATIC_CAST(int16_t, a_.u8[idx + 1]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[idx + 1])); + r_.i16[i] = (ts > INT16_MIN) ? ((ts < INT16_MAX) ? HEDLEY_STATIC_CAST(int16_t, ts) : INT16_MAX) : INT16_MIN; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maddubs_epi16 + #define _mm512_maddubs_epi16(a, b) simde_mm512_maddubs_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_maddubs_epi16 (simde__m512i src, simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_maddubs_epi16(src, k, a, b); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_maddubs_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_maddubs_epi16 + #define _mm512_mask_maddubs_epi16(src, k, a, b) simde_mm512_mask_maddubs_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_maddubs_epi16 (simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_maddubs_epi16(k, a, b); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_maddubs_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_maddubs_epi16 + #define _mm512_maskz_maddubs_epi16(k, a, b) simde_mm512_maskz_maddubs_epi16(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_MADDUBS_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/max.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/max.h new file mode 100644 index 00000000..29ef0b37 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/max.h @@ -0,0 +1,611 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_MAX_H) +#define SIMDE_X86_AVX512_MAX_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_max_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_max_epi8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] > b_.i8[i]) ? a_.i8[i] : b_.i8[i]; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) +# define _mm512_max_epi8(a, b) simde_mm512_max_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_max_epi8 (simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_max_epi8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_max_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_max_epi8 + #define _mm512_mask_max_epi8(src, k, a, b) simde_mm512_mask_max_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_max_epi8 (simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_max_epi8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_max_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_max_epi8 + #define _mm512_maskz_max_epi8(k, a, b) simde_mm512_maskz_max_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_max_epu8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_max_epu8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(a_.m256i) / sizeof(a_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_max_epu8(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] > b_.u8[i]) ? a_.u8[i] : b_.u8[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_max_epu8 + #define _mm512_max_epu8(a, b) simde_mm512_max_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_max_epu8 (simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_max_epu8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_max_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_max_epu8 + #define _mm512_mask_max_epu8(src, k, a, b) simde_mm512_mask_max_epu8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_max_epu8 (simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_max_epu8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_max_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_max_epu8 + #define _mm512_maskz_max_epu8(k, a, b) simde_mm512_maskz_max_epu8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_max_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_max_epi16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] > b_.i16[i]) ? a_.i16[i] : b_.i16[i]; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) +# define _mm512_max_epi16(a, b) simde_mm512_max_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_max_epi16 (simde__m512i src, simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_max_epi16(src, k, a, b); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_max_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_max_epi16 + #define _mm512_mask_max_epi16(src, k, a, b) simde_mm512_mask_max_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_max_epi16 (simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_max_epi16(k, a, b); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_max_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_max_epi16 + #define _mm512_maskz_max_epi16(k, a, b) simde_mm512_maskz_max_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_max_epu16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_max_epu16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(a_.m256i) / sizeof(a_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_max_epu16(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] > b_.u16[i]) ? a_.u16[i] : b_.u16[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_max_epu16 + #define _mm512_max_epu16(a, b) simde_mm512_max_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_max_epu16 (simde__m512i src, simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_max_epu16(src, k, a, b); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_max_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_max_epu16 + #define _mm512_mask_max_epu16(src, k, a, b) simde_mm512_mask_max_epu16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_max_epu16 (simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_max_epu16(k, a, b); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_max_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_max_epu16 + #define _mm512_maskz_max_epu16(k, a, b) simde_mm512_maskz_max_epu16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_max_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_max_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256i[0] = simde_mm256_max_epi32(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_max_epi32(a_.m256i[1], b_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] > b_.i32[i] ? a_.i32[i] : b_.i32[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_max_epi32 + #define _mm512_max_epi32(a, b) simde_mm512_max_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_max_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_max_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_max_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_max_epi32 + #define _mm512_mask_max_epi32(src, k, a, b) simde_mm512_mask_max_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_max_epi32(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_max_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_max_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_max_epi32 + #define _mm512_maskz_max_epi32(k, a, b) simde_mm512_maskz_max_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_max_epu32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_max_epu32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256i[0] = simde_mm256_max_epu32(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_max_epu32(a_.m256i[1], b_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] > b_.u32[i]) ? a_.u32[i] : b_.u32[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_max_epu32 + #define _mm512_max_epu32(a, b) simde_mm512_max_epu32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_max_epu32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_max_epu32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_max_epu32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_max_epu32 + #define _mm512_mask_max_epu32(src, k, a, b) simde_mm512_mask_max_epu32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_max_epu32(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_max_epu32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_max_epu32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_max_epu32 + #define _mm512_maskz_max_epu32(k, a, b) simde_mm512_maskz_max_epu32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_max_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_max_epi64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] > b_.i64[i] ? a_.i64[i] : b_.i64[i]; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_max_epi64 + #define _mm512_max_epi64(a, b) simde_mm512_max_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_max_epi64(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_max_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_max_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_max_epi64 + #define _mm512_mask_max_epi64(src, k, a, b) simde_mm512_mask_max_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_max_epi64(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_max_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_max_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_max_epi64 + #define _mm512_maskz_max_epi64(k, a, b) simde_mm512_maskz_max_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_max_epu64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_max_epu64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] > b_.u64[i]) ? a_.u64[i] : b_.u64[i]; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_max_epu64 + #define _mm512_max_epu64(a, b) simde_mm512_max_epu64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_max_epu64(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_max_epu64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_max_epu64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_max_epu64 + #define _mm512_mask_max_epu64(src, k, a, b) simde_mm512_mask_max_epu64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_max_epu64(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_max_epu64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_max_epu64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_max_epu64 + #define _mm512_maskz_max_epu64(k, a, b) simde_mm512_maskz_max_epu64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_max_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_max_ps(a, b); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256[0] = simde_mm256_max_ps(a_.m256[0], b_.m256[0]); + r_.m256[1] = simde_mm256_max_ps(a_.m256[1], b_.m256[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[i] > b_.f32[i] ? a_.f32[i] : b_.f32[i]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_max_ps + #define _mm512_max_ps(a, b) simde_mm512_max_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_max_ps(simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_max_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_max_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_max_ps + #define _mm512_mask_max_ps(src, k, a, b) simde_mm512_mask_max_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_max_ps(simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_max_ps(k, a, b); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_max_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_max_ps + #define _mm512_maskz_max_ps(k, a, b) simde_mm512_maskz_max_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_max_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_max_pd(a, b); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[i] > b_.f64[i] ? a_.f64[i] : b_.f64[i]; + } + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_max_pd + #define _mm512_max_pd(a, b) simde_mm512_max_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512h +simde_mm512_max_ph (simde__m512h a, simde__m512h b) { + #if defined(SIMDE_X86_AVX512FP16_NATIVE) + return _mm512_max_ph(a, b); + #else + simde__m512h_private + r_, + a_ = simde__m512h_to_private(a), + b_ = simde__m512h_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.f16[i] = simde_float16_to_float32(a_.f16[i]) > simde_float16_to_float32(b_.f16[i]) ? a_.f16[i] : b_.f16[i]; + } + + return simde__m512h_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) + #undef _mm512_max_ph + #define _mm512_max_ph(a, b) simde_mm512_max_ph(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_max_pd(simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_max_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_max_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_max_pd + #define _mm512_mask_max_pd(src, k, a, b) simde_mm512_mask_max_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_max_pd(simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_max_pd(k, a, b); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_max_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_max_pd + #define _mm512_maskz_max_pd(k, a, b) simde_mm512_maskz_max_pd(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_MAX_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/min.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/min.h new file mode 100644 index 00000000..2e1dd843 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/min.h @@ -0,0 +1,611 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_MIN_H) +#define SIMDE_X86_AVX512_MIN_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_min_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_min_epi8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] < b_.i8[i]) ? a_.i8[i] : b_.i8[i]; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) +# define _mm512_min_epi8(a, b) simde_mm512_min_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_min_epi8 (simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_min_epi8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_min_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_min_epi8 + #define _mm512_mask_min_epi8(src, k, a, b) simde_mm512_mask_min_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_min_epi8 (simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_min_epi8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_min_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_min_epi8 + #define _mm512_maskz_min_epi8(k, a, b) simde_mm512_maskz_min_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_min_epu8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_min_epu8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(a_.m256i) / sizeof(a_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_min_epu8(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] < b_.u8[i]) ? a_.u8[i] : b_.u8[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_min_epu8 + #define _mm512_min_epu8(a, b) simde_mm512_min_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_min_epu8 (simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_min_epu8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_min_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_min_epu8 + #define _mm512_mask_min_epu8(src, k, a, b) simde_mm512_mask_min_epu8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_min_epu8 (simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_min_epu8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_min_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_min_epu8 + #define _mm512_maskz_min_epu8(k, a, b) simde_mm512_maskz_min_epu8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_min_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_min_epi16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] < b_.i16[i]) ? a_.i16[i] : b_.i16[i]; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) +# define _mm512_min_epi16(a, b) simde_mm512_min_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_min_epi16 (simde__m512i src, simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_min_epi16(src, k, a, b); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_min_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_min_epi16 + #define _mm512_mask_min_epi16(src, k, a, b) simde_mm512_mask_min_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_min_epi16 (simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_min_epi16(k, a, b); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_min_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_min_epi16 + #define _mm512_maskz_min_epi16(k, a, b) simde_mm512_maskz_min_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_min_epu16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_min_epu16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(a_.m256i) / sizeof(a_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_min_epu16(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] < b_.u16[i]) ? a_.u16[i] : b_.u16[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_min_epu16 + #define _mm512_min_epu16(a, b) simde_mm512_min_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_min_epu16 (simde__m512i src, simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_min_epu16(src, k, a, b); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_min_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_min_epu16 + #define _mm512_mask_min_epu16(src, k, a, b) simde_mm512_mask_min_epu16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_min_epu16 (simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_min_epu16(k, a, b); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_min_epu16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_min_epu16 + #define _mm512_maskz_min_epu16(k, a, b) simde_mm512_maskz_min_epu16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_min_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_min_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256i[0] = simde_mm256_min_epi32(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_min_epi32(a_.m256i[1], b_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] < b_.i32[i] ? a_.i32[i] : b_.i32[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_min_epi32 + #define _mm512_min_epi32(a, b) simde_mm512_min_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_min_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_min_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_min_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_min_epi32 + #define _mm512_mask_min_epi32(src, k, a, b) simde_mm512_mask_min_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i + simde_mm512_maskz_min_epi32(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_min_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_min_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_min_epi32 + #define _mm512_maskz_min_epi32(k, a, b) simde_mm512_maskz_min_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_min_epu32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_min_epu32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256i[0] = simde_mm256_min_epu32(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_min_epu32(a_.m256i[1], b_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] < b_.u32[i]) ? a_.u32[i] : b_.u32[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_min_epu32 + #define _mm512_min_epu32(a, b) simde_mm512_min_epu32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_min_epu32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_min_epu32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_min_epu32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_min_epu32 + #define _mm512_mask_min_epu32(src, k, a, b) simde_mm512_mask_min_epu32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_min_epu32(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_min_epu32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_min_epu32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_min_epu32 + #define _mm512_maskz_min_epu32(k, a, b) simde_mm512_maskz_min_epu32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_min_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_min_epi64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] < b_.i64[i] ? a_.i64[i] : b_.i64[i]; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_min_epi64 + #define _mm512_min_epi64(a, b) simde_mm512_min_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_min_epi64(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_min_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_min_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_min_epi64 + #define _mm512_mask_min_epi64(src, k, a, b) simde_mm512_mask_min_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_min_epi64(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_min_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_min_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_min_epi64 + #define _mm512_maskz_min_epi64(k, a, b) simde_mm512_maskz_min_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_min_epu64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_min_epu64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] < b_.u64[i]) ? a_.u64[i] : b_.u64[i]; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_min_epu64 + #define _mm512_min_epu64(a, b) simde_mm512_min_epu64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_min_epu64(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_min_epu64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_min_epu64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_min_epu64 + #define _mm512_mask_min_epu64(src, k, a, b) simde_mm512_mask_min_epu64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_min_epu64(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_min_epu64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_min_epu64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_min_epu64 + #define _mm512_maskz_min_epu64(k, a, b) simde_mm512_maskz_min_epu64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_min_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_min_ps(a, b); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256[0] = simde_mm256_min_ps(a_.m256[0], b_.m256[0]); + r_.m256[1] = simde_mm256_min_ps(a_.m256[1], b_.m256[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[i] < b_.f32[i] ? a_.f32[i] : b_.f32[i]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_min_ps + #define _mm512_min_ps(a, b) simde_mm512_min_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_min_ps(simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_min_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_min_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_min_ps + #define _mm512_mask_min_ps(src, k, a, b) simde_mm512_mask_min_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_min_ps(simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_min_ps(k, a, b); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_min_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_min_ps + #define _mm512_maskz_min_ps(k, a, b) simde_mm512_maskz_min_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_min_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_min_pd(a, b); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[i] < b_.f64[i] ? a_.f64[i] : b_.f64[i]; + } + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_min_pd + #define _mm512_min_pd(a, b) simde_mm512_min_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_min_pd(simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_min_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_min_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_min_pd + #define _mm512_mask_min_pd(src, k, a, b) simde_mm512_mask_min_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_min_pd(simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_min_pd(k, a, b); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_min_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_min_pd + #define _mm512_maskz_min_pd(k, a, b) simde_mm512_maskz_min_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512h +simde_mm512_min_ph (simde__m512h a, simde__m512h b) { + #if defined(SIMDE_X86_AVX512FP16_NATIVE) + return _mm512_min_ph(a, b); + #else + simde__m512h_private + r_, + a_ = simde__m512h_to_private(a), + b_ = simde__m512h_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.f16[i] = simde_float16_to_float32(a_.f16[i]) < simde_float16_to_float32(b_.f16[i]) ? a_.f16[i] : b_.f16[i]; + } + + return simde__m512h_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) + #undef _mm512_min_ph + #define _mm512_min_ph(a, b) simde_mm512_min_ph(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_MIN_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mov.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mov.h new file mode 100644 index 00000000..cee9dbb3 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mov.h @@ -0,0 +1,865 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Christopher Moore + */ + +#if !defined(SIMDE_X86_AVX512_MOV_H) +#define SIMDE_X86_AVX512_MOV_H + +#include "types.h" +#include "cast.h" +#include "set.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_mov_epi8 (simde__m128i src, simde__mmask16 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_mov_epi8(src, k, a); + #else + simde__m128i_private + src_ = simde__m128i_to_private(src), + a_ = simde__m128i_to_private(a), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = ((k >> i) & 1) ? a_.i8[i] : src_.i8[i]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_mov_epi8 + #define _mm_mask_mov_epi8(src, k, a) simde_mm_mask_mov_epi8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_mov_epi16 (simde__m128i src, simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_mov_epi16(src, k, a); + #else + simde__m128i_private + src_ = simde__m128i_to_private(src), + a_ = simde__m128i_to_private(a), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = ((k >> i) & 1) ? a_.i16[i] : src_.i16[i]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_mov_epi16 + #define _mm_mask_mov_epi16(src, k, a) simde_mm_mask_mov_epi16(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_mov_epi32 (simde__m128i src, simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_mov_epi32(src, k, a); + #else + simde__m128i_private + src_ = simde__m128i_to_private(src), + a_ = simde__m128i_to_private(a), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ((k >> i) & 1) ? a_.i32[i] : src_.i32[i]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_mov_epi32 + #define _mm_mask_mov_epi32(src, k, a) simde_mm_mask_mov_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_mov_epi64 (simde__m128i src, simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_mov_epi64(src, k, a); + #else + simde__m128i_private + src_ = simde__m128i_to_private(src), + a_ = simde__m128i_to_private(a), + r_; + + /* N.B. CM: No fallbacks as there are only two elements */ + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = ((k >> i) & 1) ? a_.i64[i] : src_.i64[i]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_mov_epi64 + #define _mm_mask_mov_epi64(src, k, a) simde_mm_mask_mov_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mask_mov_pd(simde__m128d src, simde__mmask8 k, simde__m128d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_mov_pd(src, k, a); + #else + return simde_mm_castsi128_pd(simde_mm_mask_mov_epi64(simde_mm_castpd_si128(src), k, simde_mm_castpd_si128(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_mov_pd + #define _mm_mask_mov_pd(src, k, a) simde_mm_mask_mov_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mask_mov_ps (simde__m128 src, simde__mmask8 k, simde__m128 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_mov_ps(src, k, a); + #else + return simde_mm_castsi128_ps(simde_mm_mask_mov_epi32(simde_mm_castps_si128(src), k, simde_mm_castps_si128(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_mov_ps + #define _mm_mask_mov_ps(src, k, a) simde_mm_mask_mov_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_mov_epi8 (simde__m256i src, simde__mmask32 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_mov_epi8(src, k, a); + #else + simde__m256i_private + r_, + src_ = simde__m256i_to_private(src), + a_ = simde__m256i_to_private(a); + + #if defined(SIMDE_X86_SSSE3_NATIVE) + r_.m128i[0] = simde_mm_mask_mov_epi8(src_.m128i[0], HEDLEY_STATIC_CAST(simde__mmask16, k ), a_.m128i[0]); + r_.m128i[1] = simde_mm_mask_mov_epi8(src_.m128i[1], HEDLEY_STATIC_CAST(simde__mmask16, k >> 16), a_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = ((k >> i) & 1) ? a_.i8[i] : src_.i8[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_mov_epi8 + #define _mm256_mask_mov_epi8(src, k, a) simde_mm256_mask_mov_epi8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_mov_epi16 (simde__m256i src, simde__mmask16 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_mov_epi16(src, k, a); + #else + simde__m256i_private + src_ = simde__m256i_to_private(src), + a_ = simde__m256i_to_private(a), + r_; + + #if defined(SIMDE_X86_SSE2_NATIVE) + r_.m128i[0] = simde_mm_mask_mov_epi16(src_.m128i[0], HEDLEY_STATIC_CAST(simde__mmask8, k ), a_.m128i[0]); + r_.m128i[1] = simde_mm_mask_mov_epi16(src_.m128i[1], HEDLEY_STATIC_CAST(simde__mmask8, k >> 8), a_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = ((k >> i) & 1) ? a_.i16[i] : src_.i16[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_mov_epi16 + #define _mm256_mask_mov_epi16(src, k, a) simde_mm256_mask_mov_epi16(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_mov_epi32 (simde__m256i src, simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_mov_epi32(src, k, a); + #else + simde__m256i_private + src_ = simde__m256i_to_private(src), + a_ = simde__m256i_to_private(a), + r_; + + #if defined(SIMDE_X86_SSE2_NATIVE) + r_.m128i[0] = simde_mm_mask_mov_epi32(src_.m128i[0], k , a_.m128i[0]); + r_.m128i[1] = simde_mm_mask_mov_epi32(src_.m128i[1], k >> 4, a_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ((k >> i) & 1) ? a_.i32[i] : src_.i32[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_mov_epi32 + #define _mm256_mask_mov_epi32(src, k, a) simde_mm256_mask_mov_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_mov_epi64 (simde__m256i src, simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_mov_epi64(src, k, a); + #else + simde__m256i_private + src_ = simde__m256i_to_private(src), + a_ = simde__m256i_to_private(a), + r_; + + /* N.B. CM: This fallback may not be faster as there are only four elements */ + #if defined(SIMDE_X86_SSE2_NATIVE) + r_.m128i[0] = simde_mm_mask_mov_epi64(src_.m128i[0], k , a_.m128i[0]); + r_.m128i[1] = simde_mm_mask_mov_epi64(src_.m128i[1], k >> 2, a_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = ((k >> i) & 1) ? a_.i64[i] : src_.i64[i]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_mov_epi64 + #define _mm256_mask_mov_epi64(src, k, a) simde_mm256_mask_mov_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mask_mov_pd (simde__m256d src, simde__mmask8 k, simde__m256d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_mov_pd(src, k, a); + #else + return simde_mm256_castsi256_pd(simde_mm256_mask_mov_epi64(simde_mm256_castpd_si256(src), k, simde_mm256_castpd_si256(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_mov_pd + #define _mm256_mask_mov_pd(src, k, a) simde_mm256_mask_mov_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask_mov_ps (simde__m256 src, simde__mmask8 k, simde__m256 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_mov_ps(src, k, a); + #else + return simde_mm256_castsi256_ps(simde_mm256_mask_mov_epi32(simde_mm256_castps_si256(src), k, simde_mm256_castps_si256(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_mov_ps + #define _mm256_mask_mov_ps(src, k, a) simde_mm256_mask_mov_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_mov_epi8 (simde__m512i src, simde__mmask64 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_mov_epi8(src, k, a); + #else + simde__m512i_private + src_ = simde__m512i_to_private(src), + a_ = simde__m512i_to_private(a), + r_; + + #if defined(SIMDE_X86_SSSE3_NATIVE) + r_.m256i[0] = simde_mm256_mask_mov_epi8(src_.m256i[0], HEDLEY_STATIC_CAST(simde__mmask32, k ), a_.m256i[0]); + r_.m256i[1] = simde_mm256_mask_mov_epi8(src_.m256i[1], HEDLEY_STATIC_CAST(simde__mmask32, k >> 32), a_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = ((k >> i) & 1) ? a_.i8[i] : src_.i8[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_mov_epi8 + #define _mm512_mask_mov_epi8(src, k, a) simde_mm512_mask_mov_epi8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_mov_epi16 (simde__m512i src, simde__mmask32 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_mov_epi16(src, k, a); + #else + simde__m512i_private + src_ = simde__m512i_to_private(src), + a_ = simde__m512i_to_private(a), + r_; + + #if defined(SIMDE_X86_SSE2_NATIVE) + r_.m256i[0] = simde_mm256_mask_mov_epi16(src_.m256i[0], HEDLEY_STATIC_CAST(simde__mmask16, k ), a_.m256i[0]); + r_.m256i[1] = simde_mm256_mask_mov_epi16(src_.m256i[1], HEDLEY_STATIC_CAST(simde__mmask16, k >> 16), a_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = ((k >> i) & 1) ? a_.i16[i] : src_.i16[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_mov_epi16 + #define _mm512_mask_mov_epi16(src, k, a) simde_mm512_mask_mov_epi16(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_mov_epi32 (simde__m512i src, simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_mov_epi32(src, k, a); + #else + simde__m512i_private + src_ = simde__m512i_to_private(src), + a_ = simde__m512i_to_private(a), + r_; + + #if defined(SIMDE_X86_SSE2_NATIVE) + r_.m256i[0] = simde_mm256_mask_mov_epi32(src_.m256i[0], HEDLEY_STATIC_CAST(simde__mmask8, k ), a_.m256i[0]); + r_.m256i[1] = simde_mm256_mask_mov_epi32(src_.m256i[1], HEDLEY_STATIC_CAST(simde__mmask8, k >> 8), a_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ((k >> i) & 1) ? a_.i32[i] : src_.i32[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_mov_epi32 + #define _mm512_mask_mov_epi32(src, k, a) simde_mm512_mask_mov_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_mov_epi64 (simde__m512i src, simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_mov_epi64(src, k, a); + #else + simde__m512i_private + src_ = simde__m512i_to_private(src), + a_ = simde__m512i_to_private(a), + r_; + + /* N.B. CM: Without AVX2 this fallback may not be faster as there are only eight elements */ + #if defined(SIMDE_X86_SSE2_NATIVE) + r_.m256i[0] = simde_mm256_mask_mov_epi64(src_.m256i[0], k , a_.m256i[0]); + r_.m256i[1] = simde_mm256_mask_mov_epi64(src_.m256i[1], k >> 4, a_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = ((k >> i) & 1) ? a_.i64[i] : src_.i64[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_mov_epi64 + #define _mm512_mask_mov_epi64(src, k, a) simde_mm512_mask_mov_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_mov_pd (simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_mov_pd(src, k, a); + #else + return simde_mm512_castsi512_pd(simde_mm512_mask_mov_epi64(simde_mm512_castpd_si512(src), k, simde_mm512_castpd_si512(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_mov_pd + #define _mm512_mask_mov_pd(src, k, a) simde_mm512_mask_mov_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_mov_ps (simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_mov_ps(src, k, a); + #else + return simde_mm512_castsi512_ps(simde_mm512_mask_mov_epi32(simde_mm512_castps_si512(src), k, simde_mm512_castps_si512(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_mov_ps + #define _mm512_mask_mov_ps(src, k, a) simde_mm512_mask_mov_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512h +simde_x_mm512_mask_mov_ph (simde__m512h src, simde__mmask32 k, simde__m512h a) { + return simde_mm512_castsi512_ph(simde_mm512_mask_mov_epi16(simde_mm512_castph_si512(src), k, simde_mm512_castph_si512(a))); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_mov_epi8 (simde__mmask16 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_mov_epi8(k, a); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = ((k >> i) & 1) ? a_.i8[i] : INT8_C(0); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_mov_epi8 + #define _mm_maskz_mov_epi8(k, a) simde_mm_maskz_mov_epi8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_mov_epi16 (simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_mov_epi16(k, a); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = ((k >> i) & 1) ? a_.i16[i] : INT16_C(0); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_mov_epi16 + #define _mm_maskz_mov_epi16(k, a) simde_mm_maskz_mov_epi16(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_mov_epi32 (simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_mov_epi32(k, a); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ((k >> i) & 1) ? a_.i32[i] : INT32_C(0); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_mov_epi32 + #define _mm_maskz_mov_epi32(k, a) simde_mm_maskz_mov_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_mov_epi64 (simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_mov_epi64(k, a); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + r_; + + /* N.B. CM: No fallbacks as there are only two elements */ + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = ((k >> i) & 1) ? a_.i64[i] : INT64_C(0); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_mov_epi64 + #define _mm_maskz_mov_epi64(k, a) simde_mm_maskz_mov_epi64(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_maskz_mov_pd (simde__mmask8 k, simde__m128d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_mov_pd(k, a); + #else + return simde_mm_castsi128_pd(simde_mm_maskz_mov_epi64(k, simde_mm_castpd_si128(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_mov_pd + #define _mm_maskz_mov_pd(k, a) simde_mm_maskz_mov_pd(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_maskz_mov_ps (simde__mmask8 k, simde__m128 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_mov_ps(k, a); + #else + return simde_mm_castsi128_ps(simde_mm_maskz_mov_epi32(k, simde_mm_castps_si128(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_mov_ps + #define _mm_maskz_mov_ps(k, a) simde_mm_maskz_mov_ps(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_mov_epi8 (simde__mmask32 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_mov_epi8(k, a); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + r_; + + #if defined(SIMDE_X86_SSSE3_NATIVE) + r_.m128i[0] = simde_mm_maskz_mov_epi8(HEDLEY_STATIC_CAST(simde__mmask16, k ), a_.m128i[0]); + r_.m128i[1] = simde_mm_maskz_mov_epi8(HEDLEY_STATIC_CAST(simde__mmask16, k >> 16), a_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = ((k >> i) & 1) ? a_.i8[i] : INT8_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_mov_epi8 + #define _mm256_maskz_mov_epi8(k, a) simde_mm256_maskz_mov_epi8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_mov_epi16 (simde__mmask16 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_mov_epi16(k, a); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + r_; + + #if defined(SIMDE_X86_SSE2_NATIVE) + r_.m128i[0] = simde_mm_maskz_mov_epi16(HEDLEY_STATIC_CAST(simde__mmask8, k ), a_.m128i[0]); + r_.m128i[1] = simde_mm_maskz_mov_epi16(HEDLEY_STATIC_CAST(simde__mmask8, k >> 8), a_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = ((k >> i) & 1) ? a_.i16[i] : INT16_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_mov_epi16 + #define _mm256_maskz_mov_epi16(k, a) simde_mm256_maskz_mov_epi16(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_mov_epi32 (simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_mov_epi32(k, a); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + r_; + + #if defined(SIMDE_X86_SSE2_NATIVE) + r_.m128i[0] = simde_mm_maskz_mov_epi32(k , a_.m128i[0]); + r_.m128i[1] = simde_mm_maskz_mov_epi32(k >> 4, a_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ((k >> i) & 1) ? a_.i32[i] : INT32_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_mov_epi32 + #define _mm256_maskz_mov_epi32(k, a) simde_mm256_maskz_mov_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_mov_epi64 (simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_mov_epi64(k, a); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + r_; + + /* N.B. CM: This fallback may not be faster as there are only four elements */ + #if defined(SIMDE_X86_SSE2_NATIVE) + r_.m128i[0] = simde_mm_maskz_mov_epi64(k , a_.m128i[0]); + r_.m128i[1] = simde_mm_maskz_mov_epi64(k >> 2, a_.m128i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = ((k >> i) & 1) ? a_.i64[i] : INT64_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_mov_epi64 + #define _mm256_maskz_mov_epi64(k, a) simde_mm256_maskz_mov_epi64(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_maskz_mov_pd (simde__mmask8 k, simde__m256d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_mov_pd(k, a); + #else + return simde_mm256_castsi256_pd(simde_mm256_maskz_mov_epi64(k, simde_mm256_castpd_si256(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_mov_pd + #define _mm256_maskz_mov_pd(k, a) simde_mm256_maskz_mov_pd(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_maskz_mov_ps (simde__mmask8 k, simde__m256 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_mov_ps(k, a); + #else + return simde_mm256_castsi256_ps(simde_mm256_maskz_mov_epi32(k, simde_mm256_castps_si256(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_mov_ps + #define _mm256_maskz_mov_ps(k, a) simde_mm256_maskz_mov_ps(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_mov_epi8 (simde__mmask64 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_mov_epi8(k, a); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + r_; + + #if defined(SIMDE_X86_SSSE3_NATIVE) + r_.m256i[0] = simde_mm256_maskz_mov_epi8(HEDLEY_STATIC_CAST(simde__mmask32, k ), a_.m256i[0]); + r_.m256i[1] = simde_mm256_maskz_mov_epi8(HEDLEY_STATIC_CAST(simde__mmask32, k >> 32), a_.m256i[1]); + #else + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = ((k >> i) & 1) ? a_.i8[i] : INT8_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_mov_epi8 + #define _mm512_maskz_mov_epi8(k, a) simde_mm512_maskz_mov_epi8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_mov_epi16 (simde__mmask32 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_mov_epi16(k, a); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + r_; + + #if defined(SIMDE_X86_SSE2_NATIVE) + r_.m256i[0] = simde_mm256_maskz_mov_epi16(HEDLEY_STATIC_CAST(simde__mmask16, k ), a_.m256i[0]); + r_.m256i[1] = simde_mm256_maskz_mov_epi16(HEDLEY_STATIC_CAST(simde__mmask16, k >> 16), a_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = ((k >> i) & 1) ? a_.i16[i] : INT16_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_mov_epi16 + #define _mm512_maskz_mov_epi16(k, a) simde_mm512_maskz_mov_epi16(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_mov_epi32 (simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_mov_epi32(k, a); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + r_; + + #if defined(SIMDE_X86_SSE2_NATIVE) + r_.m256i[0] = simde_mm256_maskz_mov_epi32(HEDLEY_STATIC_CAST(simde__mmask8, k ), a_.m256i[0]); + r_.m256i[1] = simde_mm256_maskz_mov_epi32(HEDLEY_STATIC_CAST(simde__mmask8, k >> 8), a_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ((k >> i) & 1) ? a_.i32[i] : INT32_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_mov_epi32 + #define _mm512_maskz_mov_epi32(k, a) simde_mm512_maskz_mov_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_mov_epi64 (simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_mov_epi64(k, a); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + r_; + + /* N.B. CM: Without AVX2 this fallback may not be faster as there are only eight elements */ + #if defined(SIMDE_X86_SSE2_NATIVE) + r_.m256i[0] = simde_mm256_maskz_mov_epi64(k , a_.m256i[0]); + r_.m256i[1] = simde_mm256_maskz_mov_epi64(k >> 4, a_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = ((k >> i) & 1) ? a_.i64[i] : INT64_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_mov_epi64 + #define _mm512_maskz_mov_epi64(k, a) simde_mm512_maskz_mov_epi64(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_mov_pd (simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_mov_pd(k, a); + #else + return simde_mm512_castsi512_pd(simde_mm512_maskz_mov_epi64(k, simde_mm512_castpd_si512(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_mov_pd + #define _mm512_maskz_mov_pd(k, a) simde_mm512_maskz_mov_pd(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_mov_ps (simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_mov_ps(k, a); + #else + return simde_mm512_castsi512_ps(simde_mm512_maskz_mov_epi32(k, simde_mm512_castps_si512(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_mov_ps + #define _mm512_maskz_mov_ps(k, a) simde_mm512_maskz_mov_ps(k, a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_MOV_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mov_mask.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mov_mask.h new file mode 100644 index 00000000..1d0b1209 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mov_mask.h @@ -0,0 +1,372 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_MOV_MASK_H) +#define SIMDE_X86_AVX512_MOV_MASK_H + +#include "types.h" +#include "../avx2.h" + +#include "cast.h" +#include "set.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm_movepi8_mask (simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_movepi8_mask(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + return HEDLEY_STATIC_CAST(simde__mmask16, simde_mm_movemask_epi8(a)); + #else + simde__m128i_private a_ = simde__m128i_to_private(a); + simde__mmask16 r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i8) / sizeof(a_.i8[0])) ; i++) { + r |= (a_.i8[i] < 0) ? (UINT64_C(1) << i) : 0; + } + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_movepi8_mask + #define _mm_movepi8_mask(a) simde_mm_movepi8_mask(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_movepi16_mask (simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_movepi16_mask(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + /* There is no 32-bit _mm_movemask_* function, so we use + * _mm_movemask_epi8 then extract the odd bits. */ + uint_fast16_t r = HEDLEY_STATIC_CAST(uint_fast16_t, simde_mm_movemask_epi8(a)); + r = ( (r >> 1)) & UINT32_C(0x5555); + r = (r | (r >> 1)) & UINT32_C(0x3333); + r = (r | (r >> 2)) & UINT32_C(0x0f0f); + r = (r | (r >> 4)) & UINT32_C(0x00ff); + return HEDLEY_STATIC_CAST(simde__mmask8, r); + #else + simde__m128i_private a_ = simde__m128i_to_private(a); + simde__mmask8 r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0])) ; i++) { + r |= (a_.i16[i] < 0) ? (UINT32_C(1) << i) : 0; + } + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_movepi16_mask + #define _mm_movepi16_mask(a) simde_mm_movepi16_mask(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_movepi32_mask (simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm_movepi32_mask(a); + #elif SIMDE_NATURAL_VECTOR_SIZE_GE(128) + return HEDLEY_STATIC_CAST(simde__mmask8, simde_mm_movemask_ps(simde_mm_castsi128_ps(a))); + #else + simde__m128i_private a_ = simde__m128i_to_private(a); + simde__mmask8 r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r |= (a_.i32[i] < 0) ? (UINT32_C(1) << i) : 0; + } + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_movepi32_mask + #define _mm_movepi32_mask(a) simde_mm_movepi32_mask(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm_movepi64_mask (simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm_movepi64_mask(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + return HEDLEY_STATIC_CAST(simde__mmask8, simde_mm_movemask_pd(simde_mm_castsi128_pd(a))); + #else + simde__m128i_private a_ = simde__m128i_to_private(a); + simde__mmask8 r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r |= (a_.i64[i] < 0) ? (UINT32_C(1) << i) : 0; + } + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_movepi64_mask + #define _mm_movepi64_mask(a) simde_mm_movepi64_mask(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm256_movepi8_mask (simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_movepi8_mask(a); + #else + simde__m256i_private a_ = simde__m256i_to_private(a); + simde__mmask32 r = 0; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(a_.m128i) / sizeof(a_.m128i[0])) ; i++) { + r |= HEDLEY_STATIC_CAST(simde__mmask32, simde_mm_movepi8_mask(a_.m128i[i])) << (i * 16); + } + #else + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i8) / sizeof(a_.i8[0])) ; i++) { + r |= (a_.i8[i] < 0) ? (UINT64_C(1) << i) : 0; + } + #endif + + return HEDLEY_STATIC_CAST(simde__mmask32, r); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_movepi8_mask + #define _mm256_movepi8_mask(a) simde_mm256_movepi8_mask(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm256_movepi16_mask (simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_movepi16_mask(a); + #else + simde__m256i_private a_ = simde__m256i_to_private(a); + simde__mmask16 r = 0; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(a_.m128i) / sizeof(a_.m128i[0])) ; i++) { + r |= HEDLEY_STATIC_CAST(simde__mmask16, simde_mm_movepi16_mask(a_.m128i[i])) << (i * 8); + } + #else + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0])) ; i++) { + r |= (a_.i16[i] < 0) ? (UINT32_C(1) << i) : 0; + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_movepi16_mask + #define _mm256_movepi16_mask(a) simde_mm256_movepi16_mask(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_movepi32_mask (simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm256_movepi32_mask(a); + #else + simde__m256i_private a_ = simde__m256i_to_private(a); + simde__mmask8 r = 0; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(a_.m128i) / sizeof(a_.m128i[0])) ; i++) { + r |= HEDLEY_STATIC_CAST(simde__mmask16, simde_mm_movepi32_mask(a_.m128i[i])) << (i * 4); + } + #else + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r |= (a_.i32[i] < 0) ? (UINT32_C(1) << i) : 0; + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm256_movepi32_mask + #define _mm256_movepi32_mask(a) simde_mm256_movepi32_mask(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_movepi64_mask (simde__m256i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm256_movepi64_mask(a); + #else + simde__m256i_private a_ = simde__m256i_to_private(a); + simde__mmask8 r = 0; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(a_.m128i) / sizeof(a_.m128i[0])) ; i++) { + r |= HEDLEY_STATIC_CAST(simde__mmask8, simde_mm_movepi64_mask(a_.m128i[i])) << (i * 2); + } + #else + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r |= (a_.i64[i] < 0) ? (UINT32_C(1) << i) : 0; + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm256_movepi64_mask + #define _mm256_movepi64_mask(a) simde_mm256_movepi64_mask(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_movepi8_mask (simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_movepi8_mask(a); + #else + simde__m512i_private a_ = simde__m512i_to_private(a); + simde__mmask64 r = 0; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(a_.m256i) / sizeof(a_.m256i[0])) ; i++) { + r |= HEDLEY_STATIC_CAST(simde__mmask64, simde_mm256_movepi8_mask(a_.m256i[i])) << (i * 32); + } + #else + r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i8) / sizeof(a_.i8[0])) ; i++) { + r |= (a_.i8[i] < 0) ? (UINT64_C(1) << i) : 0; + } + #endif + + return HEDLEY_STATIC_CAST(simde__mmask64, r); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_movepi8_mask + #define _mm512_movepi8_mask(a) simde_mm512_movepi8_mask(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_movepi16_mask (simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_movepi16_mask(a); + #else + simde__m512i_private a_ = simde__m512i_to_private(a); + simde__mmask32 r = 0; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(a_.m256i) / sizeof(a_.m256i[0])) ; i++) { + r |= HEDLEY_STATIC_CAST(simde__mmask32, simde_mm256_movepi16_mask(a_.m256i[i])) << (i * 16); + } + #else + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0])) ; i++) { + r |= (a_.i16[i] < 0) ? (UINT32_C(1) << i) : 0; + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_movepi16_mask + #define _mm512_movepi16_mask(a) simde_mm512_movepi16_mask(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_movepi32_mask (simde__m512i a) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_movepi32_mask(a); + #else + simde__m512i_private a_ = simde__m512i_to_private(a); + simde__mmask16 r = 0; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(a_.m256i) / sizeof(a_.m256i[0])) ; i++) { + r |= HEDLEY_STATIC_CAST(simde__mmask16, simde_mm256_movepi32_mask(a_.m256i[i])) << (i * 8); + } + #else + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r |= (a_.i32[i] < 0) ? (UINT32_C(1) << i) : 0; + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_movepi32_mask + #define _mm512_movepi32_mask(a) simde_mm512_movepi32_mask(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_movepi64_mask (simde__m512i a) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_movepi64_mask(a); + #else + simde__m512i_private a_ = simde__m512i_to_private(a); + simde__mmask8 r = 0; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(a_.m256i) / sizeof(a_.m256i[0])) ; i++) { + r |= simde_mm256_movepi64_mask(a_.m256i[i]) << (i * 4); + } + #else + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r |= (a_.i64[i] < 0) ? (UINT32_C(1) << i) : 0; + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_movepi64_mask + #define _mm512_movepi64_mask(a) simde_mm512_movepi64_mask(a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_MOV_MASK_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/movm.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/movm.h new file mode 100644 index 00000000..452e127a --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/movm.h @@ -0,0 +1,460 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Christopher Moore + */ + +#if !defined(SIMDE_X86_AVX512_MOVM_H) +#define SIMDE_X86_AVX512_MOVM_H + +#include "types.h" +#include "../avx2.h" +#include "set.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_movm_epi8 (simde__mmask16 k) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_movm_epi8(k); + #elif defined(SIMDE_X86_SSSE3_NATIVE) + const simde__m128i zero = simde_mm_setzero_si128(); + const simde__m128i bits = simde_mm_set_epi16(0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80); + const simde__m128i shuffle = simde_mm_set_epi8(15, 13, 11, 9, 7, 5, 3, 1, 14, 12, 10, 8, 6, 4, 2, 0); + simde__m128i r; + + r = simde_mm_set1_epi16(HEDLEY_STATIC_CAST(short, k)); + r = simde_mm_mullo_epi16(r, bits); + r = simde_mm_shuffle_epi8(r, shuffle); + r = simde_mm_cmpgt_epi8(zero, r); + + return r; + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + static const int8_t pos_data[] = { 7, 6, 5, 4, 3, 2, 1, 0 }; + int8x8_t pos = vld1_s8(pos_data); + r_.neon_i8 = vcombine_s8( + vshr_n_s8(vshl_s8(vdup_n_s8(HEDLEY_STATIC_CAST(int8_t, k)), pos), 7), + vshr_n_s8(vshl_s8(vdup_n_s8(HEDLEY_STATIC_CAST(int8_t, k >> 8)), pos), 7)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = ((k >> i) & 1) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_movm_epi8 + #define _mm_movm_epi8(k) simde_mm_movm_epi8(k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_movm_epi8 (simde__mmask32 k) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_movm_epi8(k); + #elif defined(SIMDE_X86_AVX2_NATIVE) + const simde__m256i zero = simde_mm256_setzero_si256(); + const simde__m256i bits = simde_mm256_broadcastsi128_si256(simde_mm_set_epi16(0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80)); + const simde__m256i shuffle = simde_mm256_broadcastsi128_si256(simde_mm_set_epi8(15, 13, 11, 9, 7, 5, 3, 1, 14, 12, 10, 8, 6, 4, 2, 0)); + simde__m256i r; + + r = simde_mm256_set_m128i(_mm_set1_epi16(HEDLEY_STATIC_CAST(short, k >> 16)), _mm_set1_epi16(HEDLEY_STATIC_CAST(short, k))); + r = simde_mm256_mullo_epi16(r, bits); + r = simde_mm256_shuffle_epi8(r, shuffle); + r = simde_mm256_cmpgt_epi8(zero, r); + + return r; + #else + simde__m256i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_movm_epi8(HEDLEY_STATIC_CAST(simde__mmask16, k)); + r_.m128i[1] = simde_mm_movm_epi8(HEDLEY_STATIC_CAST(simde__mmask16, k >> 16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = ((k >> i) & 1) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_movm_epi8 + #define _mm256_movm_epi8(k) simde_mm256_movm_epi8(k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_movm_epi8 (simde__mmask64 k) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_movm_epi8(k); + #else + simde__m512i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_movm_epi8(HEDLEY_STATIC_CAST(simde__mmask32, k)); + r_.m256i[1] = simde_mm256_movm_epi8(HEDLEY_STATIC_CAST(simde__mmask32, k >> 32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = ((k >> i) & 1) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_movm_epi8 + #define _mm512_movm_epi8(k) simde_mm512_movm_epi8(k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_movm_epi16 (simde__mmask8 k) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_movm_epi16(k); + #elif defined(SIMDE_X86_SSE2_NATIVE) + const simde__m128i bits = simde_mm_set_epi16(0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, INT16_MIN /* 0x8000 */); + simde__m128i r; + + r = simde_mm_set1_epi16(HEDLEY_STATIC_CAST(short, k)); + r = simde_mm_mullo_epi16(r, bits); + r = simde_mm_srai_epi16(r, 15); + + return r; + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + static const int16_t pos_data[] = { 15, 14, 13, 12, 11, 10, 9, 8 }; + const int16x8_t pos = vld1q_s16(pos_data); + r_.neon_i16 = vshrq_n_s16(vshlq_s16(vdupq_n_s16(HEDLEY_STATIC_CAST(int16_t, k)), pos), 15); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = ((k >> i) & 1) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_movm_epi16 + #define _mm_movm_epi16(k) simde_mm_movm_epi16(k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_movm_epi16 (simde__mmask16 k) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_movm_epi16(k); + #elif defined(SIMDE_X86_AVX2_NATIVE) + const __m256i bits = _mm256_set_epi16(0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, + 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, INT16_MIN /* 0x8000 */); + __m256i r; + + r = _mm256_set1_epi16(HEDLEY_STATIC_CAST(short, k)); + r = _mm256_mullo_epi16(r, bits); + r = _mm256_srai_epi16(r, 15); + + return r; + #else + simde__m256i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_movm_epi16(HEDLEY_STATIC_CAST(simde__mmask8, k)); + r_.m128i[1] = simde_mm_movm_epi16(HEDLEY_STATIC_CAST(simde__mmask8, k >> 8)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = ((k >> i) & 1) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_movm_epi16 + #define _mm256_movm_epi16(k) simde_mm256_movm_epi16(k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_movm_epi16 (simde__mmask32 k) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm512_movm_epi16(k); + #else + simde__m512i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_movm_epi16(HEDLEY_STATIC_CAST(simde__mmask16, k)); + r_.m256i[1] = simde_mm256_movm_epi16(HEDLEY_STATIC_CAST(simde__mmask16, k >> 16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = ((k >> i) & 1) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_movm_epi16 + #define _mm512_movm_epi16(k) simde_mm512_movm_epi16(k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_movm_epi32 (simde__mmask8 k) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm_movm_epi32(k); + #elif defined(SIMDE_X86_AVX2_NATIVE) + const __m128i shifts = _mm_set_epi32(28, 29, 30, 31); + __m128i r; + + r = _mm_set1_epi32(HEDLEY_STATIC_CAST(int, k)); + r = _mm_sllv_epi32(r, shifts); + r = _mm_srai_epi32(r, 31); + + return r; + #elif defined(SIMDE_X86_SSE2_NATIVE) + const simde__m128i bits = simde_mm_set_epi32(0x10000000, 0x20000000, 0x40000000, INT32_MIN /* 0x80000000 */); + simde__m128i r; + + r = simde_mm_set1_epi16(HEDLEY_STATIC_CAST(short, k)); + r = simde_mm_mullo_epi16(r, bits); + r = simde_mm_srai_epi32(r, 31); + + return r; + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + static const int32_t pos_data[] = { 31, 30, 29, 28 }; + const int32x4_t pos = vld1q_s32(pos_data); + r_.neon_i32 = vshrq_n_s32(vshlq_s32(vdupq_n_s32(HEDLEY_STATIC_CAST(int32_t, k)), pos), 31); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ((k >> i) & 1) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_movm_epi32 + #define _mm_movm_epi32(k) simde_mm_movm_epi32(k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_movm_epi32 (simde__mmask8 k) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm256_movm_epi32(k); + #elif defined(SIMDE_X86_AVX2_NATIVE) + const __m256i shifts = _mm256_set_epi32(24, 25, 26, 27, 28, 29, 30, 31); + __m256i r; + + r = _mm256_set1_epi32(HEDLEY_STATIC_CAST(int, k)); + r = _mm256_sllv_epi32(r, shifts); + r = _mm256_srai_epi32(r, 31); + + return r; + #else + simde__m256i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_movm_epi32(k ); + r_.m128i[1] = simde_mm_movm_epi32(k >> 4); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ((k >> i) & 1) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_movm_epi32 + #define _mm256_movm_epi32(k) simde_mm256_movm_epi32(k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_movm_epi32 (simde__mmask16 k) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_movm_epi32(k); + #else + simde__m512i_private r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_movm_epi32(HEDLEY_STATIC_CAST(simde__mmask8, k )); + r_.m256i[1] = simde_mm256_movm_epi32(HEDLEY_STATIC_CAST(simde__mmask8, k >> 8)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ((k >> i) & 1) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_movm_epi32 + #define _mm512_movm_epi32(k) simde_mm512_movm_epi32(k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_movm_epi64 (simde__mmask8 k) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_movm_epi64(k); + /* N.B. CM: These fallbacks may not be faster as there are only two elements */ + #elif defined(SIMDE_X86_AVX2_NATIVE) + const __m128i shifts = _mm_set_epi32(30, 30, 31, 31); + __m128i r; + + r = _mm_set1_epi32(HEDLEY_STATIC_CAST(int, k)); + r = _mm_sllv_epi32(r, shifts); + r = _mm_srai_epi32(r, 31); + + return r; + #elif defined(SIMDE_X86_SSE2_NATIVE) + const simde__m128i bits = simde_mm_set_epi32(0x40000000, 0x40000000, INT32_MIN /* 0x80000000 */, INT32_MIN /* 0x80000000 */); + simde__m128i r; + + r = simde_mm_set1_epi16(HEDLEY_STATIC_CAST(short, k)); + r = simde_mm_mullo_epi16(r, bits); + r = simde_mm_srai_epi32(r, 31); + + return r; + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + static const int64_t pos_data[] = { 63, 62 }; + const int64x2_t pos = vld1q_s64(pos_data); + r_.neon_i64 = vshrq_n_s64(vshlq_s64(vdupq_n_s64(HEDLEY_STATIC_CAST(int64_t, k)), pos), 63); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = ((k >> i) & 1) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_movm_epi64 + #define _mm_movm_epi64(k) simde_mm_movm_epi64(k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_movm_epi64 (simde__mmask8 k) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_movm_epi64(k); + #elif defined(SIMDE_X86_AVX2_NATIVE) + const __m256i shifts = _mm256_set_epi32(28, 28, 29, 29, 30, 30, 31, 31); + __m256i r; + + r = _mm256_set1_epi32(HEDLEY_STATIC_CAST(int, k)); + r = _mm256_sllv_epi32(r, shifts); + r = _mm256_srai_epi32(r, 31); + + return r; + #else + simde__m256i_private r_; + + /* N.B. CM: This fallback may not be faster as there are only four elements */ + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_movm_epi64(k ); + r_.m128i[1] = simde_mm_movm_epi64(k >> 2); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = ((k >> i) & 1) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_movm_epi64 + #define _mm256_movm_epi64(k) simde_mm256_movm_epi64(k) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_movm_epi64 (simde__mmask8 k) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_movm_epi64(k); + #else + simde__m512i_private r_; + + /* N.B. CM: Without AVX2 this fallback may not be faster as there are only eight elements */ + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_movm_epi64(k ); + r_.m256i[1] = simde_mm256_movm_epi64(k >> 4); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = ((k >> i) & 1) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_movm_epi64 + #define _mm512_movm_epi64(k) simde_mm512_movm_epi64(k) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_MOVM_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mul.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mul.h new file mode 100644 index 00000000..2ca7def7 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mul.h @@ -0,0 +1,279 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_MUL_H) +#define SIMDE_X86_AVX512_MUL_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mul_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mul_ps(a, b); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = a_.f32 * b_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_mul_ps(a_.m256[i], b_.m256[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mul_ps + #define _mm512_mul_ps(a, b) simde_mm512_mul_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_mul_ps(simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_mul_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_mul_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_mul_ps + #define _mm512_mask_mul_ps(src, k, a, b) simde_mm512_mask_mul_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_mul_ps(simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_mul_ps(k, a, b); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_mul_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_mul_ps + #define _mm512_maskz_mul_ps(k, a, b) simde_mm512_maskz_mul_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mul_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mul_pd(a, b); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = a_.f64 * b_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_mul_pd(a_.m256d[i], b_.m256d[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mul_pd + #define _mm512_mul_pd(a, b) simde_mm512_mul_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_mul_pd(simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_mul_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_mul_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_mul_pd + #define _mm512_mask_mul_pd(src, k, a, b) simde_mm512_mask_mul_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_mul_pd(simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_mul_pd(k, a, b); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_mul_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_mul_pd + #define _mm512_maskz_mul_pd(k, a, b) simde_mm512_maskz_mul_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mul_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mul_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_CONVERT_VECTOR_) && defined(SIMDE_SHUFFLE_VECTOR_) + simde__m512i_private x; + __typeof__(r_.i64) ta, tb; + + /* Get even numbered 32-bit values */ + x.i32 = SIMDE_SHUFFLE_VECTOR_(32, 64, a_.i32, b_.i32, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + /* Cast to 64 bits */ + SIMDE_CONVERT_VECTOR_(ta, x.m256i_private[0].i32); + SIMDE_CONVERT_VECTOR_(tb, x.m256i_private[1].i32); + r_.i64 = ta * tb; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = HEDLEY_STATIC_CAST(int64_t, a_.i32[i << 1]) * HEDLEY_STATIC_CAST(int64_t, b_.i32[i << 1]); + } + #endif + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mul_epi32 + #define _mm512_mul_epi32(a, b) simde_mm512_mul_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_mul_epi32(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_mul_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_mul_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_mul_epi32 + #define _mm512_mask_mul_epi32(src, k, a, b) simde_mm512_mask_mul_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_mul_epi32(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_mul_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_mul_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_mul_epi32 + #define _mm512_maskz_mul_epi32(k, a, b) simde_mm512_maskz_mul_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mul_epu32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mul_epu32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_CONVERT_VECTOR_) && defined(SIMDE_SHUFFLE_VECTOR_) + simde__m512i_private x; + __typeof__(r_.u64) ta, tb; + + x.u32 = SIMDE_SHUFFLE_VECTOR_(32, 64, a_.u32, b_.u32, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); + SIMDE_CONVERT_VECTOR_(ta, x.m256i_private[0].u32); + SIMDE_CONVERT_VECTOR_(tb, x.m256i_private[1].u32); + r_.u64 = ta * tb; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = HEDLEY_STATIC_CAST(uint64_t, a_.u32[i << 1]) * HEDLEY_STATIC_CAST(uint64_t, b_.u32[i << 1]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mul_epu32 + #define _mm512_mul_epu32(a, b) simde_mm512_mul_epu32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_mul_epu32(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_mul_epu32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_mul_epu32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_mul_epu32 + #define _mm512_mask_mul_epu32(src, k, a, b) simde_mm512_mask_mul_epu32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_mul_epu32(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_mul_epu32(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_mul_epu32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_mul_epu32 + #define _mm512_maskz_mul_epu32(k, a, b) simde_mm512_maskz_mul_epu32(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_MUL_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mulhi.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mulhi.h new file mode 100644 index 00000000..19cf81ae --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mulhi.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_MULHI_H) +#define SIMDE_X86_AVX512_MULHI_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mulhi_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mulhi_epi16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, (HEDLEY_STATIC_CAST(uint32_t, HEDLEY_STATIC_CAST(int32_t, a_.i16[i]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[i])) >> 16)); + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mulhi_epi16 + #define _mm512_mulhi_epi16(a, b) simde_mm512_mulhi_epi16(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_MULHI_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mulhrs.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mulhrs.h new file mode 100644 index 00000000..8a3b169c --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mulhrs.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_MULHRS_H) +#define SIMDE_X86_AVX512_MULHRS_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mulhrs_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mulhrs_epi16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, (((HEDLEY_STATIC_CAST(int32_t, a_.i16[i]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[i])) + 0x4000) >> 15)); + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mulhrs_epi16 + #define _mm512_mulhrs_epi16(a, b) simde_mm512_mulhrs_epi16(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_MULHRS_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mullo.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mullo.h new file mode 100644 index 00000000..f0cae059 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/mullo.h @@ -0,0 +1,169 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_MULLO_H) +#define SIMDE_X86_AVX512_MULLO_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mullo_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mullo_epi16(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, a_.i16[i] * b_.i16[i]); + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mullo_epi16 + #define _mm512_mullo_epi16(a, b) simde_mm512_mullo_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mullo_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mullo_epi32(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = HEDLEY_STATIC_CAST(int32_t, a_.i32[i] * b_.i32[i]); + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mullo_epi32 + #define _mm512_mullo_epi32(a, b) simde_mm512_mullo_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_mullo_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_mullo_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_mullo_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_mullo_epi32 + #define _mm512_mask_mullo_epi32(src, k, a, b) simde_mm512_mask_mullo_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_mullo_epi32(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_mullo_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_mullo_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_mullo_epi32 + #define _mm512_maskz_mullo_epi32(k, a, b) simde_mm512_maskz_mullo_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mullo_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_mullo_epi64(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = HEDLEY_STATIC_CAST(int64_t, a_.i64[i] * b_.i64[i]); + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mullo_epi64 + #define _mm512_mullo_epi64(a, b) simde_mm512_mullo_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_mullo_epi64(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_mask_mullo_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_mullo_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_mullo_epi64 + #define _mm512_mask_mullo_epi64(src, k, a, b) simde_mm512_mask_mullo_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_mullo_epi64(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_maskz_mullo_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_mullo_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_mullo_epi64 + #define _mm512_maskz_mullo_epi64(k, a, b) simde_mm512_maskz_mullo_epi64(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_MULLO_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/multishift.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/multishift.h new file mode 100644 index 00000000..5388d0d0 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/multishift.h @@ -0,0 +1,170 @@ +#if !defined(SIMDE_X86_AVX512_MULTISHIFT_H) +#define SIMDE_X86_AVX512_MULTISHIFT_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_multishift_epi64_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_multishift_epi64_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < sizeof(r_.u8) / sizeof(r_.u8[0]) ; i++) { + r_.u8[i] = HEDLEY_STATIC_CAST(uint8_t, (b_.u64[i / 8] >> (a_.u8[i] & 63)) | (b_.u64[i / 8] << (64 - (a_.u8[i] & 63)))); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_multishift_epi64_epi8 + #define _mm_multishift_epi64_epi8(a, b) simde_mm_multishift_epi64_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_multishift_epi64_epi8 (simde__m128i src, simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_multishift_epi64_epi8(src, k, a, b); + #else + return simde_mm_mask_mov_epi8(src, k, simde_mm_multishift_epi64_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_multishift_epi64_epi8 + #define _mm_mask_multishift_epi64_epi8(src, k, a, b) simde_mm_mask_multishift_epi64_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_multishift_epi64_epi8 (simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_multishift_epi64_epi8(k, a, b); + #else + return simde_mm_maskz_mov_epi8(k, simde_mm_multishift_epi64_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_multishift_epi64_epi8 + #define _mm_maskz_multishift_epi64_epi8(k, a, b) simde_mm_maskz_multishift_epi64_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_multishift_epi64_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_multishift_epi64_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < sizeof(r_.u8) / sizeof(r_.u8[0]) ; i++) { + r_.u8[i] = HEDLEY_STATIC_CAST(uint8_t, (b_.u64[i / 8] >> (a_.u8[i] & 63)) | (b_.u64[i / 8] << (64 - (a_.u8[i] & 63)))); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_multishift_epi64_epi8 + #define _mm256_multishift_epi64_epi8(a, b) simde_mm256_multishift_epi64_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_multishift_epi64_epi8 (simde__m256i src, simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_multishift_epi64_epi8(src, k, a, b); + #else + return simde_mm256_mask_mov_epi8(src, k, simde_mm256_multishift_epi64_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_multishift_epi64_epi8 + #define _mm256_mask_multishift_epi64_epi8(src, k, a, b) simde_mm256_mask_multishift_epi64_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_multishift_epi64_epi8 (simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_multishift_epi64_epi8(k, a, b); + #else + return simde_mm256_maskz_mov_epi8(k, simde_mm256_multishift_epi64_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_multishift_epi64_epi8 + #define _mm256_maskz_multishift_epi64_epi8(k, a, b) simde_mm256_maskz_multishift_epi64_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_multishift_epi64_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) + return _mm512_multishift_epi64_epi8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < sizeof(r_.u8) / sizeof(r_.u8[0]) ; i++) { + r_.u8[i] = HEDLEY_STATIC_CAST(uint8_t, (b_.u64[i / 8] >> (a_.u8[i] & 63)) | (b_.u64[i / 8] << (64 - (a_.u8[i] & 63)))); + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) + #undef _mm512_multishift_epi64_epi8 + #define _mm512_multishift_epi64_epi8(a, b) simde_mm512_multishift_epi64_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_multishift_epi64_epi8 (simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) + return _mm512_mask_multishift_epi64_epi8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_multishift_epi64_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_multishift_epi64_epi8 + #define _mm512_mask_multishift_epi64_epi8(src, k, a, b) simde_mm512_mask_multishift_epi64_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_multishift_epi64_epi8 (simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) + return _mm512_maskz_multishift_epi64_epi8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_multishift_epi64_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_multishift_epi64_epi8 + #define _mm512_maskz_multishift_epi64_epi8(k, a, b) simde_mm512_maskz_multishift_epi64_epi8(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_MULTISHIFT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/negate.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/negate.h new file mode 100644 index 00000000..cee4ee6b --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/negate.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + */ + +#if !defined(SIMDE_X86_AVX512_NEGATE_H) +#define SIMDE_X86_AVX512_NEGATE_H + +#include "types.h" +#include "mov.h" +#include "xor.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_x_mm512_negate_ps(simde__m512 a) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return simde_mm512_xor_ps(a,_mm512_set1_ps(SIMDE_FLOAT32_C(-0.0))); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if defined(SIMDE_VECTOR_NEGATE) + r_.f32 = -a_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = -a_.f32[i]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_x_mm512_negate_pd(simde__m512d a) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return simde_mm512_xor_pd(a, _mm512_set1_pd(SIMDE_FLOAT64_C(-0.0))); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if defined(SIMDE_VECTOR_NEGATE) + r_.f64 = -a_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = -a_.f64[i]; + } + #endif + + return simde__m512d_from_private(r_); + #endif +} + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_NEGATE_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/or.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/or.h new file mode 100644 index 00000000..b8516a42 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/or.h @@ -0,0 +1,308 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_OR_H) +#define SIMDE_X86_AVX512_OR_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_or_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_or_ps(a, b); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256[0] = simde_mm256_or_ps(a_.m256[0], b_.m256[0]); + r_.m256[1] = simde_mm256_or_ps(a_.m256[1], b_.m256[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_CLANG_BAD_VI64_OPS) + r_.i32f = a_.i32f | b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] | b_.i32f[i]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_or_ps + #define _mm512_or_ps(a, b) simde_mm512_or_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_or_ps(simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_mask_or_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_or_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_or_ps + #define _mm512_mask_or_ps(src, k, a, b) simde_mm512_mask_or_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_or_ps(simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_maskz_or_ps(k, a, b); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_or_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_or_ps + #define _mm512_maskz_or_ps(k, a, b) simde_mm512_maskz_or_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_or_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_or_pd(a, b); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256d[0] = simde_mm256_or_pd(a_.m256d[0], b_.m256d[0]); + r_.m256d[1] = simde_mm256_or_pd(a_.m256d[1], b_.m256d[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_CLANG_BAD_VI64_OPS) + r_.i32f = a_.i32f | b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] | b_.i32f[i]; + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_or_pd + #define _mm512_or_pd(a, b) simde_mm512_or_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_or_pd(simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_mask_or_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_or_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_or_pd + #define _mm512_mask_or_pd(src, k, a, b) simde_mm512_mask_or_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_or_pd(simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_maskz_or_pd(k, a, b); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_or_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_or_pd + #define _mm512_maskz_or_pd(k, a, b) simde_mm512_maskz_or_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_or_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_or_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 | b_.i32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] | b_.i32[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_or_epi32 + #define _mm512_or_epi32(a, b) simde_mm512_or_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_or_epi32(simde__m512i src, simde__mmask16 k, simde__m512i v2, simde__m512i v3) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_or_epi32(src, k, v2, v3); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_or_epi32(v2, v3)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_or_epi32 + #define _mm512_mask_or_epi32(src, k, v2, v3) simde_mm512_mask_or_epi32(src, k, v2, v3) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_or_epi32(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_or_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_or_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_or_epi32 + #define _mm512_maskz_or_epi32(k, a, b) simde_mm512_maskz_or_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_or_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_or_epi64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_or_si256(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_CLANG_BAD_VI64_OPS) + r_.i32f = a_.i32f | b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] | b_.i32f[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_or_epi64 + #define _mm512_or_epi64(a, b) simde_mm512_or_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_or_epi64(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_or_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_or_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_or_epi64 + #define _mm512_mask_or_epi64(src, k, a, b) simde_mm512_mask_or_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_or_epi64(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_or_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_or_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_or_epi64 + #define _mm512_maskz_or_epi64(k, a, b) simde_mm512_maskz_or_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_or_si512 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_or_si512(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256i[0] = simde_mm256_or_si256(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_or_si256(a_.m256i[1], b_.m256i[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f | b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] | b_.i32f[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_or_si512 + #define _mm512_or_si512(a, b) simde_mm512_or_si512(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_OR_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/packs.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/packs.h new file mode 100644 index 00000000..55a39c66 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/packs.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_PACKS_H) +#define SIMDE_X86_AVX512_PACKS_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_packs_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_packs_epi16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_packs_epi16(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_packs_epi16(a_.m256i[1], b_.m256i[1]); + #else + const size_t halfway_point = (sizeof(r_.i8) / sizeof(r_.i8[0])) / 2; + const size_t quarter_point = (sizeof(r_.i8) / sizeof(r_.i8[0])) / 4; + const size_t octet_point = (sizeof(r_.i8) / sizeof(r_.i8[0])) / 8; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < octet_point ; i++) { + r_.i8[i] = (a_.i16[i] > INT8_MAX) ? INT8_MAX : ((a_.i16[i] < INT8_MIN) ? INT8_MIN : HEDLEY_STATIC_CAST(int8_t, a_.i16[i])); + r_.i8[i + octet_point] = (b_.i16[i] > INT8_MAX) ? INT8_MAX : ((b_.i16[i] < INT8_MIN) ? INT8_MIN : HEDLEY_STATIC_CAST(int8_t, b_.i16[i])); + r_.i8[quarter_point + i] = (a_.i16[octet_point + i] > INT8_MAX) ? INT8_MAX : ((a_.i16[octet_point + i] < INT8_MIN) ? INT8_MIN : HEDLEY_STATIC_CAST(int8_t, a_.i16[octet_point + i])); + r_.i8[quarter_point + i + octet_point] = (b_.i16[octet_point + i] > INT8_MAX) ? INT8_MAX : ((b_.i16[octet_point + i] < INT8_MIN) ? INT8_MIN : HEDLEY_STATIC_CAST(int8_t, b_.i16[octet_point + i])); + r_.i8[halfway_point + i] = (a_.i16[quarter_point + i] > INT8_MAX) ? INT8_MAX : ((a_.i16[quarter_point + i] < INT8_MIN) ? INT8_MIN : HEDLEY_STATIC_CAST(int8_t, a_.i16[quarter_point + i])); + r_.i8[halfway_point + i + octet_point] = (b_.i16[quarter_point + i] > INT8_MAX) ? INT8_MAX : ((b_.i16[quarter_point + i] < INT8_MIN) ? INT8_MIN : HEDLEY_STATIC_CAST(int8_t, b_.i16[quarter_point + i])); + r_.i8[halfway_point + quarter_point + i] = (a_.i16[quarter_point + octet_point + i] > INT8_MAX) ? INT8_MAX : ((a_.i16[quarter_point + octet_point + i] < INT8_MIN) ? INT8_MIN : HEDLEY_STATIC_CAST(int8_t, a_.i16[quarter_point + octet_point + i])); + r_.i8[halfway_point + quarter_point + i + octet_point] = (b_.i16[quarter_point + octet_point + i] > INT8_MAX) ? INT8_MAX : ((b_.i16[quarter_point + octet_point + i] < INT8_MIN) ? INT8_MIN : HEDLEY_STATIC_CAST(int8_t, b_.i16[quarter_point + octet_point + i])); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_packs_epi16 + #define _mm512_packs_epi16(a, b) simde_mm512_packs_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_packs_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_packs_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_packs_epi32(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_packs_epi32(a_.m256i[1], b_.m256i[1]); + #else + const size_t halfway_point = (sizeof(r_.i16) / sizeof(r_.i16[0])) / 2; + const size_t quarter_point = (sizeof(r_.i16) / sizeof(r_.i16[0])) / 4; + const size_t octet_point = (sizeof(r_.i16) / sizeof(r_.i16[0])) / 8; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < octet_point ; i++) { + r_.i16[i] = (a_.i32[i] > INT16_MAX) ? INT16_MAX : ((a_.i32[i] < INT16_MIN) ? INT16_MIN : HEDLEY_STATIC_CAST(int16_t, a_.i32[i])); + r_.i16[i + octet_point] = (b_.i32[i] > INT16_MAX) ? INT16_MAX : ((b_.i32[i] < INT16_MIN) ? INT16_MIN : HEDLEY_STATIC_CAST(int16_t, b_.i32[i])); + r_.i16[quarter_point + i] = (a_.i32[octet_point + i] > INT16_MAX) ? INT16_MAX : ((a_.i32[octet_point + i] < INT16_MIN) ? INT16_MIN : HEDLEY_STATIC_CAST(int16_t, a_.i32[octet_point + i])); + r_.i16[quarter_point + i + octet_point] = (b_.i32[octet_point + i] > INT16_MAX) ? INT16_MAX : ((b_.i32[octet_point + i] < INT16_MIN) ? INT16_MIN : HEDLEY_STATIC_CAST(int16_t, b_.i32[octet_point + i])); + r_.i16[halfway_point + i] = (a_.i32[quarter_point + i] > INT16_MAX) ? INT16_MAX : ((a_.i32[quarter_point +i] < INT16_MIN) ? INT16_MIN : HEDLEY_STATIC_CAST(int16_t, a_.i32[quarter_point + i])); + r_.i16[halfway_point + i + octet_point] = (b_.i32[quarter_point + i] > INT16_MAX) ? INT16_MAX : ((b_.i32[quarter_point + i] < INT16_MIN) ? INT16_MIN : HEDLEY_STATIC_CAST(int16_t, b_.i32[quarter_point +i])); + r_.i16[halfway_point + quarter_point + i] = (a_.i32[quarter_point + octet_point + i] > INT16_MAX) ? INT16_MAX : ((a_.i32[quarter_point + octet_point + i] < INT16_MIN) ? INT16_MIN : HEDLEY_STATIC_CAST(int16_t, a_.i32[quarter_point + octet_point + i])); + r_.i16[halfway_point + quarter_point + i + octet_point] = (b_.i32[quarter_point + octet_point + i] > INT16_MAX) ? INT16_MAX : ((b_.i32[quarter_point + octet_point + i] < INT16_MIN) ? INT16_MIN : HEDLEY_STATIC_CAST(int16_t, b_.i32[quarter_point + octet_point + i])); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_packs_epi32 + #define _mm512_packs_epi32(a, b) simde_mm512_packs_epi32(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_PACKS_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/packus.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/packus.h new file mode 100644 index 00000000..3da3b83e --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/packus.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_PACKUS_H) +#define SIMDE_X86_AVX512_PACKUS_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_packus_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_packus_epi16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_packus_epi16(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_packus_epi16(a_.m256i[1], b_.m256i[1]); + #else + const size_t halfway_point = (sizeof(r_.i8) / sizeof(r_.i8[0])) / 2; + const size_t quarter_point = (sizeof(r_.i8) / sizeof(r_.i8[0])) / 4; + const size_t octet_point = (sizeof(r_.i8) / sizeof(r_.i8[0])) / 8; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < octet_point ; i++) { + r_.u8[i] = (a_.i16[i] > UINT8_MAX) ? UINT8_MAX : ((a_.i16[i] < 0) ? UINT8_C(0) : HEDLEY_STATIC_CAST(uint8_t, a_.i16[i])); + r_.u8[i + octet_point] = (b_.i16[i] > UINT8_MAX) ? UINT8_MAX : ((b_.i16[i] < 0) ? UINT8_C(0) : HEDLEY_STATIC_CAST(uint8_t, b_.i16[i])); + r_.u8[quarter_point + i] = (a_.i16[octet_point + i] > UINT8_MAX) ? UINT8_MAX : ((a_.i16[octet_point + i] < 0) ? UINT8_C(0) : HEDLEY_STATIC_CAST(uint8_t, a_.i16[octet_point + i])); + r_.u8[quarter_point + i + octet_point] = (b_.i16[octet_point + i] > UINT8_MAX) ? UINT8_MAX : ((b_.i16[octet_point + i] < 0) ? UINT8_C(0) : HEDLEY_STATIC_CAST(uint8_t, b_.i16[octet_point + i])); + r_.u8[halfway_point + i] = (a_.i16[quarter_point + i] > UINT8_MAX) ? UINT8_MAX : ((a_.i16[quarter_point + i] < 0) ? UINT8_C(0) : HEDLEY_STATIC_CAST(uint8_t, a_.i16[quarter_point + i])); + r_.u8[halfway_point + i + octet_point] = (b_.i16[quarter_point + i] > UINT8_MAX) ? UINT8_MAX : ((b_.i16[quarter_point + i] < 0) ? UINT8_C(0) : HEDLEY_STATIC_CAST(uint8_t, b_.i16[quarter_point + i])); + r_.u8[halfway_point + quarter_point + i] = (a_.i16[quarter_point + octet_point + i] > UINT8_MAX) ? UINT8_MAX : ((a_.i16[quarter_point + octet_point + i] < 0) ? UINT8_C(0) : HEDLEY_STATIC_CAST(uint8_t, a_.i16[quarter_point + octet_point + i])); + r_.u8[halfway_point + quarter_point + i + octet_point] = (b_.i16[quarter_point + octet_point + i] > UINT8_MAX) ? UINT8_MAX : ((b_.i16[quarter_point + octet_point + i] < 0) ? UINT8_C(0) : HEDLEY_STATIC_CAST(uint8_t, b_.i16[quarter_point + octet_point + i])); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_packus_epi16 + #define _mm512_packus_epi16(a, b) simde_mm512_packus_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_packus_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_packus_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(a_.m256i) / sizeof(a_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_packus_epi32(a_.m256i[i], b_.m256i[i]); + } + #else + const size_t halfway_point = (sizeof(r_.i16) / sizeof(r_.i16[0])) / 2; + const size_t quarter_point = (sizeof(r_.i16) / sizeof(r_.i16[0])) / 4; + const size_t octet_point = (sizeof(r_.i16) / sizeof(r_.i16[0])) / 8; + SIMDE_VECTORIZE + for (size_t i = 0 ; i < octet_point ; i++) { + r_.u16[i] = (a_.i32[i] > UINT16_MAX) ? UINT16_MAX : ((a_.i32[i] < 0) ? UINT16_C(0) : HEDLEY_STATIC_CAST(uint16_t, a_.i32[i])); + r_.u16[i + octet_point] = (b_.i32[i] > UINT16_MAX) ? UINT16_MAX : ((b_.i32[i] < 0) ? UINT16_C(0) : HEDLEY_STATIC_CAST(uint16_t, b_.i32[i])); + r_.u16[quarter_point + i] = (a_.i32[octet_point + i] > UINT16_MAX) ? UINT16_MAX : ((a_.i32[octet_point + i] < 0) ? UINT16_C(0) : HEDLEY_STATIC_CAST(uint16_t, a_.i32[octet_point + i])); + r_.u16[quarter_point + i + octet_point] = (b_.i32[octet_point + i] > UINT16_MAX) ? UINT16_MAX : ((b_.i32[octet_point + i] < 0) ? UINT16_C(0) : HEDLEY_STATIC_CAST(uint16_t, b_.i32[octet_point + i])); + r_.u16[halfway_point + i] = (a_.i32[quarter_point + i] > UINT16_MAX) ? UINT16_MAX : ((a_.i32[quarter_point +i] < 0) ? UINT16_C(0) : HEDLEY_STATIC_CAST(uint16_t, a_.i32[quarter_point + i])); + r_.u16[halfway_point + i + octet_point] = (b_.i32[quarter_point + i] > UINT16_MAX) ? UINT16_MAX : ((b_.i32[quarter_point + i] < 0) ? UINT16_C(0) : HEDLEY_STATIC_CAST(uint16_t, b_.i32[quarter_point +i])); + r_.u16[halfway_point + quarter_point + i] = (a_.i32[quarter_point + octet_point + i] > UINT16_MAX) ? UINT16_MAX : ((a_.i32[quarter_point + octet_point + i] < 0) ? UINT16_C(0) : HEDLEY_STATIC_CAST(uint16_t, a_.i32[quarter_point + octet_point + i])); + r_.u16[halfway_point + quarter_point + i + octet_point] = (b_.i32[quarter_point + octet_point + i] > UINT16_MAX) ? UINT16_MAX : ((b_.i32[quarter_point + octet_point + i] < 0) ? UINT16_C(0) : HEDLEY_STATIC_CAST(uint16_t, b_.i32[quarter_point + octet_point + i])); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_packus_epi32 + #define _mm512_packus_epi32(a, b) simde_mm512_packus_epi32(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_PACKUS_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/permutex.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/permutex.h new file mode 100644 index 00000000..91c35cc2 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/permutex.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2023 Michael R. Crusoe + */ + +#if !defined(SIMDE_X86_AVX512_PERMUTEX_H) +#define SIMDE_X86_AVX512_PERMUTEX_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_permutex_epi64 (simde__m256i a, const int imm8) { + simde__m256i_private + a_ = simde__m256i_to_private(a), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[(imm8 >> (i*2)) & 3]; + } + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_permutex_epi64(a, imm8) _mm256_permutex_epi64((a), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutex_epi64 + #define _mm256_permutex_epi64(a, imm8) simde_mm256_permutex_epi64((a), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_permutex_epi64 (simde__m512i a, const int imm8) { + simde__m512i_private + a_ = simde__m512i_to_private(a), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i_private[0].i64) / sizeof(r_.m256i_private[0].i64[0])) ; i++) { + r_.m256i_private[0].i64[i] = a_.m256i_private[0].i64[(imm8 >> (i*2)) & 3]; + r_.m256i_private[1].i64[i] = a_.m256i_private[1].i64[(imm8 >> (i*2)) & 3]; + } + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_permutex_epi64(a, imm8) _mm512_permutex_epi64((a), (imm8)) +#elif defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_permutex_epi64(a, imm8) SIMDE_STATEMENT_EXPR_(({\ + simde__m512i_private simde_mm512_permutex_epi64_a_ = simde__m512i_to_private((a)), simde_mm512_permutex_epi64_r_; \ + simde_mm512_permutex_epi64_r_.m256i[0] = simde_mm256_permutex_epi64(simde_mm512_permutex_epi64_a_.m256i[0], (imm8)); \ + simde_mm512_permutex_epi64_r_.m256i[1] = simde_mm256_permutex_epi64(simde_mm512_permutex_epi64_a_.m256i[1], (imm8)); \ + simde__m512i_from_private(simde_mm512_permutex_epi64_r_); \ + })) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_permutex_epi64 + #define _mm512_permutex_epi64(a, imm8) simde_mm512_permutex_epi64((a), (imm8)) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_permutex_epi64(src, k, a, imm8) _mm512_mask_permutex_epi64((src), (k), (a), (imm8)) +#else + #define simde_mm512_mask_permutex_epi64(src, k, a, imm8) simde_mm512_mask_mov_epi64((src), (k), simde_mm512_permutex_epi64((a), (imm8))) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_permutex_epi64 + #define _mm512_mask_permutex_epi64(src, k, a, imm8) simde_mm512_mask_permutex_epi64((src), (k), (a), (imm8)) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_PERMUTEX_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/permutex2var.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/permutex2var.h new file mode 100644 index 00000000..b6480c20 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/permutex2var.h @@ -0,0 +1,1645 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Christopher Moore + */ + +#if !defined(SIMDE_X86_AVX512_PERMUTEX2VAR_H) +#define SIMDE_X86_AVX512_PERMUTEX2VAR_H + +#include "types.h" +#include "and.h" +#include "andnot.h" +#include "blend.h" +#include "mov.h" +#include "or.h" +#include "set1.h" +#include "slli.h" +#include "srli.h" +#include "test.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +/* The following generic code avoids many, nearly identical, repetitions of fairly complex code. + * If the compiler optimizes well, in particular extracting invariant code from loops + * and simplifying code involving constants passed as arguments, it should not be + * significantly slower than specific code. + * Note that when the original vector contains few elements, these implementations + * may not be faster than portable code. + */ +#if defined(SIMDE_X86_SSSE3_NATIVE) || defined(SIMDE_ARM_NEON_A64V8_NATIVE) || defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_WASM_SIMD128_NATIVE) + #define SIMDE_X_PERMUTEX2VAR_USE_GENERIC +#endif + +#if defined(SIMDE_X_PERMUTEX2VAR_USE_GENERIC) +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_permutex2var128 (const simde__m128i *a, const simde__m128i idx, const simde__m128i *b, const unsigned int log2_index_size, const unsigned int log2_data_length) { + const int idx_mask = (1 << (5 - log2_index_size + log2_data_length)) - 1; + + #if defined(SIMDE_X86_SSE3_NATIVE) + __m128i ra, rb, t, test, select, index; + const __m128i sixteen = _mm_set1_epi8(16); + + /* Avoid the mullo intrinsics which have high latency (and the 32-bit one requires SSE4.1) */ + switch (log2_index_size) { + default: /* Avoid uninitialized variable warning/error */ + case 0: + index = _mm_and_si128(idx, _mm_set1_epi8(HEDLEY_STATIC_CAST(int8_t, idx_mask))); + break; + case 1: + index = _mm_and_si128(idx, _mm_set1_epi16(HEDLEY_STATIC_CAST(int16_t, idx_mask))); + index = _mm_slli_epi32(index, 1); + t = _mm_slli_epi32(index, 8); + index = _mm_or_si128(index, t); + index = _mm_add_epi16(index, _mm_set1_epi16(0x0100)); + break; + case 2: + index = _mm_and_si128(idx, _mm_set1_epi32(HEDLEY_STATIC_CAST(int32_t, idx_mask))); + index = _mm_slli_epi32(index, 2); + t = _mm_slli_epi32(index, 8); + index = _mm_or_si128(index, t); + t = _mm_slli_epi32(index, 16); + index = _mm_or_si128(index, t); + index = _mm_add_epi32(index, _mm_set1_epi32(0x03020100)); + break; + } + + test = index; + index = _mm_and_si128(index, _mm_set1_epi8(HEDLEY_STATIC_CAST(int8_t, (1 << (4 + log2_data_length)) - 1))); + test = _mm_cmpgt_epi8(test, index); + + ra = _mm_shuffle_epi8(a[0], index); + rb = _mm_shuffle_epi8(b[0], index); + + #if defined(SIMDE_X86_SSE4_1_NATIVE) + SIMDE_VECTORIZE + for (int i = 1 ; i < (1 << log2_data_length) ; i++) { + select = _mm_cmplt_epi8(index, sixteen); + index = _mm_sub_epi8(index, sixteen); + ra = _mm_blendv_epi8(_mm_shuffle_epi8(a[i], index), ra, select); + rb = _mm_blendv_epi8(_mm_shuffle_epi8(b[i], index), rb, select); + } + + return _mm_blendv_epi8(ra, rb, test); + #else + SIMDE_VECTORIZE + for (int i = 1 ; i < (1 << log2_data_length) ; i++) { + select = _mm_cmplt_epi8(index, sixteen); + index = _mm_sub_epi8(index, sixteen); + ra = _mm_or_si128(_mm_andnot_si128(select, _mm_shuffle_epi8(a[i], index)), _mm_and_si128(select, ra)); + rb = _mm_or_si128(_mm_andnot_si128(select, _mm_shuffle_epi8(b[i], index)), _mm_and_si128(select, rb)); + } + + return _mm_or_si128(_mm_andnot_si128(test, ra), _mm_and_si128(test, rb)); + #endif + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint8x16_t index, r; + uint16x8_t index16; + uint32x4_t index32; + uint8x16x2_t table2_a, table2_b; + uint8x16x4_t table4_a, table4_b; + + switch (log2_index_size) { + case 0: + index = vandq_u8(simde__m128i_to_neon_u8(idx), vdupq_n_u8(HEDLEY_STATIC_CAST(uint8_t, idx_mask))); + break; + case 1: + index16 = vandq_u16(simde__m128i_to_neon_u16(idx), vdupq_n_u16(HEDLEY_STATIC_CAST(uint16_t, idx_mask))); + index16 = vmulq_n_u16(index16, 0x0202); + index16 = vaddq_u16(index16, vdupq_n_u16(0x0100)); + index = vreinterpretq_u8_u16(index16); + break; + case 2: + index32 = vandq_u32(simde__m128i_to_neon_u32(idx), vdupq_n_u32(HEDLEY_STATIC_CAST(uint32_t, idx_mask))); + index32 = vmulq_n_u32(index32, 0x04040404); + index32 = vaddq_u32(index32, vdupq_n_u32(0x03020100)); + index = vreinterpretq_u8_u32(index32); + break; + } + + uint8x16_t mask = vdupq_n_u8(HEDLEY_STATIC_CAST(uint8_t, (1 << (4 + log2_data_length)) - 1)); + + switch (log2_data_length) { + case 0: + r = vqtbx1q_u8(vqtbl1q_u8(simde__m128i_to_neon_u8(b[0]), vandq_u8(index, mask)), simde__m128i_to_neon_u8(a[0]), index); + break; + case 1: + table2_a.val[0] = simde__m128i_to_neon_u8(a[0]); + table2_a.val[1] = simde__m128i_to_neon_u8(a[1]); + table2_b.val[0] = simde__m128i_to_neon_u8(b[0]); + table2_b.val[1] = simde__m128i_to_neon_u8(b[1]); + r = vqtbx2q_u8(vqtbl2q_u8(table2_b, vandq_u8(index, mask)), table2_a, index); + break; + case 2: + table4_a.val[0] = simde__m128i_to_neon_u8(a[0]); + table4_a.val[1] = simde__m128i_to_neon_u8(a[1]); + table4_a.val[2] = simde__m128i_to_neon_u8(a[2]); + table4_a.val[3] = simde__m128i_to_neon_u8(a[3]); + table4_b.val[0] = simde__m128i_to_neon_u8(b[0]); + table4_b.val[1] = simde__m128i_to_neon_u8(b[1]); + table4_b.val[2] = simde__m128i_to_neon_u8(b[2]); + table4_b.val[3] = simde__m128i_to_neon_u8(b[3]); + r = vqtbx4q_u8(vqtbl4q_u8(table4_b, vandq_u8(index, mask)), table4_a, index); + break; + } + + return simde__m128i_from_neon_u8(r); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) r, ra, rb, t, index, s, thirty_two = vec_splats(HEDLEY_STATIC_CAST(uint8_t, 32)); + SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) index16; + SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) temp32, index32; + SIMDE_POWER_ALTIVEC_VECTOR(SIMDE_POWER_ALTIVEC_BOOL char) select, test; + + switch (log2_index_size) { + default: /* Avoid uninitialized variable warning/error */ + case 0: + index = vec_and(simde__m128i_to_altivec_u8(idx), vec_splats(HEDLEY_STATIC_CAST(uint8_t, idx_mask))); + break; + case 1: + index16 = simde__m128i_to_altivec_u16(idx); + index16 = vec_and(index16, vec_splats(HEDLEY_STATIC_CAST(uint16_t, idx_mask))); + index16 = vec_mladd(index16, vec_splats(HEDLEY_STATIC_CAST(unsigned short, 0x0202)), vec_splats(HEDLEY_STATIC_CAST(unsigned short, 0x0100))); + index = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), index16); + break; + case 2: + index32 = simde__m128i_to_altivec_u32(idx); + index32 = vec_and(index32, vec_splats(HEDLEY_STATIC_CAST(uint32_t, idx_mask))); + + /* Multiply index32 by 0x04040404; unfortunately vec_mul isn't available so (mis)use 16-bit vec_mladd */ + temp32 = vec_sl(index32, vec_splats(HEDLEY_STATIC_CAST(unsigned int, 16))); + index32 = vec_add(index32, temp32); + index32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned int), + vec_mladd(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned short), index32), + vec_splats(HEDLEY_STATIC_CAST(unsigned short, 0x0404)), + vec_splat_u16(0))); + + index32 = vec_add(index32, vec_splats(HEDLEY_STATIC_CAST(unsigned int, 0x03020100))); + index = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), index32); + break; + } + + if (log2_data_length == 0) { + r = vec_perm(simde__m128i_to_altivec_u8(a[0]), simde__m128i_to_altivec_u8(b[0]), HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), index)); + } + else { + s = index; + index = vec_and(index, vec_splats(HEDLEY_STATIC_CAST(uint8_t, (1 << (4 + log2_data_length)) - 1))); + test = vec_cmpgt(s, index); + + ra = vec_perm(simde__m128i_to_altivec_u8(a[0]), simde__m128i_to_altivec_u8(a[1]), index); + rb = vec_perm(simde__m128i_to_altivec_u8(b[0]), simde__m128i_to_altivec_u8(b[1]), index); + + SIMDE_VECTORIZE + for (int i = 2 ; i < (1 << log2_data_length) ; i += 2) { + select = vec_cmplt(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed char), index), + HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed char), thirty_two)); + index = vec_sub(index, thirty_two); + t = vec_perm(simde__m128i_to_altivec_u8(a[i]), simde__m128i_to_altivec_u8(a[i + 1]), index); + ra = vec_sel(t, ra, select); + t = vec_perm(simde__m128i_to_altivec_u8(b[i]), simde__m128i_to_altivec_u8(b[i + 1]), index); + rb = vec_sel(t, rb, select); + } + + r = vec_sel(ra, rb, test); + } + + return simde__m128i_from_altivec_u8(r); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + const v128_t sixteen = wasm_i8x16_splat(16); + + v128_t index = simde__m128i_to_wasm_v128(idx); + + switch (log2_index_size) { + case 0: + index = wasm_v128_and(index, wasm_i8x16_splat(HEDLEY_STATIC_CAST(int8_t, idx_mask))); + break; + case 1: + index = wasm_v128_and(index, wasm_i16x8_splat(HEDLEY_STATIC_CAST(int16_t, idx_mask))); + index = wasm_i16x8_mul(index, wasm_i16x8_splat(0x0202)); + index = wasm_i16x8_add(index, wasm_i16x8_splat(0x0100)); + break; + case 2: + index = wasm_v128_and(index, wasm_i32x4_splat(HEDLEY_STATIC_CAST(int32_t, idx_mask))); + index = wasm_i32x4_mul(index, wasm_i32x4_splat(0x04040404)); + index = wasm_i32x4_add(index, wasm_i32x4_splat(0x03020100)); + break; + } + + v128_t r = wasm_i8x16_swizzle(simde__m128i_to_wasm_v128(a[0]), index); + + SIMDE_VECTORIZE + for (int i = 1 ; i < (1 << log2_data_length) ; i++) { + index = wasm_i8x16_sub(index, sixteen); + r = wasm_v128_or(r, wasm_i8x16_swizzle(simde__m128i_to_wasm_v128(a[i]), index)); + } + + SIMDE_VECTORIZE + for (int i = 0 ; i < (1 << log2_data_length) ; i++) { + index = wasm_i8x16_sub(index, sixteen); + r = wasm_v128_or(r, wasm_i8x16_swizzle(simde__m128i_to_wasm_v128(b[i]), index)); + } + + return simde__m128i_from_wasm_v128(r); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_x_permutex2var (simde__m128i *r, const simde__m128i *a, const simde__m128i *idx, const simde__m128i *b, const unsigned int log2_index_size, const unsigned int log2_data_length) { + SIMDE_VECTORIZE + for (int i = 0 ; i < (1 << log2_data_length) ; i++) { + r[i] = simde_x_permutex2var128(a, idx[i], b, log2_index_size, log2_data_length); + } +} +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_permutex2var_epi16 (simde__m128i a, simde__m128i idx, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_permutex2var_epi16(a, idx, b); + #elif defined(SIMDE_X_PERMUTEX2VAR_USE_GENERIC) + simde__m128i r; + + simde_x_permutex2var(&r, &a, &idx, &b, 1, 0); + + return r; + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + idx_ = simde__m128i_to_private(idx), + b_ = simde__m128i_to_private(b), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = ((idx_.i16[i] & 8) ? b_ : a_).i16[idx_.i16[i] & 7]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_permutex2var_epi16 + #define _mm_permutex2var_epi16(a, idx, b) simde_mm_permutex2var_epi16(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_permutex2var_epi16 (simde__m128i a, simde__mmask8 k, simde__m128i idx, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_permutex2var_epi16(a, k, idx, b); + #else + return simde_mm_mask_mov_epi16(a, k, simde_mm_permutex2var_epi16(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_permutex2var_epi16 +#define _mm_mask_permutex2var_epi16(a, k, idx, b) simde_mm_mask_permutex2var_epi16(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask2_permutex2var_epi16 (simde__m128i a, simde__m128i idx, simde__mmask8 k, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask2_permutex2var_epi16(a, idx, k, b); + #else + return simde_mm_mask_mov_epi16(idx, k, simde_mm_permutex2var_epi16(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask2_permutex2var_epi16 +#define _mm_mask2_permutex2var_epi16(a, idx, k, b) simde_mm_mask2_permutex2var_epi16(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_permutex2var_epi16 (simde__mmask8 k, simde__m128i a, simde__m128i idx, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_permutex2var_epi16(k, a, idx, b); + #else + return simde_mm_maskz_mov_epi16(k, simde_mm_permutex2var_epi16(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_permutex2var_epi16 +#define _mm_maskz_permutex2var_epi16(k, a, idx, b) simde_mm_maskz_permutex2var_epi16(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_permutex2var_epi32 (simde__m128i a, simde__m128i idx, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_permutex2var_epi32(a, idx, b); + #elif defined(SIMDE_X_PERMUTEX2VAR_USE_GENERIC) /* This may not be faster than the portable version */ + simde__m128i r; + + simde_x_permutex2var(&r, &a, &idx, &b, 2, 0); + + return r; + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + idx_ = simde__m128i_to_private(idx), + b_ = simde__m128i_to_private(b), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ((idx_.i32[i] & 4) ? b_ : a_).i32[idx_.i32[i] & 3]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_permutex2var_epi32 + #define _mm_permutex2var_epi32(a, idx, b) simde_mm_permutex2var_epi32(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_permutex2var_epi32 (simde__m128i a, simde__mmask8 k, simde__m128i idx, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_permutex2var_epi32(a, k, idx, b); + #else + return simde_mm_mask_mov_epi32(a, k, simde_mm_permutex2var_epi32(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_permutex2var_epi32 +#define _mm_mask_permutex2var_epi32(a, k, idx, b) simde_mm_mask_permutex2var_epi32(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask2_permutex2var_epi32 (simde__m128i a, simde__m128i idx, simde__mmask8 k, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask2_permutex2var_epi32(a, idx, k, b); + #else + return simde_mm_mask_mov_epi32(idx, k, simde_mm_permutex2var_epi32(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask2_permutex2var_epi32 +#define _mm_mask2_permutex2var_epi32(a, idx, k, b) simde_mm_mask2_permutex2var_epi32(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_permutex2var_epi32 (simde__mmask8 k, simde__m128i a, simde__m128i idx, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_permutex2var_epi32(k, a, idx, b); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_permutex2var_epi32(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_permutex2var_epi32 +#define _mm_maskz_permutex2var_epi32(k, a, idx, b) simde_mm_maskz_permutex2var_epi32(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_permutex2var_epi64 (simde__m128i a, simde__m128i idx, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_permutex2var_epi64(a, idx, b); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + idx_ = simde__m128i_to_private(idx), + b_ = simde__m128i_to_private(b), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = ((idx_.i64[i] & 2) ? b_ : a_).i64[idx_.i64[i] & 1]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_permutex2var_epi64 + #define _mm_permutex2var_epi64(a, idx, b) simde_mm_permutex2var_epi64(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_permutex2var_epi64 (simde__m128i a, simde__mmask8 k, simde__m128i idx, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_permutex2var_epi64(a, k, idx, b); + #else + return simde_mm_mask_mov_epi64(a, k, simde_mm_permutex2var_epi64(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_permutex2var_epi64 +#define _mm_mask_permutex2var_epi64(a, k, idx, b) simde_mm_mask_permutex2var_epi64(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask2_permutex2var_epi64 (simde__m128i a, simde__m128i idx, simde__mmask8 k, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask2_permutex2var_epi64(a, idx, k, b); + #else + return simde_mm_mask_mov_epi64(idx, k, simde_mm_permutex2var_epi64(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask2_permutex2var_epi64 +#define _mm_mask2_permutex2var_epi64(a, idx, k, b) simde_mm_mask2_permutex2var_epi64(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_permutex2var_epi64 (simde__mmask8 k, simde__m128i a, simde__m128i idx, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_permutex2var_epi64(k, a, idx, b); + #else + return simde_mm_maskz_mov_epi64(k, simde_mm_permutex2var_epi64(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_permutex2var_epi64 +#define _mm_maskz_permutex2var_epi64(k, a, idx, b) simde_mm_maskz_permutex2var_epi64(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_permutex2var_epi8 (simde__m128i a, simde__m128i idx, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_permutex2var_epi8(a, idx, b); + #elif defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cvtepi32_epi8(_mm512_permutex2var_epi32(_mm512_cvtepu8_epi32(a), _mm512_cvtepu8_epi32(idx), _mm512_cvtepu8_epi32(b))); + #elif defined(SIMDE_X_PERMUTEX2VAR_USE_GENERIC) + simde__m128i r; + + simde_x_permutex2var(&r, &a, &idx, &b, 0, 0); + + return r; + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + idx_ = simde__m128i_to_private(idx), + b_ = simde__m128i_to_private(b), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = ((idx_.i8[i] & 0x10) ? b_ : a_).i8[idx_.i8[i] & 0x0F]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_permutex2var_epi8 + #define _mm_permutex2var_epi8(a, idx, b) simde_mm_permutex2var_epi8(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_permutex2var_epi8 (simde__m128i a, simde__mmask16 k, simde__m128i idx, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_permutex2var_epi8(a, k, idx, b); + #else + return simde_mm_mask_mov_epi8(a, k, simde_mm_permutex2var_epi8(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_permutex2var_epi8 +#define _mm_mask_permutex2var_epi8(a, k, idx, b) simde_mm_mask_permutex2var_epi8(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask2_permutex2var_epi8 (simde__m128i a, simde__m128i idx, simde__mmask16 k, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask2_permutex2var_epi8(a, idx, k, b); + #else + return simde_mm_mask_mov_epi8(idx, k, simde_mm_permutex2var_epi8(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask2_permutex2var_epi8 +#define _mm_mask2_permutex2var_epi8(a, idx, k, b) simde_mm_mask2_permutex2var_epi8(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_permutex2var_epi8 (simde__mmask16 k, simde__m128i a, simde__m128i idx, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_permutex2var_epi8(k, a, idx, b); + #else + return simde_mm_maskz_mov_epi8(k, simde_mm_permutex2var_epi8(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_permutex2var_epi8 +#define _mm_maskz_permutex2var_epi8(k, a, idx, b) simde_mm_maskz_permutex2var_epi8(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_permutex2var_pd (simde__m128d a, simde__m128i idx, simde__m128d b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_permutex2var_pd(a, idx, b); + #else + return simde_mm_castsi128_pd(simde_mm_permutex2var_epi64(simde_mm_castpd_si128(a), idx, simde_mm_castpd_si128(b))); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_permutex2var_pd + #define _mm_permutex2var_pd(a, idx, b) simde_mm_permutex2var_pd(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mask_permutex2var_pd (simde__m128d a, simde__mmask8 k, simde__m128i idx, simde__m128d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_permutex2var_pd(a, k, idx, b); + #else + return simde_mm_mask_mov_pd(a, k, simde_mm_permutex2var_pd(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_permutex2var_pd +#define _mm_mask_permutex2var_pd(a, k, idx, b) simde_mm_mask_permutex2var_pd(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mask2_permutex2var_pd (simde__m128d a, simde__m128i idx, simde__mmask8 k, simde__m128d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask2_permutex2var_pd(a, idx, k, b); + #else + return simde_mm_mask_mov_pd(simde_mm_castsi128_pd(idx), k, simde_mm_permutex2var_pd(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask2_permutex2var_pd +#define _mm_mask2_permutex2var_pd(a, idx, k, b) simde_mm_mask2_permutex2var_pd(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_maskz_permutex2var_pd (simde__mmask8 k, simde__m128d a, simde__m128i idx, simde__m128d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_permutex2var_pd(k, a, idx, b); + #else + return simde_mm_maskz_mov_pd(k, simde_mm_permutex2var_pd(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_permutex2var_pd +#define _mm_maskz_permutex2var_pd(k, a, idx, b) simde_mm_maskz_permutex2var_pd(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_permutex2var_ps (simde__m128 a, simde__m128i idx, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_permutex2var_ps(a, idx, b); + #else + return simde_mm_castsi128_ps(simde_mm_permutex2var_epi32(simde_mm_castps_si128(a), idx, simde_mm_castps_si128(b))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_permutex2var_ps + #define _mm_permutex2var_ps(a, idx, b) simde_mm_permutex2var_ps(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mask_permutex2var_ps (simde__m128 a, simde__mmask8 k, simde__m128i idx, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_permutex2var_ps(a, k, idx, b); + #else + return simde_mm_mask_mov_ps(a, k, simde_mm_permutex2var_ps(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_permutex2var_ps +#define _mm_mask_permutex2var_ps(a, k, idx, b) simde_mm_mask_permutex2var_ps(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mask2_permutex2var_ps (simde__m128 a, simde__m128i idx, simde__mmask8 k, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask2_permutex2var_ps(a, idx, k, b); + #else + return simde_mm_mask_mov_ps(simde_mm_castsi128_ps(idx), k, simde_mm_permutex2var_ps(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask2_permutex2var_ps +#define _mm_mask2_permutex2var_ps(a, idx, k, b) simde_mm_mask2_permutex2var_ps(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_maskz_permutex2var_ps (simde__mmask8 k, simde__m128 a, simde__m128i idx, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_permutex2var_ps(k, a, idx, b); + #else + return simde_mm_maskz_mov_ps(k, simde_mm_permutex2var_ps(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_permutex2var_ps +#define _mm_maskz_permutex2var_ps(k, a, idx, b) simde_mm_maskz_permutex2var_ps(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_permutex2var_epi16 (simde__m256i a, simde__m256i idx, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_permutex2var_epi16(a, idx, b); + #elif defined(SIMDE_X86_AVX2_NATIVE) + __m256i hilo, hilo2, hi, lo, idx2, ta, tb, select; + const __m256i ones = _mm256_set1_epi16(1); + + idx2 = _mm256_srli_epi32(idx, 1); + + ta = _mm256_permutevar8x32_epi32(a, idx2); + tb = _mm256_permutevar8x32_epi32(b, idx2); + select = _mm256_slli_epi32(idx2, 28); + hilo = _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(ta), + _mm256_castsi256_ps(tb), + _mm256_castsi256_ps(select))); + idx2 = _mm256_srli_epi32(idx2, 16); + + ta = _mm256_permutevar8x32_epi32(a, idx2); + tb = _mm256_permutevar8x32_epi32(b, idx2); + select = _mm256_slli_epi32(idx2, 28); + hilo2 = _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(ta), + _mm256_castsi256_ps(tb), + _mm256_castsi256_ps(select))); + + lo = _mm256_blend_epi16(_mm256_slli_epi32(hilo2, 16), hilo, 0x55); + hi = _mm256_blend_epi16(hilo2, _mm256_srli_epi32(hilo, 16), 0x55); + + select = _mm256_cmpeq_epi16(_mm256_and_si256(idx, ones), ones); + return _mm256_blendv_epi8(lo, hi, select); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + idx_ = simde__m256i_to_private(idx), + b_ = simde__m256i_to_private(b), + r_; + + #if defined(SIMDE_X_PERMUTEX2VAR_USE_GENERIC) + simde_x_permutex2var(r_.m128i, a_.m128i, idx_.m128i, b_.m128i, 1, 1); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = ((idx_.i16[i] & 0x10) ? b_ : a_).i16[idx_.i16[i] & 0x0F]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutex2var_epi16 + #define _mm256_permutex2var_epi16(a, idx, b) simde_mm256_permutex2var_epi16(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_permutex2var_epi16 (simde__m256i a, simde__mmask16 k, simde__m256i idx, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_permutex2var_epi16(a, k, idx, b); + #else + return simde_mm256_mask_mov_epi16(a, k, simde_mm256_permutex2var_epi16(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_permutex2var_epi16 +#define _mm256_mask_permutex2var_epi16(a, k, idx, b) simde_mm256_mask_permutex2var_epi16(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask2_permutex2var_epi16 (simde__m256i a, simde__m256i idx, simde__mmask16 k, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask2_permutex2var_epi16(a, idx, k, b); + #else + return simde_mm256_mask_mov_epi16(idx, k, simde_mm256_permutex2var_epi16(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask2_permutex2var_epi16 +#define _mm256_mask2_permutex2var_epi16(a, idx, k, b) simde_mm256_mask2_permutex2var_epi16(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_permutex2var_epi16 (simde__mmask16 k, simde__m256i a, simde__m256i idx, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_permutex2var_epi16(k, a, idx, b); + #else + return simde_mm256_maskz_mov_epi16(k, simde_mm256_permutex2var_epi16(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_permutex2var_epi16 +#define _mm256_maskz_permutex2var_epi16(k, a, idx, b) simde_mm256_maskz_permutex2var_epi16(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_permutex2var_epi32 (simde__m256i a, simde__m256i idx, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_permutex2var_epi32(a, idx, b); + #elif defined(SIMDE_X86_AVX2_NATIVE) + __m256i ta, tb, select; + ta = _mm256_permutevar8x32_epi32(a, idx); + tb = _mm256_permutevar8x32_epi32(b, idx); + select = _mm256_slli_epi32(idx, 28); + return _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(ta), + _mm256_castsi256_ps(tb), + _mm256_castsi256_ps(select))); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + idx_ = simde__m256i_to_private(idx), + b_ = simde__m256i_to_private(b), + r_; + + #if defined(SIMDE_X_PERMUTEX2VAR_USE_GENERIC) + simde_x_permutex2var(r_.m128i, a_.m128i, idx_.m128i, b_.m128i, 2, 1); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ((idx_.i32[i] & 8) ? b_ : a_).i32[idx_.i32[i] & 7]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutex2var_epi32 + #define _mm256_permutex2var_epi32(a, idx, b) simde_mm256_permutex2var_epi32(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_permutex2var_epi32 (simde__m256i a, simde__mmask8 k, simde__m256i idx, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_permutex2var_epi32(a, k, idx, b); + #else + return simde_mm256_mask_mov_epi32(a, k, simde_mm256_permutex2var_epi32(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_permutex2var_epi32 +#define _mm256_mask_permutex2var_epi32(a, k, idx, b) simde_mm256_mask_permutex2var_epi32(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask2_permutex2var_epi32 (simde__m256i a, simde__m256i idx, simde__mmask8 k, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask2_permutex2var_epi32(a, idx, k, b); + #else + return simde_mm256_mask_mov_epi32(idx, k, simde_mm256_permutex2var_epi32(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask2_permutex2var_epi32 +#define _mm256_mask2_permutex2var_epi32(a, idx, k, b) simde_mm256_mask2_permutex2var_epi32(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_permutex2var_epi32 (simde__mmask8 k, simde__m256i a, simde__m256i idx, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_permutex2var_epi32(k, a, idx, b); + #else + return simde_mm256_maskz_mov_epi32(k, simde_mm256_permutex2var_epi32(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_permutex2var_epi32 +#define _mm256_maskz_permutex2var_epi32(k, a, idx, b) simde_mm256_maskz_permutex2var_epi32(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_permutex2var_epi64 (simde__m256i a, simde__m256i idx, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_permutex2var_epi64(a, idx, b); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + idx_ = simde__m256i_to_private(idx), + b_ = simde__m256i_to_private(b), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = ((idx_.i64[i] & 4) ? b_ : a_).i64[idx_.i64[i] & 3]; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutex2var_epi64 + #define _mm256_permutex2var_epi64(a, idx, b) simde_mm256_permutex2var_epi64(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_permutex2var_epi64 (simde__m256i a, simde__mmask8 k, simde__m256i idx, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_permutex2var_epi64(a, k, idx, b); + #else + return simde_mm256_mask_mov_epi64(a, k, simde_mm256_permutex2var_epi64(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_permutex2var_epi64 +#define _mm256_mask_permutex2var_epi64(a, k, idx, b) simde_mm256_mask_permutex2var_epi64(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask2_permutex2var_epi64 (simde__m256i a, simde__m256i idx, simde__mmask8 k, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask2_permutex2var_epi64(a, idx, k, b); + #else + return simde_mm256_mask_mov_epi64(idx, k, simde_mm256_permutex2var_epi64(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask2_permutex2var_epi64 +#define _mm256_mask2_permutex2var_epi64(a, idx, k, b) simde_mm256_mask2_permutex2var_epi64(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_permutex2var_epi64 (simde__mmask8 k, simde__m256i a, simde__m256i idx, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_permutex2var_epi64(k, a, idx, b); + #else + return simde_mm256_maskz_mov_epi64(k, simde_mm256_permutex2var_epi64(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_permutex2var_epi64 +#define _mm256_maskz_permutex2var_epi64(k, a, idx, b) simde_mm256_maskz_permutex2var_epi64(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_permutex2var_epi8 (simde__m256i a, simde__m256i idx, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_permutex2var_epi8(a, idx, b); + #elif defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_cvtepi16_epi8(_mm512_permutex2var_epi16(_mm512_cvtepu8_epi16(a), _mm512_cvtepu8_epi16(idx), _mm512_cvtepu8_epi16(b))); + #elif defined(SIMDE_X86_AVX2_NATIVE) + __m256i t0, t1, index, select0x10, select0x20, a01, b01; + const __m256i mask = _mm256_set1_epi8(0x3F); + const __m256i a0 = _mm256_permute4x64_epi64(a, (1 << 6) + (0 << 4) + (1 << 2) + (0 << 0)); + const __m256i a1 = _mm256_permute4x64_epi64(a, (3 << 6) + (2 << 4) + (3 << 2) + (2 << 0)); + const __m256i b0 = _mm256_permute4x64_epi64(b, (1 << 6) + (0 << 4) + (1 << 2) + (0 << 0)); + const __m256i b1 = _mm256_permute4x64_epi64(b, (3 << 6) + (2 << 4) + (3 << 2) + (2 << 0)); + + index = _mm256_and_si256(idx, mask); + t0 = _mm256_shuffle_epi8(a0, index); + t1 = _mm256_shuffle_epi8(a1, index); + select0x10 = _mm256_slli_epi64(index, 3); + a01 = _mm256_blendv_epi8(t0, t1, select0x10); + t0 = _mm256_shuffle_epi8(b0, index); + t1 = _mm256_shuffle_epi8(b1, index); + b01 = _mm256_blendv_epi8(t0, t1, select0x10); + select0x20 = _mm256_slli_epi64(index, 2); + return _mm256_blendv_epi8(a01, b01, select0x20); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + idx_ = simde__m256i_to_private(idx), + b_ = simde__m256i_to_private(b), + r_; + + #if defined(SIMDE_X_PERMUTEX2VAR_USE_GENERIC) + simde_x_permutex2var(r_.m128i, a_.m128i, idx_.m128i, b_.m128i, 0, 1); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = ((idx_.i8[i] & 0x20) ? b_ : a_).i8[idx_.i8[i] & 0x1F]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutex2var_epi8 + #define _mm256_permutex2var_epi8(a, idx, b) simde_mm256_permutex2var_epi8(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_permutex2var_epi8 (simde__m256i a, simde__mmask32 k, simde__m256i idx, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_permutex2var_epi8(a, k, idx, b); + #else + return simde_mm256_mask_mov_epi8(a, k, simde_mm256_permutex2var_epi8(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_permutex2var_epi8 +#define _mm256_mask_permutex2var_epi8(a, k, idx, b) simde_mm256_mask_permutex2var_epi8(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask2_permutex2var_epi8 (simde__m256i a, simde__m256i idx, simde__mmask32 k, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask2_permutex2var_epi8(a, idx, k, b); + #else + return simde_mm256_mask_mov_epi8(idx, k, simde_mm256_permutex2var_epi8(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask2_permutex2var_epi8 +#define _mm256_mask2_permutex2var_epi8(a, idx, k, b) simde_mm256_mask2_permutex2var_epi8(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_permutex2var_epi8 (simde__mmask32 k, simde__m256i a, simde__m256i idx, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_permutex2var_epi8(k, a, idx, b); + #else + return simde_mm256_maskz_mov_epi8(k, simde_mm256_permutex2var_epi8(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_permutex2var_epi8 +#define _mm256_maskz_permutex2var_epi8(k, a, idx, b) simde_mm256_maskz_permutex2var_epi8(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_permutex2var_pd (simde__m256d a, simde__m256i idx, simde__m256d b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_permutex2var_pd(a, idx, b); + #else + return simde_mm256_castsi256_pd(simde_mm256_permutex2var_epi64(simde_mm256_castpd_si256(a), idx, simde_mm256_castpd_si256(b))); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutex2var_pd + #define _mm256_permutex2var_pd(a, idx, b) simde_mm256_permutex2var_pd(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mask_permutex2var_pd (simde__m256d a, simde__mmask8 k, simde__m256i idx, simde__m256d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_permutex2var_pd(a, k, idx, b); + #else + return simde_mm256_mask_mov_pd(a, k, simde_mm256_permutex2var_pd(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_permutex2var_pd +#define _mm256_mask_permutex2var_pd(a, k, idx, b) simde_mm256_mask_permutex2var_pd(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mask2_permutex2var_pd (simde__m256d a, simde__m256i idx, simde__mmask8 k, simde__m256d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask2_permutex2var_pd(a, idx, k, b); + #else + return simde_mm256_mask_mov_pd(simde_mm256_castsi256_pd(idx), k, simde_mm256_permutex2var_pd(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask2_permutex2var_pd +#define _mm256_mask2_permutex2var_pd(a, idx, k, b) simde_mm256_mask2_permutex2var_pd(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_maskz_permutex2var_pd (simde__mmask8 k, simde__m256d a, simde__m256i idx, simde__m256d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_permutex2var_pd(k, a, idx, b); + #else + return simde_mm256_maskz_mov_pd(k, simde_mm256_permutex2var_pd(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_permutex2var_pd +#define _mm256_maskz_permutex2var_pd(k, a, idx, b) simde_mm256_maskz_permutex2var_pd(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_permutex2var_ps (simde__m256 a, simde__m256i idx, simde__m256 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_permutex2var_ps(a, idx, b); + #else + return simde_mm256_castsi256_ps(simde_mm256_permutex2var_epi32(simde_mm256_castps_si256(a), idx, simde_mm256_castps_si256(b))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutex2var_ps + #define _mm256_permutex2var_ps(a, idx, b) simde_mm256_permutex2var_ps(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask_permutex2var_ps (simde__m256 a, simde__mmask8 k, simde__m256i idx, simde__m256 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_permutex2var_ps(a, k, idx, b); + #else + return simde_mm256_mask_mov_ps(a, k, simde_mm256_permutex2var_ps(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_permutex2var_ps +#define _mm256_mask_permutex2var_ps(a, k, idx, b) simde_mm256_mask_permutex2var_ps(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask2_permutex2var_ps (simde__m256 a, simde__m256i idx, simde__mmask8 k, simde__m256 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask2_permutex2var_ps(a, idx, k, b); + #else + return simde_mm256_mask_mov_ps(simde_mm256_castsi256_ps(idx), k, simde_mm256_permutex2var_ps(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask2_permutex2var_ps +#define _mm256_mask2_permutex2var_ps(a, idx, k, b) simde_mm256_mask2_permutex2var_ps(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_maskz_permutex2var_ps (simde__mmask8 k, simde__m256 a, simde__m256i idx, simde__m256 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_permutex2var_ps(k, a, idx, b); + #else + return simde_mm256_maskz_mov_ps(k, simde_mm256_permutex2var_ps(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_permutex2var_ps +#define _mm256_maskz_permutex2var_ps(k, a, idx, b) simde_mm256_maskz_permutex2var_ps(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_permutex2var_epi16 (simde__m512i a, simde__m512i idx, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_permutex2var_epi16(a, idx, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + idx_ = simde__m512i_to_private(idx), + b_ = simde__m512i_to_private(b), + r_; + + #if defined(SIMDE_X86_AVX2_NATIVE) + __m256i hilo, hilo1, hilo2, hi, lo, idx1, idx2, ta, tb, select; + const __m256i ones = _mm256_set1_epi16(1); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i_private) / sizeof(r_.m256i_private[0])) ; i++) { + idx1 = idx_.m256i[i]; + idx2 = _mm256_srli_epi32(idx1, 1); + + select = _mm256_slli_epi32(idx2, 27); + ta = _mm256_permutevar8x32_epi32(a_.m256i[0], idx2); + tb = _mm256_permutevar8x32_epi32(b_.m256i[0], idx2); + hilo = _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(ta), + _mm256_castsi256_ps(tb), + _mm256_castsi256_ps(select))); + ta = _mm256_permutevar8x32_epi32(a_.m256i[1], idx2); + tb = _mm256_permutevar8x32_epi32(b_.m256i[1], idx2); + hilo1 = _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(ta), + _mm256_castsi256_ps(tb), + _mm256_castsi256_ps(select))); + select = _mm256_add_epi32(select, select); + hilo1 = _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(hilo), + _mm256_castsi256_ps(hilo1), + _mm256_castsi256_ps(select))); + + idx2 = _mm256_srli_epi32(idx2, 16); + + select = _mm256_slli_epi32(idx2, 27); + ta = _mm256_permutevar8x32_epi32(a_.m256i[0], idx2); + tb = _mm256_permutevar8x32_epi32(b_.m256i[0], idx2); + hilo = _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(ta), + _mm256_castsi256_ps(tb), + _mm256_castsi256_ps(select))); + ta = _mm256_permutevar8x32_epi32(a_.m256i[1], idx2); + tb = _mm256_permutevar8x32_epi32(b_.m256i[1], idx2); + hilo2 = _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(ta), + _mm256_castsi256_ps(tb), + _mm256_castsi256_ps(select))); + select = _mm256_add_epi32(select, select); + hilo2 = _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(hilo), + _mm256_castsi256_ps(hilo2), + _mm256_castsi256_ps(select))); + + lo = _mm256_blend_epi16(_mm256_slli_epi32(hilo2, 16), hilo1, 0x55); + hi = _mm256_blend_epi16(hilo2, _mm256_srli_epi32(hilo1, 16), 0x55); + + select = _mm256_cmpeq_epi16(_mm256_and_si256(idx1, ones), ones); + r_.m256i[i] = _mm256_blendv_epi8(lo, hi, select); + } + #elif defined(SIMDE_X_PERMUTEX2VAR_USE_GENERIC) + simde_x_permutex2var(r_.m128i, a_.m128i, idx_.m128i, b_.m128i, 1, 2); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = ((idx_.i16[i] & 0x20) ? b_ : a_).i16[idx_.i16[i] & 0x1F]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm512_permutex2var_epi16 + #define _mm512_permutex2var_epi16(a, idx, b) simde_mm512_permutex2var_epi16(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_permutex2var_epi16 (simde__m512i a, simde__mmask32 k, simde__m512i idx, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_permutex2var_epi16(a, k, idx, b); + #else + return simde_mm512_mask_mov_epi16(a, k, simde_mm512_permutex2var_epi16(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_permutex2var_epi16 +#define _mm512_mask_permutex2var_epi16(a, k, idx, b) simde_mm512_mask_permutex2var_epi16(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask2_permutex2var_epi16 (simde__m512i a, simde__m512i idx, simde__mmask32 k, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask2_permutex2var_epi16(a, idx, k, b); + #else + return simde_mm512_mask_mov_epi16(idx, k, simde_mm512_permutex2var_epi16(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask2_permutex2var_epi16 +#define _mm512_mask2_permutex2var_epi16(a, idx, k, b) simde_mm512_mask2_permutex2var_epi16(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_permutex2var_epi16 (simde__mmask32 k, simde__m512i a, simde__m512i idx, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_permutex2var_epi16(k, a, idx, b); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_permutex2var_epi16(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_permutex2var_epi16 +#define _mm512_maskz_permutex2var_epi16(k, a, idx, b) simde_mm512_maskz_permutex2var_epi16(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_permutex2var_epi32 (simde__m512i a, simde__m512i idx, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_permutex2var_epi32(a, idx, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + idx_ = simde__m512i_to_private(idx), + b_ = simde__m512i_to_private(b), + r_; + + #if defined(SIMDE_X86_AVX2_NATIVE) + __m256i index, t0, t1, a01, b01, select; + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i_private) / sizeof(r_.m256i_private[0])) ; i++) { + index = idx_.m256i[i]; + t0 = _mm256_permutevar8x32_epi32(a_.m256i[0], index); + t1 = _mm256_permutevar8x32_epi32(a_.m256i[1], index); + select = _mm256_slli_epi32(index, 28); + a01 = _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(t0), + _mm256_castsi256_ps(t1), + _mm256_castsi256_ps(select))); + t0 = _mm256_permutevar8x32_epi32(b_.m256i[0], index); + t1 = _mm256_permutevar8x32_epi32(b_.m256i[1], index); + b01 = _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(t0), + _mm256_castsi256_ps(t1), + _mm256_castsi256_ps(select))); + select = _mm256_slli_epi32(index, 27); + r_.m256i[i] = _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(a01), + _mm256_castsi256_ps(b01), + _mm256_castsi256_ps(select))); + } + #elif defined(SIMDE_X_PERMUTEX2VAR_USE_GENERIC) + simde_x_permutex2var(r_.m128i, a_.m128i, idx_.m128i, b_.m128i, 2, 2); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ((idx_.i32[i] & 0x10) ? b_ : a_).i32[idx_.i32[i] & 0x0F]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_permutex2var_epi32 + #define _mm512_permutex2var_epi32(a, idx, b) simde_mm512_permutex2var_epi32(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_permutex2var_epi32 (simde__m512i a, simde__mmask16 k, simde__m512i idx, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_permutex2var_epi32(a, k, idx, b); + #else + return simde_mm512_mask_mov_epi32(a, k, simde_mm512_permutex2var_epi32(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_permutex2var_epi32 +#define _mm512_mask_permutex2var_epi32(a, k, idx, b) simde_mm512_mask_permutex2var_epi32(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask2_permutex2var_epi32 (simde__m512i a, simde__m512i idx, simde__mmask16 k, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask2_permutex2var_epi32(a, idx, k, b); + #else + return simde_mm512_mask_mov_epi32(idx, k, simde_mm512_permutex2var_epi32(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask2_permutex2var_epi32 +#define _mm512_mask2_permutex2var_epi32(a, idx, k, b) simde_mm512_mask2_permutex2var_epi32(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_permutex2var_epi32 (simde__mmask16 k, simde__m512i a, simde__m512i idx, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_permutex2var_epi32(k, a, idx, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_permutex2var_epi32(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_permutex2var_epi32 +#define _mm512_maskz_permutex2var_epi32(k, a, idx, b) simde_mm512_maskz_permutex2var_epi32(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_permutex2var_epi64 (simde__m512i a, simde__m512i idx, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_permutex2var_epi64(a, idx, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + idx_ = simde__m512i_to_private(idx), + b_ = simde__m512i_to_private(b), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = ((idx_.i64[i] & 8) ? b_ : a_).i64[idx_.i64[i] & 7]; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_permutex2var_epi64 + #define _mm512_permutex2var_epi64(a, idx, b) simde_mm512_permutex2var_epi64(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_permutex2var_epi64 (simde__m512i a, simde__mmask8 k, simde__m512i idx, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_permutex2var_epi64(a, k, idx, b); + #else + return simde_mm512_mask_mov_epi64(a, k, simde_mm512_permutex2var_epi64(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_permutex2var_epi64 +#define _mm512_mask_permutex2var_epi64(a, k, idx, b) simde_mm512_mask_permutex2var_epi64(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask2_permutex2var_epi64 (simde__m512i a, simde__m512i idx, simde__mmask8 k, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask2_permutex2var_epi64(a, idx, k, b); + #else + return simde_mm512_mask_mov_epi64(idx, k, simde_mm512_permutex2var_epi64(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask2_permutex2var_epi64 +#define _mm512_mask2_permutex2var_epi64(a, idx, k, b) simde_mm512_mask2_permutex2var_epi64(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_permutex2var_epi64 (simde__mmask8 k, simde__m512i a, simde__m512i idx, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_permutex2var_epi64(k, a, idx, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_permutex2var_epi64(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_permutex2var_epi64 +#define _mm512_maskz_permutex2var_epi64(k, a, idx, b) simde_mm512_maskz_permutex2var_epi64(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_permutex2var_epi8 (simde__m512i a, simde__m512i idx, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) + return _mm512_permutex2var_epi8(a, idx, b); + #elif defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + __m512i hilo, hi, lo, hi2, lo2, idx2; + const __m512i ones = _mm512_set1_epi8(1); + const __m512i low_bytes = _mm512_set1_epi16(0x00FF); + + idx2 = _mm512_srli_epi16(idx, 1); + hilo = _mm512_permutex2var_epi16(a, idx2, b); + __mmask64 mask = _mm512_test_epi8_mask(idx, ones); + lo = _mm512_and_si512(hilo, low_bytes); + hi = _mm512_srli_epi16(hilo, 8); + + idx2 = _mm512_srli_epi16(idx, 9); + hilo = _mm512_permutex2var_epi16(a, idx2, b); + lo2 = _mm512_slli_epi16(hilo, 8); + hi2 = _mm512_andnot_si512(low_bytes, hilo); + + lo = _mm512_or_si512(lo, lo2); + hi = _mm512_or_si512(hi, hi2); + + return _mm512_mask_blend_epi8(mask, lo, hi); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + idx_ = simde__m512i_to_private(idx), + b_ = simde__m512i_to_private(b), + r_; + + #if defined(SIMDE_X86_AVX2_NATIVE) + __m256i t0, t1, index, select0x10, select0x20, select0x40, t01, t23, a0123, b0123; + const __m256i mask = _mm256_set1_epi8(0x7F); + const __m256i a0 = _mm256_permute4x64_epi64(a_.m256i[0], (1 << 6) + (0 << 4) + (1 << 2) + (0 << 0)); + const __m256i a1 = _mm256_permute4x64_epi64(a_.m256i[0], (3 << 6) + (2 << 4) + (3 << 2) + (2 << 0)); + const __m256i a2 = _mm256_permute4x64_epi64(a_.m256i[1], (1 << 6) + (0 << 4) + (1 << 2) + (0 << 0)); + const __m256i a3 = _mm256_permute4x64_epi64(a_.m256i[1], (3 << 6) + (2 << 4) + (3 << 2) + (2 << 0)); + const __m256i b0 = _mm256_permute4x64_epi64(b_.m256i[0], (1 << 6) + (0 << 4) + (1 << 2) + (0 << 0)); + const __m256i b1 = _mm256_permute4x64_epi64(b_.m256i[0], (3 << 6) + (2 << 4) + (3 << 2) + (2 << 0)); + const __m256i b2 = _mm256_permute4x64_epi64(b_.m256i[1], (1 << 6) + (0 << 4) + (1 << 2) + (0 << 0)); + const __m256i b3 = _mm256_permute4x64_epi64(b_.m256i[1], (3 << 6) + (2 << 4) + (3 << 2) + (2 << 0)); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i_private) / sizeof(r_.m256i_private[0])) ; i++) { + index = _mm256_and_si256(idx_.m256i[i], mask); + t0 = _mm256_shuffle_epi8(a0, index); + t1 = _mm256_shuffle_epi8(a1, index); + select0x10 = _mm256_slli_epi64(index, 3); + t01 = _mm256_blendv_epi8(t0, t1, select0x10); + t0 = _mm256_shuffle_epi8(a2, index); + t1 = _mm256_shuffle_epi8(a3, index); + t23 = _mm256_blendv_epi8(t0, t1, select0x10); + select0x20 = _mm256_slli_epi64(index, 2); + a0123 = _mm256_blendv_epi8(t01, t23, select0x20); + t0 = _mm256_shuffle_epi8(b0, index); + t1 = _mm256_shuffle_epi8(b1, index); + t01 = _mm256_blendv_epi8(t0, t1, select0x10); + t0 = _mm256_shuffle_epi8(b2, index); + t1 = _mm256_shuffle_epi8(b3, index); + t23 = _mm256_blendv_epi8(t0, t1, select0x10); + b0123 = _mm256_blendv_epi8(t01, t23, select0x20); + select0x40 = _mm256_slli_epi64(index, 1); + r_.m256i[i] = _mm256_blendv_epi8(a0123, b0123, select0x40); + } + #elif defined(SIMDE_X_PERMUTEX2VAR_USE_GENERIC) + simde_x_permutex2var(r_.m128i, a_.m128i, idx_.m128i, b_.m128i, 0, 2); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = ((idx_.i8[i] & 0x40) ? b_ : a_).i8[idx_.i8[i] & 0x3F]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) + #undef _mm512_permutex2var_epi8 + #define _mm512_permutex2var_epi8(a, idx, b) simde_mm512_permutex2var_epi8(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_permutex2var_epi8 (simde__m512i a, simde__mmask64 k, simde__m512i idx, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) + return _mm512_mask_permutex2var_epi8(a, k, idx, b); + #else + return simde_mm512_mask_mov_epi8(a, k, simde_mm512_permutex2var_epi8(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_permutex2var_epi8 +#define _mm512_mask_permutex2var_epi8(a, k, idx, b) simde_mm512_mask_permutex2var_epi8(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask2_permutex2var_epi8 (simde__m512i a, simde__m512i idx, simde__mmask64 k, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) + return _mm512_mask2_permutex2var_epi8(a, idx, k, b); + #else + return simde_mm512_mask_mov_epi8(idx, k, simde_mm512_permutex2var_epi8(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask2_permutex2var_epi8 +#define _mm512_mask2_permutex2var_epi8(a, idx, k, b) simde_mm512_mask2_permutex2var_epi8(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_permutex2var_epi8 (simde__mmask64 k, simde__m512i a, simde__m512i idx, simde__m512i b) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) + return _mm512_maskz_permutex2var_epi8(k, a, idx, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_permutex2var_epi8(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_permutex2var_epi8 +#define _mm512_maskz_permutex2var_epi8(k, a, idx, b) simde_mm512_maskz_permutex2var_epi8(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_permutex2var_pd (simde__m512d a, simde__m512i idx, simde__m512d b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_permutex2var_pd(a, idx, b); + #else + return simde_mm512_castsi512_pd(simde_mm512_permutex2var_epi64(simde_mm512_castpd_si512(a), idx, simde_mm512_castpd_si512(b))); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_permutex2var_pd + #define _mm512_permutex2var_pd(a, idx, b) simde_mm512_permutex2var_pd(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_permutex2var_pd (simde__m512d a, simde__mmask8 k, simde__m512i idx, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_permutex2var_pd(a, k, idx, b); + #else + return simde_mm512_mask_mov_pd(a, k, simde_mm512_permutex2var_pd(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_permutex2var_pd +#define _mm512_mask_permutex2var_pd(a, k, idx, b) simde_mm512_mask_permutex2var_pd(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask2_permutex2var_pd (simde__m512d a, simde__m512i idx, simde__mmask8 k, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask2_permutex2var_pd(a, idx, k, b); + #else + return simde_mm512_mask_mov_pd(simde_mm512_castsi512_pd(idx), k, simde_mm512_permutex2var_pd(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask2_permutex2var_pd +#define _mm512_mask2_permutex2var_pd(a, idx, k, b) simde_mm512_mask2_permutex2var_pd(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_permutex2var_pd (simde__mmask8 k, simde__m512d a, simde__m512i idx, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_permutex2var_pd(k, a, idx, b); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_permutex2var_pd(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_permutex2var_pd +#define _mm512_maskz_permutex2var_pd(k, a, idx, b) simde_mm512_maskz_permutex2var_pd(k, a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_permutex2var_ps (simde__m512 a, simde__m512i idx, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_permutex2var_ps(a, idx, b); + #else + return simde_mm512_castsi512_ps(simde_mm512_permutex2var_epi32(simde_mm512_castps_si512(a), idx, simde_mm512_castps_si512(b))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_permutex2var_ps + #define _mm512_permutex2var_ps(a, idx, b) simde_mm512_permutex2var_ps(a, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_permutex2var_ps (simde__m512 a, simde__mmask16 k, simde__m512i idx, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_permutex2var_ps(a, k, idx, b); + #else + return simde_mm512_mask_mov_ps(a, k, simde_mm512_permutex2var_ps(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_permutex2var_ps +#define _mm512_mask_permutex2var_ps(a, k, idx, b) simde_mm512_mask_permutex2var_ps(a, k, idx, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask2_permutex2var_ps (simde__m512 a, simde__m512i idx, simde__mmask16 k, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask2_permutex2var_ps(a, idx, k, b); + #else + return simde_mm512_mask_mov_ps(simde_mm512_castsi512_ps(idx), k, simde_mm512_permutex2var_ps(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask2_permutex2var_ps +#define _mm512_mask2_permutex2var_ps(a, idx, k, b) simde_mm512_mask2_permutex2var_ps(a, idx, k, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_permutex2var_ps (simde__mmask16 k, simde__m512 a, simde__m512i idx, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_permutex2var_ps(k, a, idx, b); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_permutex2var_ps(a, idx, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_permutex2var_ps +#define _mm512_maskz_permutex2var_ps(k, a, idx, b) simde_mm512_maskz_permutex2var_ps(k, a, idx, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_PERMUTEX2VAR_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/permutexvar.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/permutexvar.h new file mode 100644 index 00000000..1b4bf7ac --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/permutexvar.h @@ -0,0 +1,1194 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Christopher Moore + */ + +#if !defined(SIMDE_X86_AVX512_PERMUTEXVAR_H) +#define SIMDE_X86_AVX512_PERMUTEXVAR_H + +#include "types.h" +#include "and.h" +#include "andnot.h" +#include "blend.h" +#include "mov.h" +#include "or.h" +#include "set1.h" +#include "slli.h" +#include "srli.h" +#include "test.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_permutexvar_epi16 (simde__m128i idx, simde__m128i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_permutexvar_epi16(idx, a); + #elif defined(SIMDE_X86_SSSE3_NATIVE) + simde__m128i mask16 = simde_mm_set1_epi16(0x0007); + simde__m128i shift16 = simde_mm_set1_epi16(0x0202); + simde__m128i byte_index16 = simde_mm_set1_epi16(0x0100); + simde__m128i index16 = simde_mm_and_si128(idx, mask16); + index16 = simde_mm_mullo_epi16(index16, shift16); + index16 = simde_mm_add_epi16(index16, byte_index16); + return simde_mm_shuffle_epi8(a, index16); + #else + simde__m128i_private + idx_ = simde__m128i_to_private(idx), + a_ = simde__m128i_to_private(a), + r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint16x8_t mask16 = vdupq_n_u16(0x0007); + uint16x8_t byte_index16 = vdupq_n_u16(0x0100); + uint16x8_t index16 = vandq_u16(idx_.neon_u16, mask16); + index16 = vmulq_n_u16(index16, 0x0202); + index16 = vaddq_u16(index16, byte_index16); + r_.neon_u8 = vqtbl1q_u8(a_.neon_u8, vreinterpretq_u8_u16(index16)); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) index16; + index16 = vec_and(idx_.altivec_u16, vec_splat_u16(7)); + index16 = vec_mladd(index16, vec_splats(HEDLEY_STATIC_CAST(unsigned short, 0x0202)), vec_splats(HEDLEY_STATIC_CAST(unsigned short, 0x0100))); + r_.altivec_u8 = vec_perm(a_.altivec_u8, a_.altivec_u8, HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), index16)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + const v128_t mask16 = wasm_i16x8_splat(0x0007); + const v128_t shift16 = wasm_i16x8_splat(0x0202); + const v128_t byte_index16 = wasm_i16x8_splat(0x0100); + v128_t index16 = wasm_v128_and(idx_.wasm_v128, mask16); + index16 = wasm_i16x8_mul(index16, shift16); + index16 = wasm_i16x8_add(index16, byte_index16); + r_.wasm_v128 = wasm_i8x16_swizzle(a_.wasm_v128, index16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[idx_.i16[i] & 0x07]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_permutexvar_epi16 + #define _mm_permutexvar_epi16(idx, a) simde_mm_permutexvar_epi16(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_permutexvar_epi16 (simde__m128i src, simde__mmask8 k, simde__m128i idx, simde__m128i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_permutexvar_epi16(src, k, idx, a); + #else + return simde_mm_mask_mov_epi16(src, k, simde_mm_permutexvar_epi16(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_permutexvar_epi16 + #define _mm_mask_permutexvar_epi16(src, k, idx, a) simde_mm_mask_permutexvar_epi16(src, k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_permutexvar_epi16 (simde__mmask8 k, simde__m128i idx, simde__m128i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_permutexvar_epi16(k, idx, a); + #else + return simde_mm_maskz_mov_epi16(k, simde_mm_permutexvar_epi16(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_permutexvar_epi16 + #define _mm_maskz_permutexvar_epi16(k, idx, a) simde_mm_maskz_permutexvar_epi16(k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_permutexvar_epi8 (simde__m128i idx, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_permutexvar_epi8(idx, a); + #elif defined(SIMDE_X86_SSSE3_NATIVE) + simde__m128i mask = simde_mm_set1_epi8(0x0F); + simde__m128i index = simde_mm_and_si128(idx, mask); + return simde_mm_shuffle_epi8(a, index); + #else + simde__m128i_private + idx_ = simde__m128i_to_private(idx), + a_ = simde__m128i_to_private(a), + r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint8x16_t mask = vdupq_n_u8(0x0F); + uint8x16_t index = vandq_u8(idx_.neon_u8, mask); + r_.neon_u8 = vqtbl1q_u8(a_.neon_u8, index); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_u8 = vec_perm(a_.altivec_u8, a_.altivec_u8, idx_.altivec_u8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + const v128_t mask = wasm_i8x16_splat(0x0F); + v128_t index = wasm_v128_and(idx_.wasm_v128, mask); + r_.wasm_v128 = wasm_i8x16_swizzle(a_.wasm_v128, index); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[idx_.i8[i] & 0x0F]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_permutexvar_epi8 + #define _mm_permutexvar_epi8(idx, a) simde_mm_permutexvar_epi8(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_permutexvar_epi8 (simde__m128i src, simde__mmask16 k, simde__m128i idx, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_permutexvar_epi8(src, k, idx, a); + #else + return simde_mm_mask_mov_epi8(src, k, simde_mm_permutexvar_epi8(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_permutexvar_epi8 + #define _mm_mask_permutexvar_epi8(src, k, idx, a) simde_mm_mask_permutexvar_epi8(src, k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_permutexvar_epi8 (simde__mmask16 k, simde__m128i idx, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_permutexvar_epi8(k, idx, a); + #else + return simde_mm_maskz_mov_epi8(k, simde_mm_permutexvar_epi8(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_permutexvar_epi8 + #define _mm_maskz_permutexvar_epi8(k, idx, a) simde_mm_maskz_permutexvar_epi8(k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_permutexvar_epi16 (simde__m256i idx, simde__m256i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_permutexvar_epi16(idx, a); + #elif defined(SIMDE_X86_AVX2_NATIVE) + simde__m256i mask16 = simde_mm256_set1_epi16(0x001F); + simde__m256i shift16 = simde_mm256_set1_epi16(0x0202); + simde__m256i byte_index16 = simde_mm256_set1_epi16(0x0100); + simde__m256i index16 = simde_mm256_and_si256(idx, mask16); + index16 = simde_mm256_mullo_epi16(index16, shift16); + simde__m256i lo = simde_mm256_permute4x64_epi64(a, (1 << 6) + (0 << 4) + (1 << 2) + (0 << 0)); + simde__m256i hi = simde_mm256_permute4x64_epi64(a, (3 << 6) + (2 << 4) + (3 << 2) + (2 << 0)); + simde__m256i select = simde_mm256_slli_epi64(index16, 3); + index16 = simde_mm256_add_epi16(index16, byte_index16); + lo = simde_mm256_shuffle_epi8(lo, index16); + hi = simde_mm256_shuffle_epi8(hi, index16); + return simde_mm256_blendv_epi8(lo, hi, select); + #else + simde__m256i_private + idx_ = simde__m256i_to_private(idx), + a_ = simde__m256i_to_private(a), + r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint8x16x2_t table = { { a_.m128i_private[0].neon_u8, + a_.m128i_private[1].neon_u8 } }; + uint16x8_t mask16 = vdupq_n_u16(0x000F); + uint16x8_t byte_index16 = vdupq_n_u16(0x0100); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + uint16x8_t index16 = vandq_u16(idx_.m128i_private[i].neon_u16, mask16); + index16 = vmulq_n_u16(index16, 0x0202); + index16 = vaddq_u16(index16, byte_index16); + r_.m128i_private[i].neon_u8 = vqtbl2q_u8(table, vreinterpretq_u8_u16(index16)); + } + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) index16, mask16, shift16, byte_index16; + mask16 = vec_splat_u16(0x000F); + shift16 = vec_splats(HEDLEY_STATIC_CAST(unsigned short, 0x0202)); + byte_index16 = vec_splats(HEDLEY_STATIC_CAST(unsigned short, 0x0100)); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + index16 = vec_and(idx_.m128i_private[i].altivec_u16, mask16); + index16 = vec_mladd(index16, shift16, byte_index16); + r_.m128i_private[i].altivec_u8 = vec_perm(a_.m128i_private[0].altivec_u8, + a_.m128i_private[1].altivec_u8, + HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), index16)); + } + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t index, index16, r, t; + const v128_t mask16 = wasm_i16x8_splat(0x000F); + const v128_t shift16 = wasm_i16x8_splat(0x0202); + const v128_t byte_index16 = wasm_i16x8_splat(0x0100); + const v128_t sixteen = wasm_i8x16_splat(16); + const v128_t a0 = a_.m128i_private[0].wasm_v128; + const v128_t a1 = a_.m128i_private[1].wasm_v128; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + index16 = wasm_v128_and(idx_.m128i_private[i].wasm_v128, mask16); + index16 = wasm_i16x8_mul(index16, shift16); + index = wasm_i16x8_add(index16, byte_index16); + r = wasm_i8x16_swizzle(a0, index); + + index = wasm_i8x16_sub(index, sixteen); + t = wasm_i8x16_swizzle(a1, index); + r_.m128i_private[i].wasm_v128 = wasm_v128_or(r, t); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[idx_.i16[i] & 0x0F]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutexvar_epi16 + #define _mm256_permutexvar_epi16(idx, a) simde_mm256_permutexvar_epi16(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_permutexvar_epi16 (simde__m256i src, simde__mmask16 k, simde__m256i idx, simde__m256i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_permutexvar_epi16(src, k, idx, a); + #else + return simde_mm256_mask_mov_epi16(src, k, simde_mm256_permutexvar_epi16(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_permutexvar_epi16 + #define _mm256_mask_permutexvar_epi16(src, k, idx, a) simde_mm256_mask_permutexvar_epi16(src, k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_permutexvar_epi16 (simde__mmask16 k, simde__m256i idx, simde__m256i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_permutexvar_epi16(k, idx, a); + #else + return simde_mm256_maskz_mov_epi16(k, simde_mm256_permutexvar_epi16(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_permutexvar_epi16 + #define _mm256_maskz_permutexvar_epi16(k, idx, a) simde_mm256_maskz_permutexvar_epi16(k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_permutexvar_epi32 (simde__m256i idx, simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_permutexvar_epi32(idx, a); + #elif defined(SIMDE_X86_AVX2_NATIVE) + return simde_mm256_permutevar8x32_epi32(a, idx); + #else + simde__m256i_private + idx_ = simde__m256i_to_private(idx), + a_ = simde__m256i_to_private(a), + r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint8x16x2_t table = { { a_.m128i_private[0].neon_u8, + a_.m128i_private[1].neon_u8 } }; + uint32x4_t mask32 = vdupq_n_u32(0x00000007); + uint32x4_t byte_index32 = vdupq_n_u32(0x03020100); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + uint32x4_t index32 = vandq_u32(idx_.m128i_private[i].neon_u32, mask32); + index32 = vmulq_n_u32(index32, 0x04040404); + index32 = vaddq_u32(index32, byte_index32); + r_.m128i_private[i].neon_u8 = vqtbl2q_u8(table, vreinterpretq_u8_u32(index32)); + } + #else + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[idx_.i32[i] & 0x07]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutexvar_epi32 + #define _mm256_permutexvar_epi32(idx, a) simde_mm256_permutexvar_epi32(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_permutexvar_epi32 (simde__m256i src, simde__mmask8 k, simde__m256i idx, simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_permutexvar_epi32(src, k, idx, a); + #else + return simde_mm256_mask_mov_epi32(src, k, simde_mm256_permutexvar_epi32(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_permutexvar_epi32 + #define _mm256_mask_permutexvar_epi32(src, k, idx, a) simde_mm256_mask_permutexvar_epi32(src, k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_permutexvar_epi32 (simde__mmask8 k, simde__m256i idx, simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_permutexvar_epi32(k, idx, a); + #else + return simde_mm256_maskz_mov_epi32(k, simde_mm256_permutexvar_epi32(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_permutexvar_epi32 + #define _mm256_maskz_permutexvar_epi32(k, idx, a) simde_mm256_maskz_permutexvar_epi32(k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_permutexvar_epi64 (simde__m256i idx, simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_permutexvar_epi64(idx, a); + #else + simde__m256i_private + idx_ = simde__m256i_to_private(idx), + a_ = simde__m256i_to_private(a), + r_; + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[idx_.i64[i] & 3]; + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutexvar_epi64 + #define _mm256_permutexvar_epi64(idx, a) simde_mm256_permutexvar_epi64(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_permutexvar_epi64 (simde__m256i src, simde__mmask8 k, simde__m256i idx, simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_permutexvar_epi64(src, k, idx, a); + #else + return simde_mm256_mask_mov_epi64(src, k, simde_mm256_permutexvar_epi64(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_permutexvar_epi64 + #define _mm256_mask_permutexvar_epi64(src, k, idx, a) simde_mm256_mask_permutexvar_epi64(src, k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_permutexvar_epi64 (simde__mmask8 k, simde__m256i idx, simde__m256i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_permutexvar_epi64(k, idx, a); + #else + return simde_mm256_maskz_mov_epi64(k, simde_mm256_permutexvar_epi64(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_permutexvar_epi64 + #define _mm256_maskz_permutexvar_epi64(k, idx, a) simde_mm256_maskz_permutexvar_epi64(k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_permutexvar_epi8 (simde__m256i idx, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_permutexvar_epi8(idx, a); + #elif defined(SIMDE_X86_AVX2_NATIVE) + simde__m256i mask = simde_mm256_set1_epi8(0x0F); + simde__m256i lo = simde_mm256_permute4x64_epi64(a, (1 << 6) + (0 << 4) + (1 << 2) + (0 << 0)); + simde__m256i hi = simde_mm256_permute4x64_epi64(a, (3 << 6) + (2 << 4) + (3 << 2) + (2 << 0)); + simde__m256i index = simde_mm256_and_si256(idx, mask); + simde__m256i select = simde_mm256_slli_epi64(idx, 3); + lo = simde_mm256_shuffle_epi8(lo, index); + hi = simde_mm256_shuffle_epi8(hi, index); + return simde_mm256_blendv_epi8(lo, hi, select); + #else + simde__m256i_private + idx_ = simde__m256i_to_private(idx), + a_ = simde__m256i_to_private(a), + r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint8x16x2_t table = { { a_.m128i_private[0].neon_u8, + a_.m128i_private[1].neon_u8 } }; + uint8x16_t mask = vdupq_n_u8(0x1F); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].neon_u8 = vqtbl2q_u8(table, vandq_u8(idx_.m128i_private[i].neon_u8, mask)); + } + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_u8 = vec_perm(a_.m128i_private[0].altivec_u8, a_.m128i_private[1].altivec_u8, idx_.m128i_private[i].altivec_u8); + } + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t index, r, t; + const v128_t mask = wasm_i8x16_splat(0x1F); + const v128_t sixteen = wasm_i8x16_splat(16); + const v128_t a0 = a_.m128i_private[0].wasm_v128; + const v128_t a1 = a_.m128i_private[1].wasm_v128; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + index = wasm_v128_and(idx_.m128i_private[i].wasm_v128, mask); + r = wasm_i8x16_swizzle(a0, index); + index = wasm_i8x16_sub(index, sixteen); + t = wasm_i8x16_swizzle(a1, index); + r_.m128i_private[i].wasm_v128 = wasm_v128_or(r, t); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[idx_.i8[i] & 0x1F]; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutexvar_epi8 + #define _mm256_permutexvar_epi8(idx, a) simde_mm256_permutexvar_epi8(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_permutexvar_epi8 (simde__m256i src, simde__mmask32 k, simde__m256i idx, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_permutexvar_epi8(src, k, idx, a); + #else + return simde_mm256_mask_mov_epi8(src, k, simde_mm256_permutexvar_epi8(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_permutexvar_epi8 + #define _mm256_mask_permutexvar_epi8(src, k, idx, a) simde_mm256_mask_permutexvar_epi8(src, k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_permutexvar_epi8 (simde__mmask32 k, simde__m256i idx, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_permutexvar_epi8(k, idx, a); + #else + return simde_mm256_maskz_mov_epi8(k, simde_mm256_permutexvar_epi8(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_permutexvar_epi8 + #define _mm256_maskz_permutexvar_epi8(k, idx, a) simde_mm256_maskz_permutexvar_epi8(k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_permutexvar_pd (simde__m256i idx, simde__m256d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_permutexvar_pd(idx, a); + #else + return simde_mm256_castsi256_pd(simde_mm256_permutexvar_epi64(idx, simde_mm256_castpd_si256(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutexvar_pd + #define _mm256_permutexvar_pd(idx, a) simde_mm256_permutexvar_pd(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mask_permutexvar_pd (simde__m256d src, simde__mmask8 k, simde__m256i idx, simde__m256d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_permutexvar_pd(src, k, idx, a); + #else + return simde_mm256_mask_mov_pd(src, k, simde_mm256_permutexvar_pd(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_permutexvar_pd + #define _mm256_mask_permutexvar_pd(src, k, idx, a) simde_mm256_mask_permutexvar_pd(src, k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_maskz_permutexvar_pd (simde__mmask8 k, simde__m256i idx, simde__m256d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_permutexvar_pd(k, idx, a); + #else + return simde_mm256_maskz_mov_pd(k, simde_mm256_permutexvar_pd(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_permutexvar_pd + #define _mm256_maskz_permutexvar_pd(k, idx, a) simde_mm256_maskz_permutexvar_pd(k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_permutexvar_ps (simde__m256i idx, simde__m256 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_permutexvar_ps(idx, a); + #elif defined(SIMDE_X86_AVX2_NATIVE) + return simde_mm256_permutevar8x32_ps(a, idx); + #else + return simde_mm256_castsi256_ps(simde_mm256_permutexvar_epi32(idx, simde_mm256_castps_si256(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_permutexvar_ps + #define _mm256_permutexvar_ps(idx, a) simde_mm256_permutexvar_ps(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask_permutexvar_ps (simde__m256 src, simde__mmask8 k, simde__m256i idx, simde__m256 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_permutexvar_ps(src, k, idx, a); + #else + return simde_mm256_mask_mov_ps(src, k, simde_mm256_permutexvar_ps(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_permutexvar_ps + #define _mm256_mask_permutexvar_ps(src, k, idx, a) simde_mm256_mask_permutexvar_ps(src, k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_maskz_permutexvar_ps (simde__mmask8 k, simde__m256i idx, simde__m256 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_permutexvar_ps(k, idx, a); + #else + return simde_mm256_maskz_mov_ps(k, simde_mm256_permutexvar_ps(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_permutexvar_ps + #define _mm256_maskz_permutexvar_ps(k, idx, a) simde_mm256_maskz_permutexvar_ps(k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_permutexvar_epi16 (simde__m512i idx, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_permutexvar_epi16(idx, a); + #else + simde__m512i_private + idx_ = simde__m512i_to_private(idx), + a_ = simde__m512i_to_private(a), + r_; + + #if defined(SIMDE_X86_AVX2_NATIVE) + simde__m256i t0, t1, index, select, a01, a23; + simde__m256i mask = simde_mm256_set1_epi16(0x001F); + simde__m256i shift = simde_mm256_set1_epi16(0x0202); + simde__m256i byte_index = simde_mm256_set1_epi16(0x0100); + simde__m256i a0 = simde_mm256_broadcastsi128_si256(a_.m128i[0]); + simde__m256i a1 = simde_mm256_broadcastsi128_si256(a_.m128i[1]); + simde__m256i a2 = simde_mm256_broadcastsi128_si256(a_.m128i[2]); + simde__m256i a3 = simde_mm256_broadcastsi128_si256(a_.m128i[3]); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i_private) / sizeof(r_.m256i_private[0])) ; i++) { + index = idx_.m256i[i]; + index = simde_mm256_and_si256(index, mask); + index = simde_mm256_mullo_epi16(index, shift); + index = simde_mm256_add_epi16(index, byte_index); + t0 = simde_mm256_shuffle_epi8(a0, index); + t1 = simde_mm256_shuffle_epi8(a1, index); + select = simde_mm256_slli_epi64(index, 3); + a01 = simde_mm256_blendv_epi8(t0, t1, select); + t0 = simde_mm256_shuffle_epi8(a2, index); + t1 = simde_mm256_shuffle_epi8(a3, index); + a23 = simde_mm256_blendv_epi8(t0, t1, select); + select = simde_mm256_slli_epi64(index, 2); + r_.m256i[i] = simde_mm256_blendv_epi8(a01, a23, select); + } + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint8x16x4_t table = { { a_.m128i_private[0].neon_u8, + a_.m128i_private[1].neon_u8, + a_.m128i_private[2].neon_u8, + a_.m128i_private[3].neon_u8 } }; + uint16x8_t mask16 = vdupq_n_u16(0x001F); + uint16x8_t byte_index16 = vdupq_n_u16(0x0100); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + uint16x8_t index16 = vandq_u16(idx_.m128i_private[i].neon_u16, mask16); + index16 = vmulq_n_u16(index16, 0x0202); + index16 = vaddq_u16(index16, byte_index16); + r_.m128i_private[i].neon_u8 = vqtbl4q_u8(table, vreinterpretq_u8_u16(index16)); + } + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) index16, mask16, shift16, byte_index16; + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) index, test, r01, r23; + mask16 = vec_splats(HEDLEY_STATIC_CAST(unsigned short, 0x001F)); + shift16 = vec_splats(HEDLEY_STATIC_CAST(unsigned short, 0x0202)); + byte_index16 = vec_splats(HEDLEY_STATIC_CAST(unsigned short, 0x0100)); + test = vec_splats(HEDLEY_STATIC_CAST(unsigned char, 0x20)); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + index16 = vec_and(idx_.m128i_private[i].altivec_u16, mask16); + index16 = vec_mladd(index16, shift16, byte_index16); + index = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), index16); + r01 = vec_perm(a_.m128i_private[0].altivec_u8, a_.m128i_private[1].altivec_u8, index); + r23 = vec_perm(a_.m128i_private[2].altivec_u8, a_.m128i_private[3].altivec_u8, index); + r_.m128i_private[i].altivec_u8 = vec_sel(r01, r23, vec_cmpeq(vec_and(index, test), test)); + } + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t index, r, t; + const v128_t mask = wasm_i16x8_splat(0x001F); + const v128_t shift = wasm_i16x8_splat(0x0202); + const v128_t byte_index = wasm_i16x8_splat(0x0100); + const v128_t sixteen = wasm_i8x16_splat(16); + const v128_t a0 = a_.m128i_private[0].wasm_v128; + const v128_t a1 = a_.m128i_private[1].wasm_v128; + const v128_t a2 = a_.m128i_private[2].wasm_v128; + const v128_t a3 = a_.m128i_private[3].wasm_v128; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + index = wasm_v128_and(idx_.m128i_private[i].wasm_v128, mask); + index = wasm_i16x8_mul(index, shift); + index = wasm_i16x8_add(index, byte_index); + r = wasm_i8x16_swizzle(a0, index); + + index = wasm_i8x16_sub(index, sixteen); + t = wasm_i8x16_swizzle(a1, index); + r = wasm_v128_or(r, t); + + index = wasm_i8x16_sub(index, sixteen); + t = wasm_i8x16_swizzle(a2, index); + r = wasm_v128_or(r, t); + + index = wasm_i8x16_sub(index, sixteen); + t = wasm_i8x16_swizzle(a3, index); + r_.m128i_private[i].wasm_v128 = wasm_v128_or(r, t); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[idx_.i16[i] & 0x1F]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_permutexvar_epi16 + #define _mm512_permutexvar_epi16(idx, a) simde_mm512_permutexvar_epi16(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_permutexvar_epi16 (simde__m512i src, simde__mmask32 k, simde__m512i idx, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_permutexvar_epi16(src, k, idx, a); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_permutexvar_epi16(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_permutexvar_epi16 + #define _mm512_mask_permutexvar_epi16(src, k, idx, a) simde_mm512_mask_permutexvar_epi16(src, k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_permutexvar_epi16 (simde__mmask32 k, simde__m512i idx, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_permutexvar_epi16(k, idx, a); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_permutexvar_epi16(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_permutexvar_epi16 + #define _mm512_maskz_permutexvar_epi16(k, idx, a) simde_mm512_maskz_permutexvar_epi16(k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_permutexvar_epi32 (simde__m512i idx, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_permutexvar_epi32(idx, a); + #else + simde__m512i_private + idx_ = simde__m512i_to_private(idx), + a_ = simde__m512i_to_private(a), + r_; + + #if defined(SIMDE_X86_AVX2_NATIVE) + simde__m256i index, r0, r1, select; + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i_private) / sizeof(r_.m256i_private[0])) ; i++) { + index = idx_.m256i[i]; + r0 = simde_mm256_permutevar8x32_epi32(a_.m256i[0], index); + r1 = simde_mm256_permutevar8x32_epi32(a_.m256i[1], index); + select = simde_mm256_slli_epi32(index, 28); + r_.m256i[i] = simde_mm256_castps_si256(simde_mm256_blendv_ps(simde_mm256_castsi256_ps(r0), + simde_mm256_castsi256_ps(r1), + simde_mm256_castsi256_ps(select))); + } + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint8x16x4_t table = { { a_.m128i_private[0].neon_u8, + a_.m128i_private[1].neon_u8, + a_.m128i_private[2].neon_u8, + a_.m128i_private[3].neon_u8 } }; + uint32x4_t mask32 = vdupq_n_u32(0x0000000F); + uint32x4_t byte_index32 = vdupq_n_u32(0x03020100); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + uint32x4_t index32 = vandq_u32(idx_.m128i_private[i].neon_u32, mask32); + index32 = vmulq_n_u32(index32, 0x04040404); + index32 = vaddq_u32(index32, byte_index32); + r_.m128i_private[i].neon_u8 = vqtbl4q_u8(table, vreinterpretq_u8_u32(index32)); + } + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) index32, mask32, byte_index32, temp32, sixteen; + SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) zero, shift; + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) index, test, r01, r23; + mask32 = vec_splats(HEDLEY_STATIC_CAST(unsigned int, 0x0000000F)); + byte_index32 = vec_splats(HEDLEY_STATIC_CAST(unsigned int, 0x03020100)); + zero = vec_splat_u16(0); + shift = vec_splats(HEDLEY_STATIC_CAST(unsigned short, 0x0404)); + sixteen = vec_splats(HEDLEY_STATIC_CAST(unsigned int, 16)); + test = vec_splats(HEDLEY_STATIC_CAST(unsigned char, 0x20)); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + index32 = vec_and(idx_.m128i_private[i].altivec_u32, mask32); + + /* Multiply index32 by 0x04040404; unfortunately vec_mul isn't available so (mis)use 16-bit vec_mladd */ + temp32 = vec_sl(index32, sixteen); + index32 = vec_add(index32, temp32); + index32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned int), + vec_mladd(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned short), index32), + shift, + zero)); + + index32 = vec_add(index32, byte_index32); + index = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), index32); + r01 = vec_perm(a_.m128i_private[0].altivec_u8, a_.m128i_private[1].altivec_u8, index); + r23 = vec_perm(a_.m128i_private[2].altivec_u8, a_.m128i_private[3].altivec_u8, index); + r_.m128i_private[i].altivec_u8 = vec_sel(r01, r23, vec_cmpeq(vec_and(index, test), test)); + } + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t index, r, t; + const v128_t mask = wasm_i32x4_splat(0x0000000F); + const v128_t shift = wasm_i32x4_splat(0x04040404); + const v128_t byte_index = wasm_i32x4_splat(0x03020100); + const v128_t sixteen = wasm_i8x16_splat(16); + const v128_t a0 = a_.m128i_private[0].wasm_v128; + const v128_t a1 = a_.m128i_private[1].wasm_v128; + const v128_t a2 = a_.m128i_private[2].wasm_v128; + const v128_t a3 = a_.m128i_private[3].wasm_v128; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + index = wasm_v128_and(idx_.m128i_private[i].wasm_v128, mask); + index = wasm_i32x4_mul(index, shift); + index = wasm_i32x4_add(index, byte_index); + r = wasm_i8x16_swizzle(a0, index); + + index = wasm_i8x16_sub(index, sixteen); + t = wasm_i8x16_swizzle(a1, index); + r = wasm_v128_or(r, t); + + index = wasm_i8x16_sub(index, sixteen); + t = wasm_i8x16_swizzle(a2, index); + r = wasm_v128_or(r, t); + + index = wasm_i8x16_sub(index, sixteen); + t = wasm_i8x16_swizzle(a3, index); + r_.m128i_private[i].wasm_v128 = wasm_v128_or(r, t); + } + #else + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[idx_.i32[i] & 0x0F]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_permutexvar_epi32 + #define _mm512_permutexvar_epi32(idx, a) simde_mm512_permutexvar_epi32(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_permutexvar_epi32 (simde__m512i src, simde__mmask16 k, simde__m512i idx, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_permutexvar_epi32(src, k, idx, a); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_permutexvar_epi32(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_permutexvar_epi32 + #define _mm512_mask_permutexvar_epi32(src, k, idx, a) simde_mm512_mask_permutexvar_epi32(src, k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_permutexvar_epi32 (simde__mmask16 k, simde__m512i idx, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_permutexvar_epi32(k, idx, a); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_permutexvar_epi32(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_permutexvar_epi32 + #define _mm512_maskz_permutexvar_epi32(k, idx, a) simde_mm512_maskz_permutexvar_epi32(k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_permutexvar_epi64 (simde__m512i idx, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_permutexvar_epi64(idx, a); + #else + simde__m512i_private + idx_ = simde__m512i_to_private(idx), + a_ = simde__m512i_to_private(a), + r_; + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[idx_.i64[i] & 7]; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_permutexvar_epi64 + #define _mm512_permutexvar_epi64(idx, a) simde_mm512_permutexvar_epi64(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_permutexvar_epi64 (simde__m512i src, simde__mmask8 k, simde__m512i idx, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_permutexvar_epi64(src, k, idx, a); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_permutexvar_epi64(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_permutexvar_epi64 + #define _mm512_mask_permutexvar_epi64(src, k, idx, a) simde_mm512_mask_permutexvar_epi64(src, k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_permutexvar_epi64 (simde__mmask8 k, simde__m512i idx, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_permutexvar_epi64(k, idx, a); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_permutexvar_epi64(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_permutexvar_epi64 + #define _mm512_maskz_permutexvar_epi64(k, idx, a) simde_mm512_maskz_permutexvar_epi64(k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_permutexvar_epi8 (simde__m512i idx, simde__m512i a) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) + return _mm512_permutexvar_epi8(idx, a); + #elif defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + simde__m512i hilo, hi, lo, hi2, lo2, idx2; + simde__m512i ones = simde_mm512_set1_epi8(1); + simde__m512i low_bytes = simde_mm512_set1_epi16(0x00FF); + + idx2 = simde_mm512_srli_epi16(idx, 1); + hilo = simde_mm512_permutexvar_epi16(idx2, a); + simde__mmask64 mask = simde_mm512_test_epi8_mask(idx, ones); + lo = simde_mm512_and_si512(hilo, low_bytes); + hi = simde_mm512_srli_epi16(hilo, 8); + + idx2 = simde_mm512_srli_epi16(idx, 9); + hilo = simde_mm512_permutexvar_epi16(idx2, a); + lo2 = simde_mm512_slli_epi16(hilo, 8); + hi2 = simde_mm512_andnot_si512(low_bytes, hilo); + + lo = simde_mm512_or_si512(lo, lo2); + hi = simde_mm512_or_si512(hi, hi2); + + return simde_mm512_mask_blend_epi8(mask, lo, hi); + #else + simde__m512i_private + idx_ = simde__m512i_to_private(idx), + a_ = simde__m512i_to_private(a), + r_; + + #if defined(SIMDE_X86_AVX2_NATIVE) + simde__m256i t0, t1, index, select, a01, a23; + simde__m256i mask = simde_mm256_set1_epi8(0x3F); + simde__m256i a0 = simde_mm256_broadcastsi128_si256(a_.m128i[0]); + simde__m256i a1 = simde_mm256_broadcastsi128_si256(a_.m128i[1]); + simde__m256i a2 = simde_mm256_broadcastsi128_si256(a_.m128i[2]); + simde__m256i a3 = simde_mm256_broadcastsi128_si256(a_.m128i[3]); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i_private) / sizeof(r_.m256i_private[0])) ; i++) { + index = idx_.m256i[i]; + index = simde_mm256_and_si256(index, mask); + select = simde_mm256_slli_epi64(index, 3); + t0 = simde_mm256_shuffle_epi8(a0, index); + t1 = simde_mm256_shuffle_epi8(a1, index); + a01 = simde_mm256_blendv_epi8(t0, t1, select); + t0 = simde_mm256_shuffle_epi8(a2, index); + t1 = simde_mm256_shuffle_epi8(a3, index); + a23 = simde_mm256_blendv_epi8(t0, t1, select); + select = simde_mm256_slli_epi64(index, 2); + r_.m256i[i] = simde_mm256_blendv_epi8(a01, a23, select); + } + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint8x16x4_t table = { { a_.m128i_private[0].neon_u8, + a_.m128i_private[1].neon_u8, + a_.m128i_private[2].neon_u8, + a_.m128i_private[3].neon_u8 } }; + uint8x16_t mask = vdupq_n_u8(0x3F); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].neon_u8 = vqtbl4q_u8(table, vandq_u8(idx_.m128i_private[i].neon_u8, mask)); + } + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) test, r01, r23; + test = vec_splats(HEDLEY_STATIC_CAST(unsigned char, 0x20)); + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r01 = vec_perm(a_.m128i_private[0].altivec_u8, a_.m128i_private[1].altivec_u8, idx_.m128i_private[i].altivec_u8); + r23 = vec_perm(a_.m128i_private[2].altivec_u8, a_.m128i_private[3].altivec_u8, idx_.m128i_private[i].altivec_u8); + r_.m128i_private[i].altivec_u8 = vec_sel(r01, r23, vec_cmpeq(vec_and(idx_.m128i_private[i].altivec_u8, test), test)); + } + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t index, r, t; + const v128_t mask = wasm_i8x16_splat(0x3F); + const v128_t sixteen = wasm_i8x16_splat(16); + const v128_t a0 = a_.m128i_private[0].wasm_v128; + const v128_t a1 = a_.m128i_private[1].wasm_v128; + const v128_t a2 = a_.m128i_private[2].wasm_v128; + const v128_t a3 = a_.m128i_private[3].wasm_v128; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + index = wasm_v128_and(idx_.m128i_private[i].wasm_v128, mask); + r = wasm_i8x16_swizzle(a0, index); + + index = wasm_i8x16_sub(index, sixteen); + t = wasm_i8x16_swizzle(a1, index); + r = wasm_v128_or(r, t); + + index = wasm_i8x16_sub(index, sixteen); + t = wasm_i8x16_swizzle(a2, index); + r = wasm_v128_or(r, t); + + index = wasm_i8x16_sub(index, sixteen); + t = wasm_i8x16_swizzle(a3, index); + r_.m128i_private[i].wasm_v128 = wasm_v128_or(r, t); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[idx_.i8[i] & 0x3F]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) + #undef _mm512_permutexvar_epi8 + #define _mm512_permutexvar_epi8(idx, a) simde_mm512_permutexvar_epi8(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_permutexvar_epi8 (simde__m512i src, simde__mmask64 k, simde__m512i idx, simde__m512i a) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) + return _mm512_mask_permutexvar_epi8(src, k, idx, a); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_permutexvar_epi8(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_permutexvar_epi8 + #define _mm512_mask_permutexvar_epi8(src, k, idx, a) simde_mm512_mask_permutexvar_epi8(src, k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_permutexvar_epi8 (simde__mmask64 k, simde__m512i idx, simde__m512i a) { + #if defined(SIMDE_X86_AVX512VBMI_NATIVE) + return _mm512_maskz_permutexvar_epi8(k, idx, a); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_permutexvar_epi8(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_permutexvar_epi8 + #define _mm512_maskz_permutexvar_epi8(k, idx, a) simde_mm512_maskz_permutexvar_epi8(k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_permutexvar_pd (simde__m512i idx, simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_permutexvar_pd(idx, a); + #else + return simde_mm512_castsi512_pd(simde_mm512_permutexvar_epi64(idx, simde_mm512_castpd_si512(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_permutexvar_pd + #define _mm512_permutexvar_pd(idx, a) simde_mm512_permutexvar_pd(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_permutexvar_pd (simde__m512d src, simde__mmask8 k, simde__m512i idx, simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_permutexvar_pd(src, k, idx, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_permutexvar_pd(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_permutexvar_pd + #define _mm512_mask_permutexvar_pd(src, k, idx, a) simde_mm512_mask_permutexvar_pd(src, k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_permutexvar_pd (simde__mmask8 k, simde__m512i idx, simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_permutexvar_pd(k, idx, a); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_permutexvar_pd(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_permutexvar_pd + #define _mm512_maskz_permutexvar_pd(k, idx, a) simde_mm512_maskz_permutexvar_pd(k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_permutexvar_ps (simde__m512i idx, simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_permutexvar_ps(idx, a); + #else + return simde_mm512_castsi512_ps(simde_mm512_permutexvar_epi32(idx, simde_mm512_castps_si512(a))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_permutexvar_ps + #define _mm512_permutexvar_ps(idx, a) simde_mm512_permutexvar_ps(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512h +simde_mm512_permutexvar_ph (simde__m512i idx, simde__m512h a) { + #if defined(SIMDE_X86_AVX512FP16_NATIVE) + return _mm512_permutexvar_ph(idx, a); + #else + return simde_mm512_castsi512_ph(simde_mm512_permutexvar_epi16(idx, simde_mm512_castph_si512(a))); + #endif +} +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) + #undef _mm512_permutexvar_ph + #define _mm512_permutexvar_ph(idx, a) simde_mm512_permutexvar_ph(idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_permutexvar_ps (simde__m512 src, simde__mmask16 k, simde__m512i idx, simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_permutexvar_ps(src, k, idx, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_permutexvar_ps(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_permutexvar_ps + #define _mm512_mask_permutexvar_ps(src, k, idx, a) simde_mm512_mask_permutexvar_ps(src, k, idx, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_permutexvar_ps (simde__mmask16 k, simde__m512i idx, simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_permutexvar_ps(k, idx, a); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_permutexvar_ps(idx, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_permutexvar_ps + #define _mm512_maskz_permutexvar_ps(k, idx, a) simde_mm512_maskz_permutexvar_ps(k, idx, a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_PERMUTEXVAR_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/popcnt.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/popcnt.h new file mode 100644 index 00000000..b3c81253 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/popcnt.h @@ -0,0 +1,1346 @@ +#if !defined(SIMDE_X86_AVX512_POPCNT_H) +#define SIMDE_X86_AVX512_POPCNT_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_popcnt_epi8 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_popcnt_epi8(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vcntq_s8(a_.neon_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_popcnt(a_.wasm_v128); + #elif defined(SIMDE_X86_SSSE3_NATIVE) + const __m128i low_nibble_set = _mm_set1_epi8(0x0f); + const __m128i high_nibble_of_input = _mm_andnot_si128(low_nibble_set, a_.n); + const __m128i low_nibble_of_input = _mm_and_si128(low_nibble_set, a_.n); + const __m128i lut = _mm_set_epi8(4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0); + + r_.n = + _mm_add_epi8( + _mm_shuffle_epi8( + lut, + low_nibble_of_input + ), + _mm_shuffle_epi8( + lut, + _mm_srli_epi16( + high_nibble_of_input, + 4 + ) + ) + ); + #elif defined(SIMDE_X86_SSE2_NATIVE) + /* v -= ((v >> 1) & UINT8_C(0x55)); */ + r_.n = + _mm_sub_epi8( + a_.n, + _mm_and_si128( + _mm_srli_epi16(a_.n, 1), + _mm_set1_epi8(0x55) + ) + ); + + /* v = (v & 0x33) + ((v >> 2) & 0x33); */ + r_.n = + _mm_add_epi8( + _mm_and_si128( + r_.n, + _mm_set1_epi8(0x33) + ), + _mm_and_si128( + _mm_srli_epi16(r_.n, 2), + _mm_set1_epi8(0x33) + ) + ); + + /* v = (v + (v >> 4)) & 0xf */ + r_.n = + _mm_and_si128( + _mm_add_epi8( + r_.n, + _mm_srli_epi16(r_.n, 4) + ), + _mm_set1_epi8(0x0f) + ); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + r_.altivec_i8 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed char), vec_popcnt(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), a_.altivec_i8))); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + a_.u8 -= ((a_.u8 >> 1) & 0x55); + a_.u8 = ((a_.u8 & 0x33) + ((a_.u8 >> 2) & 0x33)); + a_.u8 = (a_.u8 + (a_.u8 >> 4)) & 15; + r_.u8 = a_.u8 >> ((sizeof(uint8_t) - 1) * CHAR_BIT); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + uint8_t v = HEDLEY_STATIC_CAST(uint8_t, a_.u8[i]); + v -= ((v >> 1) & 0x55); + v = (v & 0x33) + ((v >> 2) & 0x33); + v = (v + (v >> 4)) & 0xf; + r_.u8[i] = v >> (sizeof(uint8_t) - 1) * CHAR_BIT; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_popcnt_epi8 + #define _mm_popcnt_epi8(a) simde_mm_popcnt_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_popcnt_epi8 (simde__m128i src, simde__mmask16 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_popcnt_epi8(src, k, a); + #else + return simde_mm_mask_mov_epi8(src, k, simde_mm_popcnt_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_popcnt_epi8 + #define _mm_mask_popcnt_epi8(src, k, a) simde_mm_mask_popcnt_epi8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_popcnt_epi8 (simde__mmask16 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_popcnt_epi8(k, a); + #else + return simde_mm_maskz_mov_epi8(k, simde_mm_popcnt_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_popcnt_epi8 + #define _mm_maskz_popcnt_epi8(k, a) simde_mm_maskz_popcnt_epi8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_popcnt_epi16 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_popcnt_epi16(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vpaddlq_s8(vcntq_s8(a_.neon_i8)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_extadd_pairwise_i8x16(wasm_i8x16_popcnt(a_.wasm_v128)); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + r_.altivec_u16 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned short), vec_popcnt(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned short), a_.altivec_u16))); + #elif defined(SIMDE_X86_XOP_NATIVE) + const __m128i low_nibble_set = _mm_set1_epi8(0x0f); + const __m128i high_nibble_of_input = _mm_andnot_si128(low_nibble_set, a_.n); + const __m128i low_nibble_of_input = _mm_and_si128(low_nibble_set, a_.n); + const __m128i lut = _mm_set_epi8(4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0); + + r_.n = + _mm_haddw_epi8( + _mm_add_epi8( + _mm_shuffle_epi8( + lut, + low_nibble_of_input + ), + _mm_shuffle_epi8( + lut, + _mm_srli_epi16(high_nibble_of_input, 4) + ) + ) + ); + #elif defined(SIMDE_X86_SSE2_NATIVE) + r_.n = + _mm_sub_epi16( + a_.n, + _mm_and_si128( + _mm_srli_epi16(a_.n, 1), + _mm_set1_epi16(0x5555) + ) + ); + + r_.n = + _mm_add_epi16( + _mm_and_si128( + r_.n, + _mm_set1_epi16(0x3333) + ), + _mm_and_si128( + _mm_srli_epi16(r_.n, 2), + _mm_set1_epi16(0x3333) + ) + ); + + r_.n = + _mm_and_si128( + _mm_add_epi16( + r_.n, + _mm_srli_epi16(r_.n, 4) + ), + _mm_set1_epi16(0x0f0f) + ); + + r_.n = + _mm_srli_epi16( + _mm_mullo_epi16( + r_.n, + _mm_set1_epi16(0x0101) + ), + (sizeof(uint16_t) - 1) * CHAR_BIT + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + a_.u16 -= ((a_.u16 >> 1) & UINT16_C(0x5555)); + a_.u16 = ((a_.u16 & UINT16_C(0x3333)) + ((a_.u16 >> 2) & UINT16_C(0x3333))); + a_.u16 = (a_.u16 + (a_.u16 >> 4)) & UINT16_C(0x0f0f); + r_.u16 = (a_.u16 * UINT16_C(0x0101)) >> ((sizeof(uint16_t) - 1) * CHAR_BIT); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + uint16_t v = HEDLEY_STATIC_CAST(uint16_t, a_.u16[i]); + v -= ((v >> 1) & UINT16_C(0x5555)); + v = ((v & UINT16_C(0x3333)) + ((v >> 2) & UINT16_C(0x3333))); + v = (v + (v >> 4)) & UINT16_C(0x0f0f); + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, (v * UINT16_C(0x0101))) >> ((sizeof(uint16_t) - 1) * CHAR_BIT); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_popcnt_epi16 + #define _mm_popcnt_epi16(a) simde_mm_popcnt_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_popcnt_epi16 (simde__m128i src, simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_popcnt_epi16(src, k, a); + #else + return simde_mm_mask_mov_epi16(src, k, simde_mm_popcnt_epi16(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_popcnt_epi16 + #define _mm_mask_popcnt_epi16(src, k, a) simde_mm_mask_popcnt_epi16(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_popcnt_epi16 (simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_popcnt_epi16(k, a); + #else + return simde_mm_maskz_mov_epi16(k, simde_mm_popcnt_epi16(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_popcnt_epi16 + #define _mm_maskz_popcnt_epi16(k, a) simde_mm_maskz_popcnt_epi16(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_popcnt_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_popcnt_epi32(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vpaddlq_s16(vpaddlq_s8(vcntq_s8(a_.neon_i8))); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + r_.altivec_u32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned int), vec_popcnt(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned int), a_.altivec_u32))); + #elif defined(SIMDE_X86_XOP_NATIVE) + const __m128i low_nibble_set = _mm_set1_epi8(0x0f); + const __m128i high_nibble_of_input = _mm_andnot_si128(low_nibble_set, a_.n); + const __m128i low_nibble_of_input = _mm_and_si128(low_nibble_set, a_.n); + const __m128i lut = _mm_set_epi8(4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0); + + r_.n = + _mm_haddd_epi8( + _mm_add_epi8( + _mm_shuffle_epi8( + lut, + low_nibble_of_input + ), + _mm_shuffle_epi8( + lut, + _mm_srli_epi16(high_nibble_of_input, 4) + ) + ) + ); + #elif defined(SIMDE_X86_SSE4_1_NATIVE) + r_.n = + _mm_sub_epi32( + a_.n, + _mm_and_si128( + _mm_srli_epi32(a_.n, 1), + _mm_set1_epi32(0x55555555) + ) + ); + + r_.n = + _mm_add_epi32( + _mm_and_si128( + r_.n, + _mm_set1_epi32(0x33333333) + ), + _mm_and_si128( + _mm_srli_epi32(r_.n, 2), + _mm_set1_epi32(0x33333333) + ) + ); + + r_.n = + _mm_and_si128( + _mm_add_epi32( + r_.n, + _mm_srli_epi32(r_.n, 4) + ), + _mm_set1_epi32(0x0f0f0f0f) + ); + + r_.n = + _mm_srli_epi32( + _mm_mullo_epi32( + r_.n, + _mm_set1_epi32(0x01010101) + ), + (sizeof(uint32_t) - 1) * CHAR_BIT + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + a_.u32 -= ((a_.u32 >> 1) & UINT32_C(0x55555555)); + a_.u32 = ((a_.u32 & UINT32_C(0x33333333)) + ((a_.u32 >> 2) & UINT32_C(0x33333333))); + a_.u32 = (a_.u32 + (a_.u32 >> 4)) & UINT32_C(0x0f0f0f0f); + r_.u32 = (a_.u32 * UINT32_C(0x01010101)) >> ((sizeof(uint32_t) - 1) * CHAR_BIT); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + uint32_t v = HEDLEY_STATIC_CAST(uint32_t, a_.u32[i]); + v -= ((v >> 1) & UINT32_C(0x55555555)); + v = ((v & UINT32_C(0x33333333)) + ((v >> 2) & UINT32_C(0x33333333))); + v = (v + (v >> 4)) & UINT32_C(0x0f0f0f0f); + r_.u32[i] = HEDLEY_STATIC_CAST(uint32_t, (v * UINT32_C(0x01010101))) >> ((sizeof(uint32_t) - 1) * CHAR_BIT); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_popcnt_epi32 + #define _mm_popcnt_epi32(a) simde_mm_popcnt_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_popcnt_epi32 (simde__m128i src, simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_popcnt_epi32(src, k, a); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_popcnt_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_popcnt_epi32 + #define _mm_mask_popcnt_epi32(src, k, a) simde_mm_mask_popcnt_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_popcnt_epi32 (simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_popcnt_epi32(k, a); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_popcnt_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_popcnt_epi32 + #define _mm_maskz_popcnt_epi32(k, a) simde_mm_maskz_popcnt_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_popcnt_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_popcnt_epi64(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vpaddlq_s32(vpaddlq_s16(vpaddlq_s8(vcntq_s8(a_.neon_i8)))); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + r_.altivec_u64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long), vec_popcnt(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long), a_.altivec_u64))); + #elif defined(SIMDE_X86_SSSE3_NATIVE) + const __m128i low_nibble_set = _mm_set1_epi8(0x0f); + const __m128i high_nibble_of_input = _mm_andnot_si128(low_nibble_set, a_.n); + const __m128i low_nibble_of_input = _mm_and_si128(low_nibble_set, a_.n); + const __m128i lut = _mm_set_epi8(4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0); + + r_.n = + _mm_sad_epu8( + _mm_add_epi8( + _mm_shuffle_epi8( + lut, + low_nibble_of_input + ), + _mm_shuffle_epi8( + lut, + _mm_srli_epi16(high_nibble_of_input, 4) + ) + ), + _mm_setzero_si128() + ); + #elif defined(SIMDE_X86_SSE2_NATIVE) + r_.n = + _mm_sub_epi8( + a_.n, + _mm_and_si128( + _mm_srli_epi16(a_.n, 1), + _mm_set1_epi8(0x55) + ) + ); + + r_.n = + _mm_add_epi8( + _mm_and_si128( + r_.n, + _mm_set1_epi8(0x33) + ), + _mm_and_si128( + _mm_srli_epi16(r_.n, 2), + _mm_set1_epi8(0x33) + ) + ); + + r_.n = + _mm_and_si128( + _mm_add_epi8( + r_.n, + _mm_srli_epi16(r_.n, 4) + ), + _mm_set1_epi8(0x0f) + ); + + r_.n = + _mm_sad_epu8( + r_.n, + _mm_setzero_si128() + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + a_.u64 -= ((a_.u64 >> 1) & UINT64_C(0x5555555555555555)); + a_.u64 = ((a_.u64 & UINT64_C(0x3333333333333333)) + ((a_.u64 >> 2) & UINT64_C(0x3333333333333333))); + a_.u64 = (a_.u64 + (a_.u64 >> 4)) & UINT64_C(0x0f0f0f0f0f0f0f0f); + r_.u64 = (a_.u64 * UINT64_C(0x0101010101010101)) >> ((sizeof(uint64_t) - 1) * CHAR_BIT); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + uint64_t v = HEDLEY_STATIC_CAST(uint64_t, a_.u64[i]); + v -= ((v >> 1) & UINT64_C(0x5555555555555555)); + v = ((v & UINT64_C(0x3333333333333333)) + ((v >> 2) & UINT64_C(0x3333333333333333))); + v = (v + (v >> 4)) & UINT64_C(0x0f0f0f0f0f0f0f0f); + r_.u64[i] = HEDLEY_STATIC_CAST(uint64_t, (v * UINT64_C(0x0101010101010101))) >> ((sizeof(uint64_t) - 1) * CHAR_BIT); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_popcnt_epi64 + #define _mm_popcnt_epi64(a) simde_mm_popcnt_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_popcnt_epi64 (simde__m128i src, simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_popcnt_epi64(src, k, a); + #else + return simde_mm_mask_mov_epi64(src, k, simde_mm_popcnt_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_popcnt_epi64 + #define _mm_mask_popcnt_epi64(src, k, a) simde_mm_mask_popcnt_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_popcnt_epi64 (simde__mmask8 k, simde__m128i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_popcnt_epi64(k, a); + #else + return simde_mm_maskz_mov_epi64(k, simde_mm_popcnt_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_popcnt_epi64 + #define _mm_maskz_popcnt_epi64(k, a) simde_mm_maskz_popcnt_epi64(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_popcnt_epi8 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_popcnt_epi8(a); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_popcnt_epi8(a_.m128i[i]); + } + #elif defined(SIMDE_X86_AVX2_NATIVE) + const __m256i low_nibble_set = _mm256_set1_epi8(0x0f); + const __m256i high_nibble_of_input = _mm256_andnot_si256(low_nibble_set, a_.n); + const __m256i low_nibble_of_input = _mm256_and_si256(low_nibble_set, a_.n); + const __m256i lut = + _mm256_set_epi8( + 4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0, + 4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0 + ); + + r_.n = + _mm256_add_epi8( + _mm256_shuffle_epi8( + lut, + low_nibble_of_input + ), + _mm256_shuffle_epi8( + lut, + _mm256_srli_epi16( + high_nibble_of_input, + 4 + ) + ) + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + a_.u8 -= ((a_.u8 >> 1) & 0x55); + a_.u8 = ((a_.u8 & 0x33) + ((a_.u8 >> 2) & 0x33)); + a_.u8 = (a_.u8 + (a_.u8 >> 4)) & 15; + r_.u8 = a_.u8 >> ((sizeof(uint8_t) - 1) * CHAR_BIT); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + uint8_t v = HEDLEY_STATIC_CAST(uint8_t, a_.u8[i]); + v -= ((v >> 1) & 0x55); + v = (v & 0x33) + ((v >> 2) & 0x33); + v = (v + (v >> 4)) & 0xf; + r_.u8[i] = v >> (sizeof(uint8_t) - 1) * CHAR_BIT; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_popcnt_epi8 + #define _mm256_popcnt_epi8(a) simde_mm256_popcnt_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_popcnt_epi8 (simde__m256i src, simde__mmask32 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_popcnt_epi8(src, k, a); + #else + return simde_mm256_mask_mov_epi8(src, k, simde_mm256_popcnt_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_popcnt_epi8 + #define _mm256_mask_popcnt_epi8(src, k, a) simde_mm256_mask_popcnt_epi8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_popcnt_epi8 (simde__mmask32 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_popcnt_epi8(k, a); + #else + return simde_mm256_maskz_mov_epi8(k, simde_mm256_popcnt_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_popcnt_epi8 + #define _mm256_maskz_popcnt_epi8(k, a) simde_mm256_maskz_popcnt_epi8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_popcnt_epi16 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_popcnt_epi16(a); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_popcnt_epi16(a_.m128i[i]); + } + #elif defined(SIMDE_X86_AVX2_NATIVE) + r_.n = + _mm256_sub_epi16( + a_.n, + _mm256_and_si256( + _mm256_srli_epi16(a_.n, 1), + _mm256_set1_epi16(0x5555) + ) + ); + + r_.n = + _mm256_add_epi16( + _mm256_and_si256( + r_.n, + _mm256_set1_epi16(0x3333) + ), + _mm256_and_si256( + _mm256_srli_epi16(r_.n, 2), + _mm256_set1_epi16(0x3333) + ) + ); + + r_.n = + _mm256_and_si256( + _mm256_add_epi16( + r_.n, + _mm256_srli_epi16(r_.n, 4) + ), + _mm256_set1_epi16(0x0f0f) + ); + + r_.n = + _mm256_srli_epi16( + _mm256_mullo_epi16( + r_.n, + _mm256_set1_epi16(0x0101) + ), + (sizeof(uint16_t) - 1) * CHAR_BIT + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + a_.u16 -= ((a_.u16 >> 1) & UINT16_C(0x5555)); + a_.u16 = ((a_.u16 & UINT16_C(0x3333)) + ((a_.u16 >> 2) & UINT16_C(0x3333))); + a_.u16 = (a_.u16 + (a_.u16 >> 4)) & UINT16_C(0x0f0f); + r_.u16 = (a_.u16 * UINT16_C(0x0101)) >> ((sizeof(uint16_t) - 1) * CHAR_BIT); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + uint16_t v = HEDLEY_STATIC_CAST(uint16_t, a_.u16[i]); + v -= ((v >> 1) & UINT16_C(0x5555)); + v = ((v & UINT16_C(0x3333)) + ((v >> 2) & UINT16_C(0x3333))); + v = (v + (v >> 4)) & UINT16_C(0x0f0f); + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, (v * UINT16_C(0x0101))) >> ((sizeof(uint16_t) - 1) * CHAR_BIT); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_popcnt_epi16 + #define _mm256_popcnt_epi16(a) simde_mm256_popcnt_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_popcnt_epi16 (simde__m256i src, simde__mmask16 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_popcnt_epi16(src, k, a); + #else + return simde_mm256_mask_mov_epi16(src, k, simde_mm256_popcnt_epi16(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_popcnt_epi16 + #define _mm256_mask_popcnt_epi16(src, k, a) simde_mm256_mask_popcnt_epi16(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_popcnt_epi16 (simde__mmask16 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_popcnt_epi16(k, a); + #else + return simde_mm256_maskz_mov_epi16(k, simde_mm256_popcnt_epi16(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_popcnt_epi16 + #define _mm256_maskz_popcnt_epi16(k, a) simde_mm256_maskz_popcnt_epi16(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_popcnt_epi32 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_popcnt_epi32(a); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_popcnt_epi32(a_.m128i[i]); + } + #elif defined(SIMDE_X86_AVX2_NATIVE) + r_.n = + _mm256_sub_epi32( + a_.n, + _mm256_and_si256( + _mm256_srli_epi32(a_.n, 1), + _mm256_set1_epi32(0x55555555) + ) + ); + + r_.n = + _mm256_add_epi32( + _mm256_and_si256( + r_.n, + _mm256_set1_epi32(0x33333333) + ), + _mm256_and_si256( + _mm256_srli_epi32(r_.n, 2), + _mm256_set1_epi32(0x33333333) + ) + ); + + r_.n = + _mm256_and_si256( + _mm256_add_epi32( + r_.n, + _mm256_srli_epi32(r_.n, 4) + ), + _mm256_set1_epi32(0x0f0f0f0f) + ); + + r_.n = + _mm256_srli_epi32( + _mm256_mullo_epi32( + r_.n, + _mm256_set1_epi32(0x01010101) + ), + (sizeof(uint32_t) - 1) * CHAR_BIT + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + a_.u32 -= ((a_.u32 >> 1) & UINT32_C(0x55555555)); + a_.u32 = ((a_.u32 & UINT32_C(0x33333333)) + ((a_.u32 >> 2) & UINT32_C(0x33333333))); + a_.u32 = (a_.u32 + (a_.u32 >> 4)) & UINT32_C(0x0f0f0f0f); + r_.u32 = (a_.u32 * UINT32_C(0x01010101)) >> ((sizeof(uint32_t) - 1) * CHAR_BIT); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + uint32_t v = HEDLEY_STATIC_CAST(uint32_t, a_.u32[i]); + v -= ((v >> 1) & UINT32_C(0x55555555)); + v = ((v & UINT32_C(0x33333333)) + ((v >> 2) & UINT32_C(0x33333333))); + v = (v + (v >> 4)) & UINT32_C(0x0f0f0f0f); + r_.u32[i] = HEDLEY_STATIC_CAST(uint32_t, (v * UINT32_C(0x01010101))) >> ((sizeof(uint32_t) - 1) * CHAR_BIT); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_popcnt_epi32 + #define _mm256_popcnt_epi32(a) simde_mm256_popcnt_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_popcnt_epi32 (simde__m256i src, simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_popcnt_epi32(src, k, a); + #else + return simde_mm256_mask_mov_epi32(src, k, simde_mm256_popcnt_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_popcnt_epi32 + #define _mm256_mask_popcnt_epi32(src, k, a) simde_mm256_mask_popcnt_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_popcnt_epi32 (simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_popcnt_epi32(k, a); + #else + return simde_mm256_maskz_mov_epi32(k, simde_mm256_popcnt_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_popcnt_epi32 + #define _mm256_maskz_popcnt_epi32(k, a) simde_mm256_maskz_popcnt_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_popcnt_epi64 (simde__m256i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_popcnt_epi64(a); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < sizeof(r_.m128i) / sizeof(r_.m128i[0]) ; i++) { + r_.m128i[i] = simde_mm_popcnt_epi64(a_.m128i[i]); + } + #elif defined(SIMDE_X86_AVX2_NATIVE) + const __m256i low_nibble_set = _mm256_set1_epi8(0x0f); + const __m256i high_nibble_of_input = _mm256_andnot_si256(low_nibble_set, a_.n); + const __m256i low_nibble_of_input = _mm256_and_si256(low_nibble_set, a_.n); + const __m256i lut = + _mm256_set_epi8( + 4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0, + 4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0 + ); + + r_.n = + _mm256_sad_epu8( + _mm256_add_epi8( + _mm256_shuffle_epi8( + lut, + low_nibble_of_input + ), + _mm256_shuffle_epi8( + lut, + _mm256_srli_epi16(high_nibble_of_input, 4) + ) + ), + _mm256_setzero_si256() + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + a_.u64 -= ((a_.u64 >> 1) & UINT64_C(0x5555555555555555)); + a_.u64 = ((a_.u64 & UINT64_C(0x3333333333333333)) + ((a_.u64 >> 2) & UINT64_C(0x3333333333333333))); + a_.u64 = (a_.u64 + (a_.u64 >> 4)) & UINT64_C(0x0f0f0f0f0f0f0f0f); + r_.u64 = (a_.u64 * UINT64_C(0x0101010101010101)) >> ((sizeof(uint64_t) - 1) * CHAR_BIT); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + uint64_t v = HEDLEY_STATIC_CAST(uint64_t, a_.u64[i]); + v -= ((v >> 1) & UINT64_C(0x5555555555555555)); + v = ((v & UINT64_C(0x3333333333333333)) + ((v >> 2) & UINT64_C(0x3333333333333333))); + v = (v + (v >> 4)) & UINT64_C(0x0f0f0f0f0f0f0f0f); + r_.u64[i] = HEDLEY_STATIC_CAST(uint64_t, (v * UINT64_C(0x0101010101010101))) >> ((sizeof(uint64_t) - 1) * CHAR_BIT); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_popcnt_epi64 + #define _mm256_popcnt_epi64(a) simde_mm256_popcnt_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_popcnt_epi64 (simde__m256i src, simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_popcnt_epi64(src, k, a); + #else + return simde_mm256_mask_mov_epi64(src, k, simde_mm256_popcnt_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_popcnt_epi64 + #define _mm256_mask_popcnt_epi64(src, k, a) simde_mm256_mask_popcnt_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_popcnt_epi64 (simde__mmask8 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_popcnt_epi64(k, a); + #else + return simde_mm256_maskz_mov_epi64(k, simde_mm256_popcnt_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_popcnt_epi64 + #define _mm256_maskz_popcnt_epi64(k, a) simde_mm256_maskz_popcnt_epi64(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_popcnt_epi8 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) + return _mm512_popcnt_epi8(a); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_popcnt_epi8(a_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_popcnt_epi8(a_.m256i[i]); + } + #elif defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + const __m512i low_nibble_set = _mm512_set1_epi8(0x0f); + const __m512i high_nibble_of_input = _mm512_andnot_si512(low_nibble_set, a_.n); + const __m512i low_nibble_of_input = _mm512_and_si512(low_nibble_set, a_.n); + const __m512i lut = + simde_mm512_set_epi8( + 4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0, + 4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0, + 4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0, + 4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0 + ); + + r_.n = + _mm512_add_epi8( + _mm512_shuffle_epi8( + lut, + low_nibble_of_input + ), + _mm512_shuffle_epi8( + lut, + _mm512_srli_epi16( + high_nibble_of_input, + 4 + ) + ) + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + a_.u8 -= ((a_.u8 >> 1) & 0x55); + a_.u8 = ((a_.u8 & 0x33) + ((a_.u8 >> 2) & 0x33)); + a_.u8 = (a_.u8 + (a_.u8 >> 4)) & 15; + r_.u8 = a_.u8 >> ((sizeof(uint8_t) - 1) * CHAR_BIT); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + uint8_t v = HEDLEY_STATIC_CAST(uint8_t, a_.u8[i]); + v -= ((v >> 1) & 0x55); + v = (v & 0x33) + ((v >> 2) & 0x33); + v = (v + (v >> 4)) & 0xf; + r_.u8[i] = v >> (sizeof(uint8_t) - 1) * CHAR_BIT; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) + #undef _mm512_popcnt_epi8 + #define _mm512_popcnt_epi8(a) simde_mm512_popcnt_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_popcnt_epi8 (simde__m512i src, simde__mmask64 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) + return _mm512_mask_popcnt_epi8(src, k, a); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_popcnt_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_popcnt_epi8 + #define _mm512_mask_popcnt_epi8(src, k, a) simde_mm512_mask_popcnt_epi8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_popcnt_epi8 (simde__mmask64 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) + return _mm512_maskz_popcnt_epi8(k, a); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_popcnt_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_popcnt_epi8 + #define _mm512_maskz_popcnt_epi8(k, a) simde_mm512_maskz_popcnt_epi8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_popcnt_epi16 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) + return _mm512_popcnt_epi16(a); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_popcnt_epi16(a_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_popcnt_epi16(a_.m256i[i]); + } + #elif defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + r_.n = + _mm512_sub_epi16( + a_.n, + _mm512_and_si512( + _mm512_srli_epi16(a_.n, 1), + _mm512_set1_epi16(0x5555) + ) + ); + + r_.n = + _mm512_add_epi16( + _mm512_and_si512( + r_.n, + _mm512_set1_epi16(0x3333) + ), + _mm512_and_si512( + _mm512_srli_epi16(r_.n, 2), + _mm512_set1_epi16(0x3333) + ) + ); + + r_.n = + _mm512_and_si512( + _mm512_add_epi16( + r_.n, + _mm512_srli_epi16(r_.n, 4) + ), + _mm512_set1_epi16(0x0f0f) + ); + + r_.n = + _mm512_srli_epi16( + _mm512_mullo_epi16( + r_.n, + _mm512_set1_epi16(0x0101) + ), + (sizeof(uint16_t) - 1) * CHAR_BIT + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + a_.u16 -= ((a_.u16 >> 1) & UINT16_C(0x5555)); + a_.u16 = ((a_.u16 & UINT16_C(0x3333)) + ((a_.u16 >> 2) & UINT16_C(0x3333))); + a_.u16 = (a_.u16 + (a_.u16 >> 4)) & UINT16_C(0x0f0f); + r_.u16 = (a_.u16 * UINT16_C(0x0101)) >> ((sizeof(uint16_t) - 1) * CHAR_BIT); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + uint16_t v = HEDLEY_STATIC_CAST(uint16_t, a_.u16[i]); + v -= ((v >> 1) & UINT16_C(0x5555)); + v = ((v & UINT16_C(0x3333)) + ((v >> 2) & UINT16_C(0x3333))); + v = (v + (v >> 4)) & UINT16_C(0x0f0f); + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, (v * UINT16_C(0x0101))) >> ((sizeof(uint16_t) - 1) * CHAR_BIT); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) + #undef _mm512_popcnt_epi16 + #define _mm512_popcnt_epi16(a) simde_mm512_popcnt_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_popcnt_epi16 (simde__m512i src, simde__mmask32 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) + return _mm512_mask_popcnt_epi16(src, k, a); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_popcnt_epi16(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_popcnt_epi16 + #define _mm512_mask_popcnt_epi16(src, k, a) simde_mm512_mask_popcnt_epi16(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_popcnt_epi16 (simde__mmask32 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BITALG_NATIVE) + return _mm512_maskz_popcnt_epi16(k, a); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_popcnt_epi16(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BITALG_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_popcnt_epi16 + #define _mm512_maskz_popcnt_epi16(k, a) simde_mm512_maskz_popcnt_epi16(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_popcnt_epi32 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) + return _mm512_popcnt_epi32(a); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_popcnt_epi32(a_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_popcnt_epi32(a_.m256i[i]); + } + #elif defined(SIMDE_X86_AVX512F_NATIVE) + r_.n = + _mm512_sub_epi32( + a_.n, + _mm512_and_si512( + _mm512_srli_epi32(a_.n, 1), + _mm512_set1_epi32(0x55555555) + ) + ); + + r_.n = + _mm512_add_epi32( + _mm512_and_si512( + r_.n, + _mm512_set1_epi32(0x33333333) + ), + _mm512_and_si512( + _mm512_srli_epi32(r_.n, 2), + _mm512_set1_epi32(0x33333333) + ) + ); + + r_.n = + _mm512_and_si512( + _mm512_add_epi32( + r_.n, + _mm512_srli_epi32(r_.n, 4) + ), + _mm512_set1_epi32(0x0f0f0f0f) + ); + + r_.n = + _mm512_srli_epi32( + _mm512_mullo_epi32( + r_.n, + _mm512_set1_epi32(0x01010101) + ), + (sizeof(uint32_t) - 1) * CHAR_BIT + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + a_.u32 -= ((a_.u32 >> 1) & UINT32_C(0x55555555)); + a_.u32 = ((a_.u32 & UINT32_C(0x33333333)) + ((a_.u32 >> 2) & UINT32_C(0x33333333))); + a_.u32 = (a_.u32 + (a_.u32 >> 4)) & UINT32_C(0x0f0f0f0f); + r_.u32 = (a_.u32 * UINT32_C(0x01010101)) >> ((sizeof(uint32_t) - 1) * CHAR_BIT); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + uint32_t v = HEDLEY_STATIC_CAST(uint32_t, a_.u32[i]); + v -= ((v >> 1) & UINT32_C(0x55555555)); + v = ((v & UINT32_C(0x33333333)) + ((v >> 2) & UINT32_C(0x33333333))); + v = (v + (v >> 4)) & UINT32_C(0x0f0f0f0f); + r_.u32[i] = HEDLEY_STATIC_CAST(uint32_t, (v * UINT32_C(0x01010101))) >> ((sizeof(uint32_t) - 1) * CHAR_BIT); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_popcnt_epi32 + #define _mm512_popcnt_epi32(a) simde_mm512_popcnt_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_popcnt_epi32 (simde__m512i src, simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) + return _mm512_mask_popcnt_epi32(src, k, a); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_popcnt_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_popcnt_epi32 + #define _mm512_mask_popcnt_epi32(src, k, a) simde_mm512_mask_popcnt_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_popcnt_epi32 (simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) + return _mm512_maskz_popcnt_epi32(k, a); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_popcnt_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_popcnt_epi32 + #define _mm512_maskz_popcnt_epi32(k, a) simde_mm512_maskz_popcnt_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_popcnt_epi64 (simde__m512i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) + return _mm512_popcnt_epi64(a); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_popcnt_epi64(a_.m128i[i]); + } + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < sizeof(r_.m256i) / sizeof(r_.m256i[0]) ; i++) { + r_.m256i[i] = simde_mm256_popcnt_epi64(a_.m256i[i]); + } + #elif defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + const __m512i low_nibble_set = _mm512_set1_epi8(0x0f); + const __m512i high_nibble_of_input = _mm512_andnot_si512(low_nibble_set, a_.n); + const __m512i low_nibble_of_input = _mm512_and_si512(low_nibble_set, a_.n); + const __m512i lut = + simde_mm512_set_epi8( + 4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0, + 4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0, + 4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0, + 4, 3, 3, 2, 3, 2, 2, 1, 3, 2, 2, 1, 2, 1, 1, 0 + ); + + r_.n = + _mm512_sad_epu8( + _mm512_add_epi8( + _mm512_shuffle_epi8( + lut, + low_nibble_of_input + ), + _mm512_shuffle_epi8( + lut, + _mm512_srli_epi16(high_nibble_of_input, 4) + ) + ), + _mm512_setzero_si512() + ); + #elif defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512DQ_NATIVE) + r_.n = + _mm512_sub_epi64( + a_.n, + _mm512_and_si512( + _mm512_srli_epi64(a_.n, 1), + _mm512_set1_epi64(0x5555555555555555) + ) + ); + + r_.n = + _mm512_add_epi64( + _mm512_and_si512( + r_.n, + _mm512_set1_epi64(0x3333333333333333) + ), + _mm512_and_si512( + _mm512_srli_epi64(r_.n, 2), + _mm512_set1_epi64(0x3333333333333333) + ) + ); + + r_.n = + _mm512_and_si512( + _mm512_add_epi64( + r_.n, + _mm512_srli_epi64(r_.n, 4) + ), + _mm512_set1_epi64(0x0f0f0f0f0f0f0f0f) + ); + + r_.n = + _mm512_srli_epi64( + _mm512_mullo_epi64( + r_.n, + _mm512_set1_epi64(0x0101010101010101) + ), + (sizeof(uint64_t) - 1) * CHAR_BIT + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + a_.u64 -= ((a_.u64 >> 1) & UINT64_C(0x5555555555555555)); + a_.u64 = ((a_.u64 & UINT64_C(0x3333333333333333)) + ((a_.u64 >> 2) & UINT64_C(0x3333333333333333))); + a_.u64 = (a_.u64 + (a_.u64 >> 4)) & UINT64_C(0x0f0f0f0f0f0f0f0f); + r_.u64 = (a_.u64 * UINT64_C(0x0101010101010101)) >> ((sizeof(uint64_t) - 1) * CHAR_BIT); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + uint64_t v = HEDLEY_STATIC_CAST(uint64_t, a_.u64[i]); + v -= ((v >> 1) & UINT64_C(0x5555555555555555)); + v = ((v & UINT64_C(0x3333333333333333)) + ((v >> 2) & UINT64_C(0x3333333333333333))); + v = (v + (v >> 4)) & UINT64_C(0x0f0f0f0f0f0f0f0f); + r_.u64[i] = HEDLEY_STATIC_CAST(uint64_t, (v * UINT64_C(0x0101010101010101))) >> ((sizeof(uint64_t) - 1) * CHAR_BIT); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_popcnt_epi64 + #define _mm512_popcnt_epi64(a) simde_mm512_popcnt_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_popcnt_epi64 (simde__m512i src, simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) + return _mm512_mask_popcnt_epi64(src, k, a); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_popcnt_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_popcnt_epi64 + #define _mm512_mask_popcnt_epi64(src, k, a) simde_mm512_mask_popcnt_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_popcnt_epi64 (simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512VPOPCNTDQ_NATIVE) + return _mm512_maskz_popcnt_epi64(k, a); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_popcnt_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512VPOPCNTDQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_popcnt_epi64 + #define _mm512_maskz_popcnt_epi64(k, a) simde_mm512_maskz_popcnt_epi64(k, a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_POPCNT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/range.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/range.h new file mode 100644 index 00000000..1d8c0fb4 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/range.h @@ -0,0 +1,745 @@ +#if !defined(SIMDE_X86_AVX512_RANGE_H) +#define SIMDE_X86_AVX512_RANGE_H + +#include "types.h" +#include "max.h" +#include "min.h" +#include "set1.h" +#include "copysign.h" +#include "abs.h" +#include "setzero.h" +#include "cmp.h" +#include "or.h" +#include "andnot.h" +#include "insert.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_range_ps (simde__m128 a, simde__m128 b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m128 r; + + switch (imm8 & 3) { + case 0: + r = simde_mm_min_ps(a, b); + break; + case 1: + r = simde_mm_max_ps(a, b); + break; + case 2: + r = simde_x_mm_select_ps(b, a, simde_mm_cmple_ps(simde_x_mm_abs_ps(a), simde_x_mm_abs_ps(b))); + break; + case 3: + r = simde_x_mm_select_ps(b, a, simde_mm_cmpge_ps(simde_x_mm_abs_ps(a), simde_x_mm_abs_ps(b))); + break; + default: + break; + } + + switch (imm8 & 12) { + case 0: + r = simde_x_mm_copysign_ps(r, a); + break; + case 8: + r = simde_mm_andnot_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C(-0.0)), r); + break; + case 12: + r = simde_mm_or_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C(-0.0)), r); + break; + default: + break; + } + + return r; +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_range_ps(a, b, imm8) _mm_range_ps((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_range_ps + #define _mm_range_ps(a, b, imm8) simde_mm_range_ps(a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_mask_range_ps(src, k, a, b, imm8) _mm_mask_range_ps(src, k, a, b, imm8) +#else + #define simde_mm_mask_range_ps(src, k, a, b, imm8) simde_mm_mask_mov_ps(src, k, simde_mm_range_ps(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_range_ps + #define _mm_mask_range_ps(src, k, a, b, imm8) simde_mm_mask_range_ps(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_maskz_range_ps(k, a, b, imm8) _mm_maskz_range_ps(k, a, b, imm8) +#else + #define simde_mm_maskz_range_ps(k, a, b, imm8) simde_mm_maskz_mov_ps(k, simde_mm_range_ps(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_range_ps + #define _mm_maskz_range_ps(k, a, b, imm8) simde_mm_maskz_range_ps(k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_range_ps (simde__m256 a, simde__m256 b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m256 r; + + switch (imm8 & 3) { + case 0: + r = simde_mm256_min_ps(a, b); + break; + case 1: + r = simde_mm256_max_ps(a, b); + break; + case 2: + r = simde_x_mm256_select_ps(b, a, simde_mm256_cmp_ps(simde_x_mm256_abs_ps(a), simde_x_mm256_abs_ps(b), SIMDE_CMP_LE_OQ)); + break; + case 3: + r = simde_x_mm256_select_ps(b, a, simde_mm256_cmp_ps(simde_x_mm256_abs_ps(a), simde_x_mm256_abs_ps(b), SIMDE_CMP_GE_OQ)); + break; + default: + break; + } + + switch (imm8 & 12) { + case 0: + r = simde_x_mm256_copysign_ps(r, a); + break; + case 8: + r = simde_mm256_andnot_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(-0.0)), r); + break; + case 12: + r = simde_mm256_or_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(-0.0)), r); + break; + default: + break; + } + + return r; +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_range_ps(a, b, imm8) _mm256_range_ps((a), (b), (imm8)) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm256_range_ps(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m256_private \ + simde_mm256_range_ps_r_ = simde__m256_to_private(simde_mm256_setzero_ps()), \ + simde_mm256_range_ps_a_ = simde__m256_to_private(a), \ + simde_mm256_range_ps_b_ = simde__m256_to_private(b); \ + \ + for (size_t simde_mm256_range_ps_i = 0 ; simde_mm256_range_ps_i < (sizeof(simde_mm256_range_ps_r_.m128) / sizeof(simde_mm256_range_ps_r_.m128[0])) ; simde_mm256_range_ps_i++) { \ + simde_mm256_range_ps_r_.m128[simde_mm256_range_ps_i] = simde_mm_range_ps(simde_mm256_range_ps_a_.m128[simde_mm256_range_ps_i], simde_mm256_range_ps_b_.m128[simde_mm256_range_ps_i], imm8); \ + } \ + \ + simde__m256_from_private(simde_mm256_range_ps_r_); \ + })) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_range_ps + #define _mm256_range_ps(a, b, imm8) simde_mm256_range_ps(a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_range_ps(src, k, a, b, imm8) _mm256_mask_range_ps(src, k, a, b, imm8) +#else + #define simde_mm256_mask_range_ps(src, k, a, b, imm8) simde_mm256_mask_mov_ps(src, k, simde_mm256_range_ps(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_range_ps + #define _mm256_mask_range_ps(src, k, a, b, imm8) simde_mm256_mask_range_ps(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_maskz_range_ps(k, a, b, imm8) _mm256_maskz_range_ps(k, a, b, imm8) +#else + #define simde_mm256_maskz_range_ps(k, a, b, imm8) simde_mm256_maskz_mov_ps(k, simde_mm256_range_ps(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_range_ps + #define _mm256_maskz_range_ps(k, a, b, imm8) simde_mm256_maskz_range_ps(k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_range_ps (simde__m512 a, simde__m512 b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m512 r; + + switch (imm8 & 3) { + case 0: + r = simde_mm512_min_ps(a, b); + break; + case 1: + r = simde_mm512_max_ps(a, b); + break; + case 2: + r = simde_mm512_mask_mov_ps(b, simde_mm512_cmp_ps_mask(simde_mm512_abs_ps(a), simde_mm512_abs_ps(b), SIMDE_CMP_LE_OS), a); + break; + case 3: + r = simde_mm512_mask_mov_ps(a, simde_mm512_cmp_ps_mask(simde_mm512_abs_ps(b), simde_mm512_abs_ps(a), SIMDE_CMP_GE_OS), b); + break; + default: + break; + } + + switch (imm8 & 12) { + case 0: + r = simde_x_mm512_copysign_ps(r, a); + break; + case 8: + r = simde_mm512_andnot_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(-0.0)), r); + break; + case 12: + r = simde_mm512_or_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(-0.0)), r); + break; + default: + break; + } + + return r; +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm512_range_ps(a, b, imm8) _mm512_range_ps((a), (b), (imm8)) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_range_ps(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512_private \ + simde_mm512_range_ps_r_ = simde__m512_to_private(simde_mm512_setzero_ps()), \ + simde_mm512_range_ps_a_ = simde__m512_to_private(a), \ + simde_mm512_range_ps_b_ = simde__m512_to_private(b); \ + \ + for (size_t simde_mm512_range_ps_i = 0 ; simde_mm512_range_ps_i < (sizeof(simde_mm512_range_ps_r_.m128) / sizeof(simde_mm512_range_ps_r_.m128[0])) ; simde_mm512_range_ps_i++) { \ + simde_mm512_range_ps_r_.m128[simde_mm512_range_ps_i] = simde_mm_range_ps(simde_mm512_range_ps_a_.m128[simde_mm512_range_ps_i], simde_mm512_range_ps_b_.m128[simde_mm512_range_ps_i], imm8); \ + } \ + \ + simde__m512_from_private(simde_mm512_range_ps_r_); \ + })) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_range_ps(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512_private \ + simde_mm512_range_ps_r_ = simde__m512_to_private(simde_mm512_setzero_ps()), \ + simde_mm512_range_ps_a_ = simde__m512_to_private(a), \ + simde_mm512_range_ps_b_ = simde__m512_to_private(b); \ + \ + for (size_t simde_mm512_range_ps_i = 0 ; simde_mm512_range_ps_i < (sizeof(simde_mm512_range_ps_r_.m256) / sizeof(simde_mm512_range_ps_r_.m256[0])) ; simde_mm512_range_ps_i++) { \ + simde_mm512_range_ps_r_.m256[simde_mm512_range_ps_i] = simde_mm256_range_ps(simde_mm512_range_ps_a_.m256[simde_mm512_range_ps_i], simde_mm512_range_ps_b_.m256[simde_mm512_range_ps_i], imm8); \ + } \ + \ + simde__m512_from_private(simde_mm512_range_ps_r_); \ + })) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_range_ps + #define _mm512_range_ps(a, b, imm8) simde_mm512_range_ps(a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_mask_range_ps(src, k, a, b, imm8) _mm512_mask_range_ps(src, k, a, b, imm8) +#else + #define simde_mm512_mask_range_ps(src, k, a, b, imm8) simde_mm512_mask_mov_ps(src, k, simde_mm512_range_ps(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_range_ps + #define _mm512_mask_range_ps(src, k, a, b, imm8) simde_mm512_mask_range_ps(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_maskz_range_ps(k, a, b, imm8) _mm512_maskz_range_ps(k, a, b, imm8) +#else + #define simde_mm512_maskz_range_ps(k, a, b, imm8) simde_mm512_maskz_mov_ps(k, simde_mm512_range_ps(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_range_ps + #define _mm512_maskz_range_ps(k, a, b, imm8) simde_mm512_maskz_range_ps(k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_range_pd (simde__m128d a, simde__m128d b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m128d r; + + switch (imm8 & 3) { + case 0: + r = simde_mm_min_pd(a, b); + break; + case 1: + r = simde_mm_max_pd(a, b); + break; + case 2: + r = simde_x_mm_select_pd(b, a, simde_mm_cmple_pd(simde_x_mm_abs_pd(a), simde_x_mm_abs_pd(b))); + break; + case 3: + r = simde_x_mm_select_pd(b, a, simde_mm_cmpge_pd(simde_x_mm_abs_pd(a), simde_x_mm_abs_pd(b))); + break; + default: + break; + } + + switch (imm8 & 12) { + case 0: + r = simde_x_mm_copysign_pd(r, a); + break; + case 8: + r = simde_mm_andnot_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(-0.0)), r); + break; + case 12: + r = simde_mm_or_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(-0.0)), r); + break; + default: + break; + } + + return r; +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_range_pd(a, b, imm8) _mm_range_pd((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_range_pd + #define _mm_range_pd(a, b, imm8) simde_mm_range_pd(a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_mask_range_pd(src, k, a, b, imm8) _mm_mask_range_pd(src, k, a, b, imm8) +#else + #define simde_mm_mask_range_pd(src, k, a, b, imm8) simde_mm_mask_mov_pd(src, k, simde_mm_range_pd(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_range_pd + #define _mm_mask_range_pd(src, k, a, b, imm8) simde_mm_mask_range_pd(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_maskz_range_pd(k, a, b, imm8) _mm_maskz_range_pd(k, a, b, imm8) +#else + #define simde_mm_maskz_range_pd(k, a, b, imm8) simde_mm_maskz_mov_pd(k, simde_mm_range_pd(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_range_pd + #define _mm_maskz_range_pd(k, a, b, imm8) simde_mm_maskz_range_pd(k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_range_pd (simde__m256d a, simde__m256d b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m256d r; + + switch (imm8 & 3) { + case 0: + r = simde_mm256_min_pd(a, b); + break; + case 1: + r = simde_mm256_max_pd(a, b); + break; + case 2: + r = simde_x_mm256_select_pd(b, a, simde_mm256_cmp_pd(simde_x_mm256_abs_pd(a), simde_x_mm256_abs_pd(b), SIMDE_CMP_LE_OQ)); + break; + case 3: + r = simde_x_mm256_select_pd(b, a, simde_mm256_cmp_pd(simde_x_mm256_abs_pd(a), simde_x_mm256_abs_pd(b), SIMDE_CMP_GE_OQ)); + break; + default: + break; + } + + switch (imm8 & 12) { + case 0: + r = simde_x_mm256_copysign_pd(r, a); + break; + case 8: + r = simde_mm256_andnot_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(-0.0)), r); + break; + case 12: + r = simde_mm256_or_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(-0.0)), r); + break; + default: + break; + } + + return r; +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_range_pd(a, b, imm8) _mm256_range_pd((a), (b), (imm8)) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm256_range_pd(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m256d_private \ + simde_mm256_range_pd_r_ = simde__m256d_to_private(simde_mm256_setzero_pd()), \ + simde_mm256_range_pd_a_ = simde__m256d_to_private(a), \ + simde_mm256_range_pd_b_ = simde__m256d_to_private(b); \ + \ + for (size_t simde_mm256_range_pd_i = 0 ; simde_mm256_range_pd_i < (sizeof(simde_mm256_range_pd_r_.m128d) / sizeof(simde_mm256_range_pd_r_.m128d[0])) ; simde_mm256_range_pd_i++) { \ + simde_mm256_range_pd_r_.m128d[simde_mm256_range_pd_i] = simde_mm_range_pd(simde_mm256_range_pd_a_.m128d[simde_mm256_range_pd_i], simde_mm256_range_pd_b_.m128d[simde_mm256_range_pd_i], imm8); \ + } \ + \ + simde__m256d_from_private(simde_mm256_range_pd_r_); \ + })) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_range_pd + #define _mm256_range_pd(a, b, imm8) simde_mm256_range_pd(a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_range_pd(src, k, a, b, imm8) _mm256_mask_range_pd(src, k, a, b, imm8) +#else + #define simde_mm256_mask_range_pd(src, k, a, b, imm8) simde_mm256_mask_mov_pd(src, k, simde_mm256_range_pd(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_range_pd + #define _mm256_mask_range_pd(src, k, a, b, imm8) simde_mm256_mask_range_pd(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_maskz_range_pd(k, a, b, imm8) _mm256_maskz_range_pd(k, a, b, imm8) +#else + #define simde_mm256_maskz_range_pd(k, a, b, imm8) simde_mm256_maskz_mov_pd(k, simde_mm256_range_pd(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_range_pd + #define _mm256_maskz_range_pd(k, a, b, imm8) simde_mm256_maskz_range_pd(k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_range_pd (simde__m512d a, simde__m512d b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m512d r; + + switch (imm8 & 3) { + case 0: + r = simde_mm512_min_pd(a, b); + break; + case 1: + r = simde_mm512_max_pd(a, b); + break; + case 2: + r = simde_mm512_mask_mov_pd(b, simde_mm512_cmp_pd_mask(simde_mm512_abs_pd(a), simde_mm512_abs_pd(b), SIMDE_CMP_LE_OS), a); + break; + case 3: + r = simde_mm512_mask_mov_pd(a, simde_mm512_cmp_pd_mask(simde_mm512_abs_pd(b), simde_mm512_abs_pd(a), SIMDE_CMP_GE_OS), b); + break; + default: + break; + } + + switch (imm8 & 12) { + case 0: + r = simde_x_mm512_copysign_pd(r, a); + break; + case 8: + r = simde_mm512_andnot_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(-0.0)), r); + break; + case 12: + r = simde_mm512_or_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(-0.0)), r); + break; + default: + break; + } + + return r; +} +#if defined(SIMDE_X86_AVX512DQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm512_range_pd(a, b, imm8) _mm512_range_pd((a), (b), (imm8)) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_range_pd(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d_private \ + simde_mm512_range_pd_r_ = simde__m512d_to_private(simde_mm512_setzero_pd()), \ + simde_mm512_range_pd_a_ = simde__m512d_to_private(a), \ + simde_mm512_range_pd_b_ = simde__m512d_to_private(b); \ + \ + for (size_t simde_mm512_range_pd_i = 0 ; simde_mm512_range_pd_i < (sizeof(simde_mm512_range_pd_r_.m128d) / sizeof(simde_mm512_range_pd_r_.m128d[0])) ; simde_mm512_range_pd_i++) { \ + simde_mm512_range_pd_r_.m128d[simde_mm512_range_pd_i] = simde_mm_range_pd(simde_mm512_range_pd_a_.m128d[simde_mm512_range_pd_i], simde_mm512_range_pd_b_.m128d[simde_mm512_range_pd_i], imm8); \ + } \ + \ + simde__m512d_from_private(simde_mm512_range_pd_r_); \ + })) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_range_pd(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d_private \ + simde_mm512_range_pd_r_ = simde__m512d_to_private(simde_mm512_setzero_pd()), \ + simde_mm512_range_pd_a_ = simde__m512d_to_private(a), \ + simde_mm512_range_pd_b_ = simde__m512d_to_private(b); \ + \ + for (size_t simde_mm512_range_pd_i = 0 ; simde_mm512_range_pd_i < (sizeof(simde_mm512_range_pd_r_.m256d) / sizeof(simde_mm512_range_pd_r_.m256d[0])) ; simde_mm512_range_pd_i++) { \ + simde_mm512_range_pd_r_.m256d[simde_mm512_range_pd_i] = simde_mm256_range_pd(simde_mm512_range_pd_a_.m256d[simde_mm512_range_pd_i], simde_mm512_range_pd_b_.m256d[simde_mm512_range_pd_i], imm8); \ + } \ + \ + simde__m512d_from_private(simde_mm512_range_pd_r_); \ + })) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_range_pd + #define _mm512_range_pd(a, b, imm8) simde_mm512_range_pd(a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_mask_range_pd(src, k, a, b, imm8) _mm512_mask_range_pd(src, k, a, b, imm8) +#else + #define simde_mm512_mask_range_pd(src, k, a, b, imm8) simde_mm512_mask_mov_pd(src, k, simde_mm512_range_pd(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_range_pd + #define _mm512_mask_range_pd(src, k, a, b, imm8) simde_mm512_mask_range_pd(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_maskz_range_pd(k, a, b, imm8) _mm512_maskz_range_pd(k, a, b, imm8) +#else + #define simde_mm512_maskz_range_pd(k, a, b, imm8) simde_mm512_maskz_mov_pd(k, simde_mm512_range_pd(a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_range_pd + #define _mm512_maskz_range_pd(k, a, b, imm8) simde_mm512_maskz_range_pd(k, a, b, imm8) +#endif + +#if (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + #define simde_x_mm_range_ss(a, b, imm8) simde_mm_move_ss(a, simde_mm_range_ps(a, b, imm8)) +#elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + #define simde_x_mm_range_ss(a, b, imm8) simde_mm_move_ss(a, simde_mm_range_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b), imm8)) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_x_mm_range_ss (simde__m128 a, simde__m128 b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m128_private + r_ = simde__m128_to_private(a), + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + simde_float32 abs_a = simde_uint32_as_float32(a_.u32[0] & UINT32_C(2147483647)); + simde_float32 abs_b = simde_uint32_as_float32(b_.u32[0] & UINT32_C(2147483647)); + + switch (imm8 & 3) { + case 0: + r_ = simde__m128_to_private(simde_mm_min_ss(a, b)); + break; + case 1: + r_ = simde__m128_to_private(simde_mm_max_ss(a, b)); + break; + case 2: + r_.f32[0] = abs_a <= abs_b ? a_.f32[0] : b_.f32[0]; + break; + case 3: + r_.f32[0] = abs_b >= abs_a ? b_.f32[0] : a_.f32[0]; + break; + default: + break; + } + + switch (imm8 & 12) { + case 0: + r_.f32[0] = simde_uint32_as_float32((a_.u32[0] & UINT32_C(2147483648)) ^ (r_.u32[0] & UINT32_C(2147483647))); + break; + case 8: + r_.f32[0] = simde_uint32_as_float32(r_.u32[0] & UINT32_C(2147483647)); + break; + case 12: + r_.f32[0] = simde_uint32_as_float32(r_.u32[0] | UINT32_C(2147483648)); + break; + default: + break; + } + + return simde__m128_from_private(r_); + } +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm_mask_range_ss(src, k, a, b, imm8) _mm_mask_range_ss(src, k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm_mask_range_ss(src, k, a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128_private \ + simde_mm_mask_range_ss_r_ = simde__m128_to_private(a), \ + simde_mm_mask_range_ss_src_ = simde__m128_to_private(src); \ + \ + if (k & 1) \ + simde_mm_mask_range_ss_r_ = simde__m128_to_private(simde_x_mm_range_ss(a, b, imm8)); \ + else \ + simde_mm_mask_range_ss_r_.f32[0] = simde_mm_mask_range_ss_src_.f32[0]; \ + \ + simde__m128_from_private(simde_mm_mask_range_ss_r_); \ + })) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_mask_range_ss (simde__m128 src, simde__mmask8 k, simde__m128 a, simde__m128 b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m128_private + r_ = simde__m128_to_private(a), + src_ = simde__m128_to_private(src); + + if (k & 1) + r_ = simde__m128_to_private(simde_x_mm_range_ss(a, b, imm8)); + else + r_.f32[0] = src_.f32[0]; + + return simde__m128_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_range_ss + #define _mm_mask_range_ss(src, k, a, b, imm8) simde_mm_mask_range_ss(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm_maskz_range_ss(k, a, b, imm8) _mm_maskz_range_ss(k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm_maskz_range_ss(k, a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128_private simde_mm_maskz_range_ss_r_ = simde__m128_to_private(a); \ + \ + if (k & 1) \ + simde_mm_maskz_range_ss_r_ = simde__m128_to_private(simde_x_mm_range_ss(a, b, imm8)); \ + else \ + simde_mm_maskz_range_ss_r_.f32[0] = SIMDE_FLOAT32_C(0.0); \ + \ + simde__m128_from_private(simde_mm_maskz_range_ss_r_); \ + })) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_maskz_range_ss (simde__mmask8 k, simde__m128 a, simde__m128 b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m128_private r_ = simde__m128_to_private(a); + + if (k & 1) + r_ = simde__m128_to_private(simde_x_mm_range_ss(a, b, imm8)); + else + r_.f32[0] = SIMDE_FLOAT32_C(0.0); + + return simde__m128_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_range_ss + #define _mm_maskz_range_ss(k, a, b, imm8) simde_mm_maskz_range_ss(k, a, b, imm8) +#endif + +#if (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + #define simde_x_mm_range_sd(a, b, imm8) simde_mm_move_sd(a, simde_mm_range_pd(a, b, imm8)) +#elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + #define simde_x_mm_range_sd(a, b, imm8) simde_mm_move_sd(a, simde_mm_range_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b), imm8)) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_x_mm_range_sd (simde__m128d a, simde__m128d b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m128d_private + r_ = simde__m128d_to_private(a), + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + simde_float64 abs_a = simde_uint64_as_float64(a_.u64[0] & UINT64_C(9223372036854775807)); + simde_float64 abs_b = simde_uint64_as_float64(b_.u64[0] & UINT64_C(9223372036854775807)); + + switch (imm8 & 3) { + case 0: + r_ = simde__m128d_to_private(simde_mm_min_sd(a, b)); + break; + case 1: + r_ = simde__m128d_to_private(simde_mm_max_sd(a, b)); + break; + case 2: + r_.f64[0] = abs_a <= abs_b ? a_.f64[0] : b_.f64[0]; + break; + case 3: + r_.f64[0] = abs_b >= abs_a ? b_.f64[0] : a_.f64[0]; + break; + default: + break; + } + + switch (imm8 & 12) { + case 0: + r_.f64[0] = simde_uint64_as_float64((a_.u64[0] & UINT64_C(9223372036854775808)) ^ (r_.u64[0] & UINT64_C(9223372036854775807))); + break; + case 8: + r_.f64[0] = simde_uint64_as_float64(r_.u64[0] & UINT64_C(9223372036854775807)); + break; + case 12: + r_.f64[0] = simde_uint64_as_float64(r_.u64[0] | UINT64_C(9223372036854775808)); + break; + default: + break; + } + + return simde__m128d_from_private(r_); + } +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm_mask_range_sd(src, k, a, b, imm8) _mm_mask_range_sd(src, k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm_mask_range_sd(src, k, a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128d_private \ + simde_mm_mask_range_sd_r_ = simde__m128d_to_private(a), \ + simde_mm_mask_range_sd_src_ = simde__m128d_to_private(src); \ + \ + if (k & 1) \ + simde_mm_mask_range_sd_r_ = simde__m128d_to_private(simde_x_mm_range_sd(a, b, imm8)); \ + else \ + simde_mm_mask_range_sd_r_.f64[0] = simde_mm_mask_range_sd_src_.f64[0]; \ + \ + simde__m128d_from_private(simde_mm_mask_range_sd_r_); \ + })) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_mask_range_sd (simde__m128d src, simde__mmask8 k, simde__m128d a, simde__m128d b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m128d_private + r_ = simde__m128d_to_private(a), + src_ = simde__m128d_to_private(src); + + if (k & 1) + r_ = simde__m128d_to_private(simde_x_mm_range_sd(a, b, imm8)); + else + r_.f64[0] = src_.f64[0]; + + return simde__m128d_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_range_sd + #define _mm_mask_range_sd(src, k, a, b, imm8) simde_mm_mask_range_sd(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm_maskz_range_sd(k, a, b, imm8) _mm_maskz_range_sd(k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm_maskz_range_sd(k, a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128d_private simde_mm_maskz_range_sd_r_ = simde__m128d_to_private(a); \ + \ + if (k & 1) \ + simde_mm_maskz_range_sd_r_ = simde__m128d_to_private(simde_x_mm_range_sd(a, b, imm8)); \ + else \ + simde_mm_maskz_range_sd_r_.f64[0] = SIMDE_FLOAT64_C(0.0); \ + \ + simde__m128d_from_private(simde_mm_maskz_range_sd_r_); \ + })) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_maskz_range_sd (simde__mmask8 k, simde__m128d a, simde__m128d b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m128d_private r_ = simde__m128d_to_private(a); + + if (k & 1) + r_ = simde__m128d_to_private(simde_x_mm_range_sd(a, b, imm8)); + else + r_.f64[0] = SIMDE_FLOAT64_C(0.0); + + return simde__m128d_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_range_sd + #define _mm_maskz_range_sd(k, a, b, imm8) simde_mm_maskz_range_sd(k, a, b, imm8) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_RANGE_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/range_round.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/range_round.h new file mode 100644 index 00000000..7bf13207 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/range_round.h @@ -0,0 +1,686 @@ +#if !defined(SIMDE_X86_AVX512_RANGE_ROUND_H) +#define SIMDE_X86_AVX512_RANGE_ROUND_H + +#include "types.h" +#include "range.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_range_round_ps(a, b, imm8, sae) _mm512_range_round_ps(a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_range_round_ps(a, b, imm8, sae) simde_mm512_range_ps(a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_range_round_ps(a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512 simde_mm512_range_round_ps_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_range_round_ps_envp; \ + int simde_mm512_range_round_ps_x = feholdexcept(&simde_mm512_range_round_ps_envp); \ + simde_mm512_range_round_ps_r = simde_mm512_range_ps(a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_range_round_ps_x == 0)) \ + fesetenv(&simde_mm512_range_round_ps_envp); \ + } \ + else { \ + simde_mm512_range_round_ps_r = simde_mm512_range_ps(a, b, imm8); \ + } \ + \ + simde_mm512_range_round_ps_r; \ + })) + #else + #define simde_mm512_range_round_ps(a, b, imm8, sae) simde_mm512_range_ps(a, b, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512 + simde_mm512_range_round_ps (simde__m512 a, simde__m512 b, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m512 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_range_ps(a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_range_ps(a, b, imm8); + #endif + } + else { + r = simde_mm512_range_ps(a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_range_round_ps + #define _mm512_range_round_ps(a, b, imm8, sae) simde_mm512_range_round_ps(a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_mask_range_round_ps(src, k, a, b, imm8, sae) _mm512_mask_range_round_ps(src, k, a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_mask_range_round_ps(src, k, a, b, imm8, sae) simde_mm512_mask_range_ps(src, k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_mask_range_round_ps(src, k, a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512 simde_mm512_mask_range_round_ps_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_mask_range_round_ps_envp; \ + int simde_mm512_mask_range_round_ps_x = feholdexcept(&simde_mm512_mask_range_round_ps_envp); \ + simde_mm512_mask_range_round_ps_r = simde_mm512_mask_range_ps(src, k, a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_mask_range_round_ps_x == 0)) \ + fesetenv(&simde_mm512_mask_range_round_ps_envp); \ + } \ + else { \ + simde_mm512_mask_range_round_ps_r = simde_mm512_mask_range_ps(src, k, a, b, imm8); \ + } \ + \ + simde_mm512_mask_range_round_ps_r; \ + })) + #else + #define simde_mm512_mask_range_round_ps(src, k, a, b, imm8, sae) simde_mm512_mask_range_ps(src, k, a, b, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512 + simde_mm512_mask_range_round_ps (simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m512 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_mask_range_ps(src, k, a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_mask_range_ps(src, k, a, b, imm8); + #endif + } + else { + r = simde_mm512_mask_range_ps(src, k, a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_range_round_ps + #define _mm512_mask_range_round_ps(src, k, a, b, imm8, sae) simde_mm512_mask_range_round_ps(src, k, a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_maskz_range_round_ps(k, a, b, imm8, sae) _mm512_maskz_range_round_ps(k, a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_maskz_range_round_ps(k, a, b, imm8, sae) simde_mm512_maskz_range_ps(k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_maskz_range_round_ps(k, a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512 simde_mm512_maskz_range_round_ps_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_maskz_range_round_ps_envp; \ + int simde_mm512_maskz_range_round_ps_x = feholdexcept(&simde_mm512_maskz_range_round_ps_envp); \ + simde_mm512_maskz_range_round_ps_r = simde_mm512_maskz_range_ps(k, a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_maskz_range_round_ps_x == 0)) \ + fesetenv(&simde_mm512_maskz_range_round_ps_envp); \ + } \ + else { \ + simde_mm512_maskz_range_round_ps_r = simde_mm512_maskz_range_ps(k, a, b, imm8); \ + } \ + \ + simde_mm512_maskz_range_round_ps_r; \ + })) + #else + #define simde_mm512_maskz_range_round_ps(k, a, b, imm8, sae) simde_mm512_maskz_range_ps(k, a, b, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512 + simde_mm512_maskz_range_round_ps (simde__mmask16 k, simde__m512 a, simde__m512 b, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m512 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_maskz_range_ps(k, a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_maskz_range_ps(k, a, b, imm8); + #endif + } + else { + r = simde_mm512_maskz_range_ps(k, a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_range_round_ps + #define _mm512_maskz_range_round_ps(k, a, b, imm8, sae) simde_mm512_maskz_range_round_ps(k, a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_range_round_pd(a, b, imm8, sae) _mm512_range_round_pd(a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_range_round_pd(a, b, imm8, sae) simde_mm512_range_pd(a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_range_round_pd(a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d simde_mm512_range_round_pd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_range_round_pd_envp; \ + int simde_mm512_range_round_pd_x = feholdexcept(&simde_mm512_range_round_pd_envp); \ + simde_mm512_range_round_pd_r = simde_mm512_range_pd(a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_range_round_pd_x == 0)) \ + fesetenv(&simde_mm512_range_round_pd_envp); \ + } \ + else { \ + simde_mm512_range_round_pd_r = simde_mm512_range_pd(a, b, imm8); \ + } \ + \ + simde_mm512_range_round_pd_r; \ + })) + #else + #define simde_mm512_range_round_pd(a, b, imm8, sae) simde_mm512_range_pd(a, b, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512d + simde_mm512_range_round_pd (simde__m512d a, simde__m512d b, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m512d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_range_pd(a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_range_pd(a, b, imm8); + #endif + } + else { + r = simde_mm512_range_pd(a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_range_round_pd + #define _mm512_range_round_pd(a, b, imm8, sae) simde_mm512_range_round_pd(a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_mask_range_round_pd(src, k, a, b, imm8, sae) _mm512_mask_range_round_pd(src, k, a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_mask_range_round_pd(src, k, a, b, imm8, sae) simde_mm512_mask_range_pd(src, k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_mask_range_round_pd(src, k, a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d simde_mm512_mask_range_round_pd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_mask_range_round_pd_envp; \ + int simde_mm512_mask_range_round_pd_x = feholdexcept(&simde_mm512_mask_range_round_pd_envp); \ + simde_mm512_mask_range_round_pd_r = simde_mm512_mask_range_pd(src, k, a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_mask_range_round_pd_x == 0)) \ + fesetenv(&simde_mm512_mask_range_round_pd_envp); \ + } \ + else { \ + simde_mm512_mask_range_round_pd_r = simde_mm512_mask_range_pd(src, k, a, b, imm8); \ + } \ + \ + simde_mm512_mask_range_round_pd_r; \ + })) + #else + #define simde_mm512_mask_range_round_pd(src, k, a, b, imm8, sae) simde_mm512_mask_range_pd(src, k, a, b, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512d + simde_mm512_mask_range_round_pd (simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m512d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_mask_range_pd(src, k, a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_mask_range_pd(src, k, a, b, imm8); + #endif + } + else { + r = simde_mm512_mask_range_pd(src, k, a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_range_round_pd + #define _mm512_mask_range_round_pd(src, k, a, b, imm8, sae) simde_mm512_mask_range_round_pd(src, k, a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm512_maskz_range_round_pd(k, a, b, imm8, sae) _mm512_maskz_range_round_pd(k, a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_maskz_range_round_pd(k, a, b, imm8, sae) simde_mm512_maskz_range_pd(k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_maskz_range_round_pd(k, a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d simde_mm512_maskz_range_round_pd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_maskz_range_round_pd_envp; \ + int simde_mm512_maskz_range_round_pd_x = feholdexcept(&simde_mm512_maskz_range_round_pd_envp); \ + simde_mm512_maskz_range_round_pd_r = simde_mm512_maskz_range_pd(k, a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_maskz_range_round_pd_x == 0)) \ + fesetenv(&simde_mm512_maskz_range_round_pd_envp); \ + } \ + else { \ + simde_mm512_maskz_range_round_pd_r = simde_mm512_maskz_range_pd(k, a, b, imm8); \ + } \ + \ + simde_mm512_maskz_range_round_pd_r; \ + })) + #else + #define simde_mm512_maskz_range_round_pd(k, a, b, imm8, sae) simde_mm512_maskz_range_pd(k, a, b, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512d + simde_mm512_maskz_range_round_pd (simde__mmask8 k, simde__m512d a, simde__m512d b, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m512d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_maskz_range_pd(k, a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_maskz_range_pd(k, a, b, imm8); + #endif + } + else { + r = simde_mm512_maskz_range_pd(k, a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_range_round_pd + #define _mm512_maskz_range_round_pd(k, a, b, imm8, sae) simde_mm512_maskz_range_round_pd(k, a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm_range_round_ss(a, b, imm8, sae) _mm_range_round_ss(a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_range_round_ss(a, b, imm8, sae) simde_x_mm_range_ss(a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_range_round_ss(a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128 simde_mm_range_round_ss_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_range_round_ss_envp; \ + int simde_mm_range_round_ss_x = feholdexcept(&simde_mm_range_round_ss_envp); \ + simde_mm_range_round_ss_r = simde_x_mm_range_ss(a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm_range_round_ss_x == 0)) \ + fesetenv(&simde_mm_range_round_ss_envp); \ + } \ + else { \ + simde_mm_range_round_ss_r = simde_x_mm_range_ss(a, b, imm8); \ + } \ + \ + simde_mm_range_round_ss_r; \ + })) + #else + #define simde_mm_range_round_ss(a, b, imm8, sae) simde_x_mm_range_ss(a, b, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_range_round_ss (simde__m128 a, simde__m128 b, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_x_mm_range_ss(a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_x_mm_range_ss(a, b, imm8); + #endif + } + else { + r = simde_x_mm_range_ss(a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_range_round_ss + #define _mm_range_round_ss(a, b, imm8, sae) simde_mm_range_round_ss(a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm_mask_range_round_ss(src, k, a, b, imm8, sae) _mm_mask_range_round_ss(src, k, a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_mask_range_round_ss(src, k, a, b, imm8, sae) simde_mm_mask_range_ss(src, k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_mask_range_round_ss(src, k, a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128 simde_mm_mask_range_round_ss_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_mask_range_round_ss_envp; \ + int simde_mm_mask_range_round_ss_x = feholdexcept(&simde_mm_mask_range_round_ss_envp); \ + simde_mm_mask_range_round_ss_r = simde_mm_mask_range_ss(src, k, a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm_mask_range_round_ss_x == 0)) \ + fesetenv(&simde_mm_mask_range_round_ss_envp); \ + } \ + else { \ + simde_mm_mask_range_round_ss_r = simde_mm_mask_range_ss(src, k, a, b, imm8); \ + } \ + \ + simde_mm_mask_range_round_ss_r; \ + })) + #else + #define simde_mm_mask_range_round_ss(src, k, a, b, imm8, sae) simde_mm_mask_range_ss(src, k, a, b, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_mask_range_round_ss (simde__m128 src, simde__mmask8 k, simde__m128 a, simde__m128 b, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_mask_range_ss(src, k, a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_mask_range_ss(src, k, a, b, imm8); + #endif + } + else { + r = simde_mm_mask_range_ss(src, k, a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_range_round_ss + #define _mm_mask_range_round_ss(src, k, a, b, imm8, sae) simde_mm_mask_range_round_ss(src, k, a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm_maskz_range_round_ss(k, a, b, imm8, sae) _mm_maskz_range_round_ss(k, a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_maskz_range_round_ss(k, a, b, imm8, sae) simde_mm_maskz_range_ss(k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_maskz_range_round_ss(k, a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128 simde_mm_maskz_range_round_ss_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_maskz_range_round_ss_envp; \ + int simde_mm_maskz_range_round_ss_x = feholdexcept(&simde_mm_maskz_range_round_ss_envp); \ + simde_mm_maskz_range_round_ss_r = simde_mm_maskz_range_ss(k, a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm_maskz_range_round_ss_x == 0)) \ + fesetenv(&simde_mm_maskz_range_round_ss_envp); \ + } \ + else { \ + simde_mm_maskz_range_round_ss_r = simde_mm_maskz_range_ss(k, a, b, imm8); \ + } \ + \ + simde_mm_maskz_range_round_ss_r; \ + })) + #else + #define simde_mm_maskz_range_round_ss(k, a, b, imm8, sae) simde_mm_maskz_range_ss(k, a, b, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_maskz_range_round_ss (simde__mmask8 k, simde__m128 a, simde__m128 b, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_maskz_range_ss(k, a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_maskz_range_ss(k, a, b, imm8); + #endif + } + else { + r = simde_mm_maskz_range_ss(k, a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_range_round_ss + #define _mm_maskz_range_round_ss(k, a, b, imm8, sae) simde_mm_maskz_range_round_ss(k, a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm_range_round_sd(a, b, imm8, sae) _mm_range_round_sd(a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_range_round_sd(a, b, imm8, sae) simde_x_mm_range_sd(a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_range_round_sd(a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128d simde_mm_range_round_sd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_range_round_sd_envp; \ + int simde_mm_range_round_sd_x = feholdexcept(&simde_mm_range_round_sd_envp); \ + simde_mm_range_round_sd_r = simde_x_mm_range_sd(a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm_range_round_sd_x == 0)) \ + fesetenv(&simde_mm_range_round_sd_envp); \ + } \ + else { \ + simde_mm_range_round_sd_r = simde_x_mm_range_sd(a, b, imm8); \ + } \ + \ + simde_mm_range_round_sd_r; \ + })) + #else + #define simde_mm_range_round_sd(a, b, imm8, sae) simde_x_mm_range_sd(a, b, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_range_round_sd (simde__m128d a, simde__m128d b, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_x_mm_range_sd(a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_x_mm_range_sd(a, b, imm8); + #endif + } + else { + r = simde_x_mm_range_sd(a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_range_round_sd + #define _mm_range_round_sd(a, b, imm8, sae) simde_mm_range_round_sd(a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm_mask_range_round_sd(src, k, a, b, imm8, sae) _mm_mask_range_round_sd(src, k, a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_mask_range_round_sd(src, k, a, b, imm8, sae) simde_mm_mask_range_sd(src, k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_mask_range_round_sd(src, k, a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128d simde_mm_mask_range_round_sd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_mask_range_round_sd_envp; \ + int simde_mm_mask_range_round_sd_x = feholdexcept(&simde_mm_mask_range_round_sd_envp); \ + simde_mm_mask_range_round_sd_r = simde_mm_mask_range_sd(src, k, a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm_mask_range_round_sd_x == 0)) \ + fesetenv(&simde_mm_mask_range_round_sd_envp); \ + } \ + else { \ + simde_mm_mask_range_round_sd_r = simde_mm_mask_range_sd(src, k, a, b, imm8); \ + } \ + \ + simde_mm_mask_range_round_sd_r; \ + })) + #else + #define simde_mm_mask_range_round_sd(src, k, a, b, imm8, sae) simde_mm_mask_range_sd(src, k, a, b, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_mask_range_round_sd (simde__m128d src, simde__mmask8 k, simde__m128d a, simde__m128d b, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_mask_range_sd(src, k, a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_mask_range_sd(src, k, a, b, imm8); + #endif + } + else { + r = simde_mm_mask_range_sd(src, k, a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_range_round_sd + #define _mm_mask_range_round_sd(src, k, a, b, imm8, sae) simde_mm_mask_range_round_sd(src, k, a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512DQ_NATIVE) + #define simde_mm_maskz_range_round_sd(k, a, b, imm8, sae) _mm_maskz_range_round_sd(k, a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_maskz_range_round_sd(k, a, b, imm8, sae) simde_mm_maskz_range_sd(k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_maskz_range_round_sd(k, a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128d simde_mm_maskz_range_round_sd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_maskz_range_round_sd_envp; \ + int simde_mm_maskz_range_round_sd_x = feholdexcept(&simde_mm_maskz_range_round_sd_envp); \ + simde_mm_maskz_range_round_sd_r = simde_mm_maskz_range_sd(k, a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm_maskz_range_round_sd_x == 0)) \ + fesetenv(&simde_mm_maskz_range_round_sd_envp); \ + } \ + else { \ + simde_mm_maskz_range_round_sd_r = simde_mm_maskz_range_sd(k, a, b, imm8); \ + } \ + \ + simde_mm_maskz_range_round_sd_r; \ + })) + #else + #define simde_mm_maskz_range_round_sd(k, a, b, imm8, sae) simde_mm_maskz_range_sd(k, a, b, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_maskz_range_round_sd (simde__mmask8 k, simde__m128d a, simde__m128d b, int imm8, int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_maskz_range_sd(k, a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_maskz_range_sd(k, a, b, imm8); + #endif + } + else { + r = simde_mm_maskz_range_sd(k, a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_range_round_sd + #define _mm_maskz_range_round_sd(k, a, b, imm8, sae) simde_mm_maskz_range_round_sd(k, a, b, imm8, sae) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_RANGE_ROUND_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rcp.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rcp.h new file mode 100644 index 00000000..b1b394cf --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rcp.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2023 Michael R. Crusoe + */ + +#if !defined(SIMDE_X86_AVX512_RCP_H) +#define SIMDE_X86_AVX512_RCP_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +// TODO: "The maximum relative error for this approximation is less than 2^-14." +// vs 1.5*2^-12 for _mm{,256}_rcp_ps + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_rcp14_ps (simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rcp14_ps(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = SIMDE_FLOAT32_C(1.0) / a_.f32[i]; + } + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_rcp14_ps + #define _mm512_rcp14_ps(a) simde_mm512_rcp14_ps(a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_RCP_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/reduce.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/reduce.h new file mode 100644 index 00000000..c007572e --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/reduce.h @@ -0,0 +1,355 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2023 Michael R. Crusoe + */ + +#if !defined(SIMDE_X86_AVX512_REDUCE_H) +#define SIMDE_X86_AVX512_REDUCE_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#if defined(__clang__) && SIMDE_FLOAT16_API == SIMDE_FLOAT16_API_FP16 +SIMDE_DIAGNOSTIC_DISABLE_DOUBLE_PROMOTION_ +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde_float16 +simde_mm512_reduce_max_ph(simde__m512h a) { + #if defined(SIMDE_X86_AVX512FP16_NATIVE) + return _mm512_reduce_max_ph(a); + #else + simde__m512h_private a_; + simde_float16 r; + a_ = simde__m512h_to_private(a); + + r = SIMDE_NINFINITYHF; + #if defined(SIMDE_FLOAT16_VECTOR) + SIMDE_VECTORIZE_REDUCTION(max:r) + #endif + for (size_t i = 0 ; i < (sizeof(a_.f16) / sizeof(a_.f16[0])) ; i++) { + r = simde_float16_to_float32(a_.f16[i]) > simde_float16_to_float32(r) ? a_.f16[i] : r; + } + return r; + #endif +} +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) +# define _mm512_reduce_max_ph(a) simde_mm512_reduce_max_ph((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde_float16 +simde_mm512_reduce_min_ph(simde__m512h a) { + #if defined(SIMDE_X86_AVX512FP16_NATIVE) + return _mm512_reduce_min_ph(a); + #else + simde__m512h_private a_; + simde_float16 r; + a_ = simde__m512h_to_private(a); + + r = SIMDE_INFINITYHF; + #if defined(SIMDE_FLOAT16_VECTOR) + SIMDE_VECTORIZE_REDUCTION(min:r) + #endif + for (size_t i = 0 ; i < (sizeof(a_.f16) / sizeof(a_.f16[0])) ; i++) { + r = simde_float16_to_float32(a_.f16[i]) < simde_float16_to_float32(r) ? a_.f16[i] : r; + } + return r; + #endif +} +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) +# define _mm512_reduce_min_ph(a) simde_mm512_reduce_min_ph((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm512_reduce_max_epi32(simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_reduce_max_epi32(a); + #else + simde__m512i_private a_; + int32_t r; + a_ = simde__m512i_to_private(a); + + r = -INT32_MAX; + SIMDE_VECTORIZE_REDUCTION(max:r) + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r = a_.i32[i] > r ? a_.i32[i] : r; + } + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) +# define _mm512_reduce_max_epi32(a) simde_mm512_reduce_max_epi32((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int64_t +simde_mm512_reduce_max_epi64(simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_reduce_max_epi64(a); + #else + simde__m512i_private a_; + int64_t r; + a_ = simde__m512i_to_private(a); + + r = -INT64_MAX; + SIMDE_VECTORIZE_REDUCTION(max:r) + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r = a_.i64[i] > r ? a_.i64[i] : r; + } + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) +# define _mm512_reduce_max_epi64(a) simde_mm512_reduce_max_epi64((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +uint32_t +simde_mm512_reduce_max_epu32(simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_reduce_max_epu32(a); + #else + simde__m512i_private a_; + uint32_t r; + a_ = simde__m512i_to_private(a); + + r = 0; + SIMDE_VECTORIZE_REDUCTION(max:r) + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + r = a_.u32[i] > r ? a_.u32[i] : r; + } + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) +# define _mm512_reduce_max_epu32(a) simde_mm512_reduce_max_epu32((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +uint64_t +simde_mm512_reduce_max_epu64(simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_reduce_max_epu64(a); + #else + simde__m512i_private a_; + uint64_t r; + a_ = simde__m512i_to_private(a); + + r = 0; + SIMDE_VECTORIZE_REDUCTION(max:r) + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + r = a_.u64[i] > r ? a_.u64[i] : r; + } + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) +# define _mm512_reduce_max_epu64(a) simde_mm512_reduce_max_epu64((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde_float64 +simde_mm512_reduce_max_pd(simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_reduce_max_pd(a); + #else + simde__m512d_private a_; + simde_float64 r; + a_ = simde__m512d_to_private(a); + + r = -SIMDE_MATH_INFINITY; + SIMDE_VECTORIZE_REDUCTION(max:r) + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + r = a_.f64[i] > r ? a_.f64[i] : r; + } + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) +# define _mm512_reduce_max_pd(a) simde_mm512_reduce_max_pd((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde_float32 +simde_mm512_reduce_max_ps(simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_reduce_max_ps(a); + #else + simde__m512_private a_; + simde_float32 r; + a_ = simde__m512_to_private(a); + + r = -SIMDE_MATH_INFINITYF; + SIMDE_VECTORIZE_REDUCTION(max:r) + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + r = a_.f32[i] > r ? a_.f32[i] : r; + } + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) +# define _mm512_reduce_max_ps(a) simde_mm512_reduce_max_ps((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm512_reduce_min_epi32(simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_reduce_min_epi32(a); + #else + simde__m512i_private a_; + int32_t r; + a_ = simde__m512i_to_private(a); + + r = INT32_MAX; + SIMDE_VECTORIZE_REDUCTION(min:r) + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r = a_.i32[i] < r ? a_.i32[i] : r; + } + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) +# define _mm512_reduce_min_epi32(a) simde_mm512_reduce_min_epi32((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int64_t +simde_mm512_reduce_min_epi64(simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_reduce_min_epi64(a); + #else + simde__m512i_private a_; + int64_t r; + a_ = simde__m512i_to_private(a); + + r = INT64_MAX; + SIMDE_VECTORIZE_REDUCTION(min:r) + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r = a_.i64[i] < r ? a_.i64[i] : r; + } + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) +# define _mm512_reduce_min_epi64(a) simde_mm512_reduce_min_epi64((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +uint32_t +simde_mm512_reduce_min_epu32(simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_reduce_min_epu32(a); + #else + simde__m512i_private a_; + uint32_t r; + a_ = simde__m512i_to_private(a); + + r = UINT32_MAX; + SIMDE_VECTORIZE_REDUCTION(min:r) + for (size_t i = 0 ; i < (sizeof(a_.u32) / sizeof(a_.u32[0])) ; i++) { + r = a_.u32[i] < r ? a_.u32[i] : r; + } + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) +# define _mm512_reduce_min_epu32(a) simde_mm512_reduce_min_epu32((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +uint64_t +simde_mm512_reduce_min_epu64(simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_reduce_min_epu64(a); + #else + simde__m512i_private a_; + uint64_t r; + a_ = simde__m512i_to_private(a); + + r = UINT64_MAX; + SIMDE_VECTORIZE_REDUCTION(min:r) + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + r = a_.u64[i] < r ? a_.u64[i] : r; + } + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) +# define _mm512_reduce_min_epu64(a) simde_mm512_reduce_min_epu64((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde_float64 +simde_mm512_reduce_min_pd(simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_reduce_min_pd(a); + #else + simde__m512d_private a_; + simde_float64 r; + a_ = simde__m512d_to_private(a); + + r = SIMDE_MATH_INFINITY; + SIMDE_VECTORIZE_REDUCTION(min:r) + for (size_t i = 0 ; i < (sizeof(a_.f64) / sizeof(a_.f64[0])) ; i++) { + r = a_.f64[i] < r ? a_.f64[i] : r; + } + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) +# define _mm512_reduce_min_pd(a) simde_mm512_reduce_min_pd((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde_float32 +simde_mm512_reduce_min_ps(simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_reduce_min_ps(a); + #else + simde__m512_private a_; + simde_float32 r; + a_ = simde__m512_to_private(a); + + r = SIMDE_MATH_INFINITYF; + SIMDE_VECTORIZE_REDUCTION(min:r) + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + r = a_.f32[i] < r ? a_.f32[i] : r; + } + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) +# define _mm512_reduce_min_ps(a) simde_mm512_reduce_min_ps((a)) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_REDUCE_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rol.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rol.h new file mode 100644 index 00000000..5bdf98bc --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rol.h @@ -0,0 +1,410 @@ +#if !defined(SIMDE_X86_AVX512_ROL_H) +#define SIMDE_X86_AVX512_ROL_H + +#include "types.h" +#include "mov.h" +#include "or.h" +#include "srli.h" +#include "slli.h" +#include "../avx2.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_rol_epi32(a, imm8) _mm_rol_epi32(a, imm8) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128i + simde_mm_rol_epi32 (simde__m128i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE (imm8, 0, 255) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i32 = vec_rl(a_.altivec_i32, vec_splats(HEDLEY_STATIC_CAST(unsigned int, imm8))); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + switch (imm8 & 31) { + case 0: + r_ = a_; + break; + default: + r_.u32 = (a_.u32 << (imm8 & 31)) | (a_.u32 >> (32 - (imm8 & 31))); + break; + } + #else + switch (imm8 & 31) { + case 0: + r_ = a_; + break; + default: + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] << (imm8 & 31)) | (a_.u32[i] >> (32 - (imm8 & 31))); + } + break; + } + #endif + + return simde__m128i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_rol_epi32 + #define _mm_rol_epi32(a, imm8) simde_mm_rol_epi32(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_mask_rol_epi32(src, k, a, imm8) _mm_mask_rol_epi32(src, k, a, imm8) +#else + #define simde_mm_mask_rol_epi32(src, k, a, imm8) simde_mm_mask_mov_epi32(src, k, simde_mm_rol_epi32(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_rol_epi32 + #define _mm_mask_rol_epi32(src, k, a, imm8) simde_mm_mask_rol_epi32(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_maskz_rol_epi32(k, a, imm8) _mm_maskz_rol_epi32(k, a, imm8) +#else + #define simde_mm_maskz_rol_epi32(k, a, imm8) simde_mm_maskz_mov_epi32(k, simde_mm_rol_epi32(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_rol_epi32 + #define _mm_maskz_rol_epi32(k, a, imm8) simde_mm_maskz_rol_epi32(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_rol_epi32(a, imm8) _mm256_rol_epi32(a, imm8) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m256i + simde_mm256_rol_epi32 (simde__m256i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE (imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_i32 = vec_rl(a_.m128i_private[i].altivec_i32, vec_splats(HEDLEY_STATIC_CAST(unsigned int, imm8))); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + switch (imm8 & 31) { + case 0: + r_ = a_; + break; + default: + r_.u32 = (a_.u32 << (imm8 & 31)) | (a_.u32 >> (32 - (imm8 & 31))); + break; + } + #else + switch (imm8 & 31) { + case 0: + r_ = a_; + break; + default: + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] << (imm8 & 31)) | (a_.u32[i] >> (32 - (imm8 & 31))); + } + break; + } + #endif + + return simde__m256i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_rol_epi32 + #define _mm256_rol_epi32(a, imm8) simde_mm256_rol_epi32(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_rol_epi32(src, k, a, imm8) _mm256_mask_rol_epi32(src, k, a, imm8) +#else + #define simde_mm256_mask_rol_epi32(src, k, a, imm8) simde_mm256_mask_mov_epi32(src, k, simde_mm256_rol_epi32(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_rol_epi32 + #define _mm256_mask_rol_epi32(src, k, a, imm8) simde_mm256_mask_rol_epi32(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_maskz_rol_epi32(k, a, imm8) _mm256_maskz_rol_epi32(k, a, imm8) +#else + #define simde_mm256_maskz_rol_epi32(k, a, imm8) simde_mm256_maskz_mov_epi32(k, simde_mm256_rol_epi32(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_rol_epi32 + #define _mm256_maskz_rol_epi32(k, a, imm8) simde_mm256_maskz_rol_epi32(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_rol_epi32(a, imm8) _mm512_rol_epi32(a, imm8) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512i + simde_mm512_rol_epi32 (simde__m512i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE (imm8, 0, 255) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_i32 = vec_rl(a_.m128i_private[i].altivec_i32, vec_splats(HEDLEY_STATIC_CAST(unsigned int, imm8))); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + switch (imm8 & 31) { + case 0: + r_ = a_; + break; + default: + r_.u32 = (a_.u32 << (imm8 & 31)) | (a_.u32 >> (32 - (imm8 & 31))); + break; + } + #else + switch (imm8 & 31) { + case 0: + r_ = a_; + break; + default: + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] << (imm8 & 31)) | (a_.u32[i] >> (32 - (imm8 & 31))); + } + break; + } + #endif + + return simde__m512i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_rol_epi32 + #define _mm512_rol_epi32(a, imm8) simde_mm512_rol_epi32(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_rol_epi32(src, k, a, imm8) _mm512_mask_rol_epi32(src, k, a, imm8) +#else + #define simde_mm512_mask_rol_epi32(src, k, a, imm8) simde_mm512_mask_mov_epi32(src, k, simde_mm512_rol_epi32(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_rol_epi32 + #define _mm512_mask_rol_epi32(src, k, a, imm8) simde_mm512_mask_rol_epi32(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_maskz_rol_epi32(k, a, imm8) _mm512_maskz_rol_epi32(k, a, imm8) +#else + #define simde_mm512_maskz_rol_epi32(k, a, imm8) simde_mm512_maskz_mov_epi32(k, simde_mm512_rol_epi32(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_rol_epi32 + #define _mm512_maskz_rol_epi32(k, a, imm8) simde_mm512_maskz_rol_epi32(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_rol_epi64(a, imm8) _mm_rol_epi64(a, imm8) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128i + simde_mm_rol_epi64 (simde__m128i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE (imm8, 0, 255) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + r_.altivec_i64 = vec_rl(a_.altivec_i64, vec_splats(HEDLEY_STATIC_CAST(unsigned long long, imm8))); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + switch (imm8 & 63) { + case 0: + r_ = a_; + break; + default: + r_.u64 = (a_.u64 << (imm8 & 63)) | (a_.u64 >> (64 - (imm8 & 63))); + break; + } + #else + switch (imm8 & 63) { + case 0: + r_ = a_; + break; + default: + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] << (imm8 & 63)) | (a_.u64[i] >> (64 - (imm8 & 63))); + } + break; + } + #endif + + return simde__m128i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_rol_epi64 + #define _mm_rol_epi64(a, imm8) simde_mm_rol_epi64(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_mask_rol_epi64(src, k, a, imm8) _mm_mask_rol_epi64(src, k, a, imm8) +#else + #define simde_mm_mask_rol_epi64(src, k, a, imm8) simde_mm_mask_mov_epi64(src, k, simde_mm_rol_epi64(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_rol_epi64 + #define _mm_mask_rol_epi64(src, k, a, imm8) simde_mm_mask_rol_epi64(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_maskz_rol_epi64(k, a, imm8) _mm_maskz_rol_epi64(k, a, imm8) +#else + #define simde_mm_maskz_rol_epi64(k, a, imm8) simde_mm_maskz_mov_epi64(k, simde_mm_rol_epi64(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_rol_epi64 + #define _mm_maskz_rol_epi64(k, a, imm8) simde_mm_maskz_rol_epi64(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_rol_epi64(a, imm8) _mm256_rol_epi64(a, imm8) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m256i + simde_mm256_rol_epi64 (simde__m256i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE (imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_i64 = vec_rl(a_.m128i_private[i].altivec_i64, vec_splats(HEDLEY_STATIC_CAST(unsigned long long, imm8))); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + switch (imm8 & 63) { + case 0: + r_ = a_; + break; + default: + r_.u64 = (a_.u64 << (imm8 & 63)) | (a_.u64 >> (64 - (imm8 & 63))); + break; + } + #else + switch (imm8 & 63) { + case 0: + r_ = a_; + break; + default: + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] << (imm8 & 63)) | (a_.u64[i] >> (64 - (imm8 & 63))); + } + break; + } + #endif + + return simde__m256i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_rol_epi64 + #define _mm256_rol_epi64(a, imm8) simde_mm256_rol_epi64(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_rol_epi64(src, k, a, imm8) _mm256_mask_rol_epi64(src, k, a, imm8) +#else + #define simde_mm256_mask_rol_epi64(src, k, a, imm8) simde_mm256_mask_mov_epi64(src, k, simde_mm256_rol_epi64(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_rol_epi64 + #define _mm256_mask_rol_epi64(src, k, a, imm8) simde_mm256_mask_rol_epi64(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_maskz_rol_epi64(k, a, imm8) _mm256_maskz_rol_epi64(k, a, imm8) +#else + #define simde_mm256_maskz_rol_epi64(k, a, imm8) simde_mm256_maskz_mov_epi64(k, simde_mm256_rol_epi64(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_rol_epi64 + #define _mm256_maskz_rol_epi64(k, a, imm8) simde_mm256_maskz_rol_epi64(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_rol_epi64(a, imm8) _mm512_rol_epi64(a, imm8) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512i + simde_mm512_rol_epi64 (simde__m512i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE (imm8, 0, 255) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_i64 = vec_rl(a_.m128i_private[i].altivec_i64, vec_splats(HEDLEY_STATIC_CAST(unsigned long long, imm8))); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + switch (imm8 & 63) { + case 0: + r_ = a_; + break; + default: + r_.u64 = (a_.u64 << (imm8 & 63)) | (a_.u64 >> (64 - (imm8 & 63))); + break; + } + #else + switch (imm8 & 63) { + case 0: + r_ = a_; + break; + default: + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] << (imm8 & 63)) | (a_.u64[i] >> (64 - (imm8 & 63))); + } + break; + } + #endif + + return simde__m512i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_rol_epi64 + #define _mm512_rol_epi64(a, imm8) simde_mm512_rol_epi64(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_rol_epi64(src, k, a, imm8) _mm512_mask_rol_epi64(src, k, a, imm8) +#else + #define simde_mm512_mask_rol_epi64(src, k, a, imm8) simde_mm512_mask_mov_epi64(src, k, simde_mm512_rol_epi64(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_rol_epi64 + #define _mm512_mask_rol_epi64(src, k, a, imm8) simde_mm512_mask_rol_epi64(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_maskz_rol_epi64(k, a, imm8) _mm512_maskz_rol_epi64(k, a, imm8) +#else + #define simde_mm512_maskz_rol_epi64(k, a, imm8) simde_mm512_maskz_mov_epi64(k, simde_mm512_rol_epi64(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_rol_epi64 + #define _mm512_maskz_rol_epi64(k, a, imm8) simde_mm512_maskz_rol_epi64(k, a, imm8) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_ROL_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rolv.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rolv.h new file mode 100644 index 00000000..a14442ff --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rolv.h @@ -0,0 +1,415 @@ +#if !defined(SIMDE_X86_AVX512_ROLV_H) +#define SIMDE_X86_AVX512_ROLV_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" +#include "srlv.h" +#include "sllv.h" +#include "or.h" +#include "and.h" +#include "sub.h" +#include "set1.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rolv_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_rolv_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_u32 = vec_rl(a_.altivec_u32, b_.altivec_u32); + + return simde__m128i_from_private(r_); + #else + HEDLEY_STATIC_CAST(void, r_); + HEDLEY_STATIC_CAST(void, a_); + HEDLEY_STATIC_CAST(void, b_); + + simde__m128i + count1 = simde_mm_and_si128(b, simde_mm_set1_epi32(31)), + count2 = simde_mm_sub_epi32(simde_mm_set1_epi32(32), count1); + + return simde_mm_or_si128(simde_mm_sllv_epi32(a, count1), simde_mm_srlv_epi32(a, count2)); + #endif + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_rolv_epi32 + #define _mm_rolv_epi32(a, b) simde_mm_rolv_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_rolv_epi32 (simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_rolv_epi32(src, k, a, b); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_rolv_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_rolv_epi32 + #define _mm_mask_rolv_epi32(src, k, a, b) simde_mm_mask_rolv_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_rolv_epi32 (simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_rolv_epi32(k, a, b); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_rolv_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_rolv_epi32 + #define _mm_maskz_rolv_epi32(k, a, b) simde_mm_maskz_rolv_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_rolv_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_rolv_epi32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_u32 = vec_rl(a_.m128i_private[i].altivec_u32, b_.m128i_private[i].altivec_u32); + } + + return simde__m256i_from_private(r_); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_rolv_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_rolv_epi32(a_.m128i[1], b_.m128i[1]); + + return simde__m256i_from_private(r_); + #else + HEDLEY_STATIC_CAST(void, r_); + HEDLEY_STATIC_CAST(void, a_); + HEDLEY_STATIC_CAST(void, b_); + + simde__m256i + count1 = simde_mm256_and_si256(b, simde_mm256_set1_epi32(31)), + count2 = simde_mm256_sub_epi32(simde_mm256_set1_epi32(32), count1); + + return simde_mm256_or_si256(simde_mm256_sllv_epi32(a, count1), simde_mm256_srlv_epi32(a, count2)); + #endif + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_rolv_epi32 + #define _mm256_rolv_epi32(a, b) simde_mm256_rolv_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_rolv_epi32 (simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_rolv_epi32(src, k, a, b); + #else + return simde_mm256_mask_mov_epi32(src, k, simde_mm256_rolv_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_rolv_epi32 + #define _mm256_mask_rolv_epi32(src, k, a, b) simde_mm256_mask_rolv_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_rolv_epi32 (simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_rolv_epi32(k, a, b); + #else + return simde_mm256_maskz_mov_epi32(k, simde_mm256_rolv_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_rolv_epi32 + #define _mm256_maskz_rolv_epi32(k, a, b) simde_mm256_maskz_rolv_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_rolv_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rolv_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_u32 = vec_rl(a_.m128i_private[i].altivec_u32, b_.m128i_private[i].altivec_u32); + } + + return simde__m512i_from_private(r_); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_rolv_epi32(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_rolv_epi32(a_.m256i[1], b_.m256i[1]); + + return simde__m512i_from_private(r_); + #else + HEDLEY_STATIC_CAST(void, r_); + HEDLEY_STATIC_CAST(void, a_); + HEDLEY_STATIC_CAST(void, b_); + + simde__m512i + count1 = simde_mm512_and_si512(b, simde_mm512_set1_epi32(31)), + count2 = simde_mm512_sub_epi32(simde_mm512_set1_epi32(32), count1); + + return simde_mm512_or_si512(simde_mm512_sllv_epi32(a, count1), simde_mm512_srlv_epi32(a, count2)); + #endif + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_rolv_epi32 + #define _mm512_rolv_epi32(a, b) simde_mm512_rolv_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_rolv_epi32 (simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_rolv_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_rolv_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_rolv_epi32 + #define _mm512_mask_rolv_epi32(src, k, a, b) simde_mm512_mask_rolv_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_rolv_epi32 (simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_rolv_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_rolv_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_rolv_epi32 + #define _mm512_maskz_rolv_epi32(k, a, b) simde_mm512_maskz_rolv_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rolv_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_rolv_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + r_.altivec_u64 = vec_rl(a_.altivec_u64, b_.altivec_u64); + + return simde__m128i_from_private(r_); + #else + HEDLEY_STATIC_CAST(void, r_); + HEDLEY_STATIC_CAST(void, a_); + HEDLEY_STATIC_CAST(void, b_); + + simde__m128i + count1 = simde_mm_and_si128(b, simde_mm_set1_epi64x(63)), + count2 = simde_mm_sub_epi64(simde_mm_set1_epi64x(64), count1); + + return simde_mm_or_si128(simde_mm_sllv_epi64(a, count1), simde_mm_srlv_epi64(a, count2)); + #endif + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_rolv_epi64 + #define _mm_rolv_epi64(a, b) simde_mm_rolv_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_rolv_epi64 (simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_rolv_epi64(src, k, a, b); + #else + return simde_mm_mask_mov_epi64(src, k, simde_mm_rolv_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_rolv_epi64 + #define _mm_mask_rolv_epi64(src, k, a, b) simde_mm_mask_rolv_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_rolv_epi64 (simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_rolv_epi64(k, a, b); + #else + return simde_mm_maskz_mov_epi64(k, simde_mm_rolv_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_rolv_epi64 + #define _mm_maskz_rolv_epi64(k, a, b) simde_mm_maskz_rolv_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_rolv_epi64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_rolv_epi64(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_u64 = vec_rl(a_.m128i_private[i].altivec_u64, b_.m128i_private[i].altivec_u64); + } + + return simde__m256i_from_private(r_); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) + r_.m128i[0] = simde_mm_rolv_epi64(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_rolv_epi64(a_.m128i[1], b_.m128i[1]); + + return simde__m256i_from_private(r_); + #else + HEDLEY_STATIC_CAST(void, r_); + HEDLEY_STATIC_CAST(void, a_); + HEDLEY_STATIC_CAST(void, b_); + + simde__m256i + count1 = simde_mm256_and_si256(b, simde_mm256_set1_epi64x(63)), + count2 = simde_mm256_sub_epi64(simde_mm256_set1_epi64x(64), count1); + + return simde_mm256_or_si256(simde_mm256_sllv_epi64(a, count1), simde_mm256_srlv_epi64(a, count2)); + #endif + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_rolv_epi64 + #define _mm256_rolv_epi64(a, b) simde_mm256_rolv_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_rolv_epi64 (simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_rolv_epi64(src, k, a, b); + #else + return simde_mm256_mask_mov_epi64(src, k, simde_mm256_rolv_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_rolv_epi64 + #define _mm256_mask_rolv_epi64(src, k, a, b) simde_mm256_mask_rolv_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_rolv_epi64 (simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_rolv_epi64(k, a, b); + #else + return simde_mm256_maskz_mov_epi64(k, simde_mm256_rolv_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_rolv_epi64 + #define _mm256_maskz_rolv_epi64(k, a, b) simde_mm256_maskz_rolv_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_rolv_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rolv_epi64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_u64 = vec_rl(a_.m128i_private[i].altivec_u64, b_.m128i_private[i].altivec_u64); + } + + return simde__m512i_from_private(r_); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_rolv_epi64(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_rolv_epi64(a_.m256i[1], b_.m256i[1]); + + return simde__m512i_from_private(r_); + #else + HEDLEY_STATIC_CAST(void, r_); + HEDLEY_STATIC_CAST(void, a_); + HEDLEY_STATIC_CAST(void, b_); + + simde__m512i + count1 = simde_mm512_and_si512(b, simde_mm512_set1_epi64(63)), + count2 = simde_mm512_sub_epi64(simde_mm512_set1_epi64(64), count1); + + return simde_mm512_or_si512(simde_mm512_sllv_epi64(a, count1), simde_mm512_srlv_epi64(a, count2)); + #endif + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_rolv_epi64 + #define _mm512_rolv_epi64(a, b) simde_mm512_rolv_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_rolv_epi64 (simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_rolv_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_rolv_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_rolv_epi64 + #define _mm512_mask_rolv_epi64(src, k, a, b) simde_mm512_mask_rolv_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_rolv_epi64 (simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_rolv_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_rolv_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_rolv_epi64 + #define _mm512_maskz_rolv_epi64(k, a, b) simde_mm512_maskz_rolv_epi64(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_ROLV_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/ror.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/ror.h new file mode 100644 index 00000000..7cac56c7 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/ror.h @@ -0,0 +1,410 @@ +#if !defined(SIMDE_X86_AVX512_ROR_H) +#define SIMDE_X86_AVX512_ROR_H + +#include "types.h" +#include "mov.h" +#include "or.h" +#include "srli.h" +#include "slli.h" +#include "../avx2.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_ror_epi32(a, imm8) _mm_ror_epi32(a, imm8) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128i + simde_mm_ror_epi32 (simde__m128i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE (imm8, 0, 255) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i32 = vec_rl(a_.altivec_i32, vec_splats(HEDLEY_STATIC_CAST(unsigned int, 32 - imm8))); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + switch (imm8 & 31) { + case 0: + r_ = a_; + break; + default: + r_.u32 = (a_.u32 >> (imm8 & 31)) | (a_.u32 << (32 - (imm8 & 31))); + break; + } + #else + switch (imm8 & 31) { + case 0: + r_ = a_; + break; + default: + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] >> (imm8 & 31)) | (a_.u32[i] << (32 - (imm8 & 31))); + } + break; + } + #endif + + return simde__m128i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_ror_epi32 + #define _mm_ror_epi32(a, imm8) simde_mm_ror_epi32(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_mask_ror_epi32(src, k, a, imm8) _mm_mask_ror_epi32(src, k, a, imm8) +#else + #define simde_mm_mask_ror_epi32(src, k, a, imm8) simde_mm_mask_mov_epi32(src, k, simde_mm_ror_epi32(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_ror_epi32 + #define _mm_mask_ror_epi32(src, k, a, imm8) simde_mm_mask_ror_epi32(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_maskz_ror_epi32(k, a, imm8) _mm_maskz_ror_epi32(k, a, imm8) +#else + #define simde_mm_maskz_ror_epi32(k, a, imm8) simde_mm_maskz_mov_epi32(k, simde_mm_ror_epi32(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_ror_epi32 + #define _mm_maskz_ror_epi32(k, a, imm8) simde_mm_maskz_ror_epi32(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_ror_epi32(a, imm8) _mm256_ror_epi32(a, imm8) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m256i + simde_mm256_ror_epi32 (simde__m256i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE (imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_i32 = vec_rl(a_.m128i_private[i].altivec_i32, vec_splats(HEDLEY_STATIC_CAST(unsigned int, 32 - imm8))); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + switch (imm8 & 31) { + case 0: + r_ = a_; + break; + default: + r_.u32 = (a_.u32 >> (imm8 & 31)) | (a_.u32 << (32 - (imm8 & 31))); + break; + } + #else + switch (imm8 & 31) { + case 0: + r_ = a_; + break; + default: + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] >> (imm8 & 31)) | (a_.u32[i] << (32 - (imm8 & 31))); + } + break; + } + #endif + + return simde__m256i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_ror_epi32 + #define _mm256_ror_epi32(a, imm8) simde_mm256_ror_epi32(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_ror_epi32(src, k, a, imm8) _mm256_mask_ror_epi32(src, k, a, imm8) +#else + #define simde_mm256_mask_ror_epi32(src, k, a, imm8) simde_mm256_mask_mov_epi32(src, k, simde_mm256_ror_epi32(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_ror_epi32 + #define _mm256_mask_ror_epi32(src, k, a, imm8) simde_mm256_mask_ror_epi32(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_maskz_ror_epi32(k, a, imm8) _mm256_maskz_ror_epi32(k, a, imm8) +#else + #define simde_mm256_maskz_ror_epi32(k, a, imm8) simde_mm256_maskz_mov_epi32(k, simde_mm256_ror_epi32(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_ror_epi32 + #define _mm256_maskz_ror_epi32(k, a, imm8) simde_mm256_maskz_ror_epi32(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_ror_epi32(a, imm8) _mm512_ror_epi32(a, imm8) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512i + simde_mm512_ror_epi32 (simde__m512i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE (imm8, 0, 255) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_i32 = vec_rl(a_.m128i_private[i].altivec_i32, vec_splats(HEDLEY_STATIC_CAST(unsigned int, 32 - imm8))); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + switch (imm8 & 31) { + case 0: + r_ = a_; + break; + default: + r_.u32 = (a_.u32 >> (imm8 & 31)) | (a_.u32 << (32 - (imm8 & 31))); + break; + } + #else + switch (imm8 & 31) { + case 0: + r_ = a_; + break; + default: + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] >> (imm8 & 31)) | (a_.u32[i] << (32 - (imm8 & 31))); + } + break; + } + #endif + + return simde__m512i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_ror_epi32 + #define _mm512_ror_epi32(a, imm8) simde_mm512_ror_epi32(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_ror_epi32(src, k, a, imm8) _mm512_mask_ror_epi32(src, k, a, imm8) +#else + #define simde_mm512_mask_ror_epi32(src, k, a, imm8) simde_mm512_mask_mov_epi32(src, k, simde_mm512_ror_epi32(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_ror_epi32 + #define _mm512_mask_ror_epi32(src, k, a, imm8) simde_mm512_mask_ror_epi32(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_maskz_ror_epi32(k, a, imm8) _mm512_maskz_ror_epi32(k, a, imm8) +#else + #define simde_mm512_maskz_ror_epi32(k, a, imm8) simde_mm512_maskz_mov_epi32(k, simde_mm512_ror_epi32(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_ror_epi32 + #define _mm512_maskz_ror_epi32(k, a, imm8) simde_mm512_maskz_ror_epi32(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_ror_epi64(a, imm8) _mm_ror_epi64(a, imm8) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128i + simde_mm_ror_epi64 (simde__m128i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE (imm8, 0, 255) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + r_.altivec_i64 = vec_rl(a_.altivec_i64, vec_splats(HEDLEY_STATIC_CAST(unsigned long long, 64 - imm8))); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + switch (imm8 & 63) { + case 0: + r_ = a_; + break; + default: + r_.u64 = (a_.u64 >> (imm8 & 63)) | (a_.u64 << (64 - (imm8 & 63))); + break; + } + #else + switch (imm8 & 63) { + case 0: + r_ = a_; + break; + default: + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] >> (imm8 & 63)) | (a_.u64[i] << (64 - (imm8 & 63))); + } + break; + } + #endif + + return simde__m128i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_ror_epi64 + #define _mm_ror_epi64(a, imm8) simde_mm_ror_epi64(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_mask_ror_epi64(src, k, a, imm8) _mm_mask_ror_epi64(src, k, a, imm8) +#else + #define simde_mm_mask_ror_epi64(src, k, a, imm8) simde_mm_mask_mov_epi64(src, k, simde_mm_ror_epi64(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_ror_epi64 + #define _mm_mask_ror_epi64(src, k, a, imm8) simde_mm_mask_ror_epi64(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_maskz_ror_epi64(k, a, imm8) _mm_maskz_ror_epi64(k, a, imm8) +#else + #define simde_mm_maskz_ror_epi64(k, a, imm8) simde_mm_maskz_mov_epi64(k, simde_mm_ror_epi64(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_ror_epi64 + #define _mm_maskz_ror_epi64(k, a, imm8) simde_mm_maskz_ror_epi64(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_ror_epi64(a, imm8) _mm256_ror_epi64(a, imm8) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m256i + simde_mm256_ror_epi64 (simde__m256i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE (imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_i64 = vec_rl(a_.m128i_private[i].altivec_i64, vec_splats(HEDLEY_STATIC_CAST(unsigned long long, 64 - imm8))); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + switch (imm8 & 63) { + case 0: + r_ = a_; + break; + default: + r_.u64 = (a_.u64 >> (imm8 & 63)) | (a_.u64 << (64 - (imm8 & 63))); + break; + } + #else + switch (imm8 & 63) { + case 0: + r_ = a_; + break; + default: + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] >> (imm8 & 63)) | (a_.u64[i] << (64 - (imm8 & 63))); + } + break; + } + #endif + + return simde__m256i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_ror_epi64 + #define _mm256_ror_epi64(a, imm8) simde_mm256_ror_epi64(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_ror_epi64(src, k, a, imm8) _mm256_mask_ror_epi64(src, k, a, imm8) +#else + #define simde_mm256_mask_ror_epi64(src, k, a, imm8) simde_mm256_mask_mov_epi64(src, k, simde_mm256_ror_epi64(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_ror_epi64 + #define _mm256_mask_ror_epi64(src, k, a, imm8) simde_mm256_mask_ror_epi64(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_maskz_ror_epi64(k, a, imm8) _mm256_maskz_ror_epi64(k, a, imm8) +#else + #define simde_mm256_maskz_ror_epi64(k, a, imm8) simde_mm256_maskz_mov_epi64(k, simde_mm256_ror_epi64(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_ror_epi64 + #define _mm256_maskz_ror_epi64(k, a, imm8) simde_mm256_maskz_ror_epi64(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_ror_epi64(a, imm8) _mm512_ror_epi64(a, imm8) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512i + simde_mm512_ror_epi64 (simde__m512i a, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE (imm8, 0, 255) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_i64 = vec_rl(a_.m128i_private[i].altivec_i64, vec_splats(HEDLEY_STATIC_CAST(unsigned long long, 64 - imm8))); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + switch (imm8 & 63) { + case 0: + r_ = a_; + break; + default: + r_.u64 = (a_.u64 >> (imm8 & 63)) | (a_.u64 << (64 - (imm8 & 63))); + break; + } + #else + switch (imm8 & 63) { + case 0: + r_ = a_; + break; + default: + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] >> (imm8 & 63)) | (a_.u64[i] << (64 - (imm8 & 63))); + } + break; + } + #endif + + return simde__m512i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_ror_epi64 + #define _mm512_ror_epi64(a, imm8) simde_mm512_ror_epi64(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_ror_epi64(src, k, a, imm8) _mm512_mask_ror_epi64(src, k, a, imm8) +#else + #define simde_mm512_mask_ror_epi64(src, k, a, imm8) simde_mm512_mask_mov_epi64(src, k, simde_mm512_ror_epi64(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_ror_epi64 + #define _mm512_mask_ror_epi64(src, k, a, imm8) simde_mm512_mask_ror_epi64(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_maskz_ror_epi64(k, a, imm8) _mm512_maskz_ror_epi64(k, a, imm8) +#else + #define simde_mm512_maskz_ror_epi64(k, a, imm8) simde_mm512_maskz_mov_epi64(k, simde_mm512_ror_epi64(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_ror_epi64 + #define _mm512_maskz_ror_epi64(k, a, imm8) simde_mm512_maskz_ror_epi64(k, a, imm8) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_ROR_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rorv.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rorv.h new file mode 100644 index 00000000..ae87cec8 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/rorv.h @@ -0,0 +1,391 @@ +#if !defined(SIMDE_X86_AVX512_RORV_H) +#define SIMDE_X86_AVX512_RORV_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" +#include "srlv.h" +#include "sllv.h" +#include "or.h" +#include "and.h" +#include "sub.h" +#include "set1.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rorv_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_rorv_epi32(a, b); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + r_.altivec_i32 = vec_rl(a_.altivec_i32, vec_sub(vec_splats(HEDLEY_STATIC_CAST(unsigned int, 32)), b_.altivec_u32)); + return simde__m128i_from_private(r_); + #else + simde__m128i + count1 = simde_mm_and_si128(b, simde_mm_set1_epi32(31)), + count2 = simde_mm_sub_epi32(simde_mm_set1_epi32(32), count1); + return simde_mm_or_si128(simde_mm_srlv_epi32(a, count1), simde_mm_sllv_epi32(a, count2)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_rorv_epi32 + #define _mm_rorv_epi32(a, b) simde_mm_rorv_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_rorv_epi32 (simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_rorv_epi32(src, k, a, b); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_rorv_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_rorv_epi32 + #define _mm_mask_rorv_epi32(src, k, a, b) simde_mm_mask_rorv_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_rorv_epi32 (simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_rorv_epi32(k, a, b); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_rorv_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_rorv_epi32 + #define _mm_maskz_rorv_epi32(k, a, b) simde_mm_maskz_rorv_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_rorv_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_rorv_epi32(a, b); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_i32 = vec_rl(a_.m128i_private[i].altivec_i32, vec_sub(vec_splats(HEDLEY_STATIC_CAST(unsigned int, 32)), b_.m128i_private[i].altivec_u32)); + } + + return simde__m256i_from_private(r_); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + r_.m128i[0] = simde_mm_rorv_epi32(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_rorv_epi32(a_.m128i[1], b_.m128i[1]); + + return simde__m256i_from_private(r_); + #else + simde__m256i + count1 = simde_mm256_and_si256(b, simde_mm256_set1_epi32(31)), + count2 = simde_mm256_sub_epi32(simde_mm256_set1_epi32(32), count1); + return simde_mm256_or_si256(simde_mm256_srlv_epi32(a, count1), simde_mm256_sllv_epi32(a, count2)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_rorv_epi32 + #define _mm256_rorv_epi32(a, b) simde_mm256_rorv_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_rorv_epi32 (simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_rorv_epi32(src, k, a, b); + #else + return simde_mm256_mask_mov_epi32(src, k, simde_mm256_rorv_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_rorv_epi32 + #define _mm256_mask_rorv_epi32(src, k, a, b) simde_mm256_mask_rorv_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_rorv_epi32 (simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_rorv_epi32(k, a, b); + #else + return simde_mm256_maskz_mov_epi32(k, simde_mm256_rorv_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_rorv_epi32 + #define _mm256_maskz_rorv_epi32(k, a, b) simde_mm256_maskz_rorv_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_rorv_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rorv_epi32(a, b); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_i32 = vec_rl(a_.m128i_private[i].altivec_i32, vec_sub(vec_splats(HEDLEY_STATIC_CAST(unsigned int, 32)), b_.m128i_private[i].altivec_u32)); + } + + return simde__m512i_from_private(r_); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + r_.m256i[0] = simde_mm256_rorv_epi32(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_rorv_epi32(a_.m256i[1], b_.m256i[1]); + + return simde__m512i_from_private(r_); + #else + simde__m512i + count1 = simde_mm512_and_si512(b, simde_mm512_set1_epi32(31)), + count2 = simde_mm512_sub_epi32(simde_mm512_set1_epi32(32), count1); + return simde_mm512_or_si512(simde_mm512_srlv_epi32(a, count1), simde_mm512_sllv_epi32(a, count2)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_rorv_epi32 + #define _mm512_rorv_epi32(a, b) simde_mm512_rorv_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_rorv_epi32 (simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_rorv_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_rorv_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_rorv_epi32 + #define _mm512_mask_rorv_epi32(src, k, a, b) simde_mm512_mask_rorv_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_rorv_epi32 (simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_rorv_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_rorv_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_rorv_epi32 + #define _mm512_maskz_rorv_epi32(k, a, b) simde_mm512_maskz_rorv_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rorv_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_rorv_epi64(a, b); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + r_.altivec_i64 = vec_rl(a_.altivec_i64, vec_sub(vec_splats(HEDLEY_STATIC_CAST(unsigned long long, 64)), b_.altivec_u64)); + return simde__m128i_from_private(r_); + #else + simde__m128i + count1 = simde_mm_and_si128(b, simde_mm_set1_epi64x(63)), + count2 = simde_mm_sub_epi64(simde_mm_set1_epi64x(64), count1); + return simde_mm_or_si128(simde_mm_srlv_epi64(a, count1), simde_mm_sllv_epi64(a, count2)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_rorv_epi64 + #define _mm_rorv_epi64(a, b) simde_mm_rorv_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_rorv_epi64 (simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_rorv_epi64(src, k, a, b); + #else + return simde_mm_mask_mov_epi64(src, k, simde_mm_rorv_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_rorv_epi64 + #define _mm_mask_rorv_epi64(src, k, a, b) simde_mm_mask_rorv_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_rorv_epi64 (simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_rorv_epi64(k, a, b); + #else + return simde_mm_maskz_mov_epi64(k, simde_mm_rorv_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_rorv_epi64 + #define _mm_maskz_rorv_epi64(k, a, b) simde_mm_maskz_rorv_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_rorv_epi64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_rorv_epi64(a, b); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_i64 = vec_rl(a_.m128i_private[i].altivec_i64, vec_sub(vec_splats(HEDLEY_STATIC_CAST(unsigned long long, 64)), b_.m128i_private[i].altivec_u64)); + } + + return simde__m256i_from_private(r_); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + r_.m128i[0] = simde_mm_rorv_epi64(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_rorv_epi64(a_.m128i[1], b_.m128i[1]); + + return simde__m256i_from_private(r_); + #else + simde__m256i + count1 = simde_mm256_and_si256(b, simde_mm256_set1_epi64x(63)), + count2 = simde_mm256_sub_epi64(simde_mm256_set1_epi64x(64), count1); + return simde_mm256_or_si256(simde_mm256_srlv_epi64(a, count1), simde_mm256_sllv_epi64(a, count2)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_rorv_epi64 + #define _mm256_rorv_epi64(a, b) simde_mm256_rorv_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_rorv_epi64 (simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_rorv_epi64(src, k, a, b); + #else + return simde_mm256_mask_mov_epi64(src, k, simde_mm256_rorv_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_rorv_epi64 + #define _mm256_mask_rorv_epi64(src, k, a, b) simde_mm256_mask_rorv_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_rorv_epi64 (simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_rorv_epi64(k, a, b); + #else + return simde_mm256_maskz_mov_epi64(k, simde_mm256_rorv_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_rorv_epi64 + #define _mm256_maskz_rorv_epi64(k, a, b) simde_mm256_maskz_rorv_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_rorv_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rorv_epi64(a, b); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + for (size_t i = 0 ; i < (sizeof(r_.m128i_private) / sizeof(r_.m128i_private[0])) ; i++) { + r_.m128i_private[i].altivec_i64 = vec_rl(a_.m128i_private[i].altivec_i64, vec_sub(vec_splats(HEDLEY_STATIC_CAST(unsigned long long, 64)), b_.m128i_private[i].altivec_u64)); + } + + return simde__m512i_from_private(r_); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + r_.m256i[0] = simde_mm256_rorv_epi64(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_rorv_epi64(a_.m256i[1], b_.m256i[1]); + + return simde__m512i_from_private(r_); + #else + simde__m512i + count1 = simde_mm512_and_si512(b, simde_mm512_set1_epi64(63)), + count2 = simde_mm512_sub_epi64(simde_mm512_set1_epi64(64), count1); + return simde_mm512_or_si512(simde_mm512_srlv_epi64(a, count1), simde_mm512_sllv_epi64(a, count2)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_rorv_epi64 + #define _mm512_rorv_epi64(a, b) simde_mm512_rorv_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_rorv_epi64 (simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_rorv_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_rorv_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_rorv_epi64 + #define _mm512_mask_rorv_epi64(src, k, a, b) simde_mm512_mask_rorv_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_rorv_epi64 (simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_rorv_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_rorv_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_rorv_epi64 + #define _mm512_maskz_rorv_epi64(k, a, b) simde_mm512_maskz_rorv_epi64(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_RORV_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/round.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/round.h new file mode 100644 index 00000000..684dbe04 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/round.h @@ -0,0 +1,282 @@ +#if !defined(SIMDE_X86_AVX512_ROUND_H) +#define SIMDE_X86_AVX512_ROUND_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#if SIMDE_NATURAL_VECTOR_SIZE_LE(256) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_x_mm512_round_ps(a, rounding) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512_private \ + simde_x_mm512_round_ps_r_ = simde__m512_to_private(simde_mm512_setzero_ps()), \ + simde_x_mm512_round_ps_a_ = simde__m512_to_private(a); \ + \ + for (size_t simde_x_mm512_round_ps_i = 0 ; simde_x_mm512_round_ps_i < (sizeof(simde_x_mm512_round_ps_r_.m256) / sizeof(simde_x_mm512_round_ps_r_.m256[0])) ; simde_x_mm512_round_ps_i++) { \ + simde_x_mm512_round_ps_r_.m256[simde_x_mm512_round_ps_i] = simde_mm256_round_ps(simde_x_mm512_round_ps_a_.m256[simde_x_mm512_round_ps_i], rounding); \ + } \ + \ + simde__m512_from_private(simde_x_mm512_round_ps_r_); \ + })) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512 + simde_x_mm512_round_ps (simde__m512 a, int rounding) + SIMDE_REQUIRE_CONSTANT_RANGE(rounding, 0, 15) { + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + /* For architectures which lack a current direction SIMD instruction. + * + * Note that NEON actually has a current rounding mode instruction, + * but in ARMv8+ the rounding mode is ignored and nearest is always + * used, so we treat ARMv7 as having a rounding mode but ARMv8 as + * not. */ + #if \ + defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || \ + defined(SIMDE_ARM_NEON_A32V8) + if ((rounding & 7) == SIMDE_MM_FROUND_CUR_DIRECTION) + rounding = HEDLEY_STATIC_CAST(int, SIMDE_MM_GET_ROUNDING_MODE()) << 13; + #endif + + switch (rounding & ~SIMDE_MM_FROUND_NO_EXC) { + case SIMDE_MM_FROUND_CUR_DIRECTION: + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128_private) / sizeof(r_.m128_private[0])) ; i++) { + r_.m128_private[i].altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_round(a_.m128_private[i].altivec_f32)); + } + #elif defined(SIMDE_ARM_NEON_A32V8_NATIVE) && !defined(SIMDE_BUG_GCC_95399) + for (size_t i = 0 ; i < (sizeof(r_.m128_private) / sizeof(r_.m128_private[0])) ; i++) { + r_.m128_private[i].neon_f32 = vrndiq_f32(a_.m128_private[i].neon_f32); + } + #elif defined(simde_math_nearbyintf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_nearbyintf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm512_setzero_ps()); + #endif + break; + + case SIMDE_MM_FROUND_TO_NEAREST_INT: + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128_private) / sizeof(r_.m128_private[0])) ; i++) { + r_.m128_private[i].altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_rint(a_.m128_private[i].altivec_f32)); + } + #elif defined(SIMDE_ARM_NEON_A32V8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128_private) / sizeof(r_.m128_private[0])) ; i++) { + r_.m128_private[i].neon_f32 = vrndnq_f32(a_.m128_private[i].neon_f32); + } + #elif defined(simde_math_roundevenf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_roundevenf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm512_setzero_ps()); + #endif + break; + + case SIMDE_MM_FROUND_TO_NEG_INF: + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128_private) / sizeof(r_.m128_private[0])) ; i++) { + r_.m128_private[i].altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_floor(a_.m128_private[i].altivec_f32)); + } + #elif defined(SIMDE_ARM_NEON_A32V8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128_private) / sizeof(r_.m128_private[0])) ; i++) { + r_.m128_private[i].neon_f32 = vrndmq_f32(a_.m128_private[i].neon_f32); + } + #elif defined(simde_math_floorf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_floorf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm512_setzero_ps()); + #endif + break; + + case SIMDE_MM_FROUND_TO_POS_INF: + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128_private) / sizeof(r_.m128_private[0])) ; i++) { + r_.m128_private[i].altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_ceil(a_.m128_private[i].altivec_f32)); + } + #elif defined(SIMDE_ARM_NEON_A32V8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128_private) / sizeof(r_.m128_private[0])) ; i++) { + r_.m128_private[i].neon_f32 = vrndpq_f32(a_.m128_private[i].neon_f32); + } + #elif defined(simde_math_ceilf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_ceilf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm512_setzero_ps()); + #endif + break; + + case SIMDE_MM_FROUND_TO_ZERO: + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128_private) / sizeof(r_.m128_private[0])) ; i++) { + r_.m128_private[i].altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_trunc(a_.m128_private[i].altivec_f32)); + } + #elif defined(SIMDE_ARM_NEON_A32V8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128_private) / sizeof(r_.m128_private[0])) ; i++) { + r_.m128_private[i].neon_f32 = vrndq_f32(a_.m128_private[i].neon_f32); + } + #elif defined(simde_math_truncf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_truncf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm512_setzero_ps()); + #endif + break; + + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm512_setzero_ps()); + } + + return simde__m512_from_private(r_); + } +#endif + +#if SIMDE_NATURAL_VECTOR_SIZE_LE(256) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_x_mm512_round_pd(a, rounding) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d_private \ + simde_x_mm512_round_pd_r_ = simde__m512d_to_private(simde_mm512_setzero_pd()), \ + simde_x_mm512_round_pd_a_ = simde__m512d_to_private(a); \ + \ + for (size_t simde_x_mm512_round_pd_i = 0 ; simde_x_mm512_round_pd_i < (sizeof(simde_x_mm512_round_pd_r_.m256d) / sizeof(simde_x_mm512_round_pd_r_.m256d[0])) ; simde_x_mm512_round_pd_i++) { \ + simde_x_mm512_round_pd_r_.m256d[simde_x_mm512_round_pd_i] = simde_mm256_round_pd(simde_x_mm512_round_pd_a_.m256d[simde_x_mm512_round_pd_i], rounding); \ + } \ + \ + simde__m512d_from_private(simde_x_mm512_round_pd_r_); \ + })) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512d + simde_x_mm512_round_pd (simde__m512d a, int rounding) + SIMDE_REQUIRE_CONSTANT_RANGE(rounding, 0, 15) { + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + /* For architectures which lack a current direction SIMD instruction. */ + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + if ((rounding & 7) == SIMDE_MM_FROUND_CUR_DIRECTION) + rounding = HEDLEY_STATIC_CAST(int, SIMDE_MM_GET_ROUNDING_MODE()) << 13; + #endif + + switch (rounding & ~SIMDE_MM_FROUND_NO_EXC) { + case SIMDE_MM_FROUND_CUR_DIRECTION: + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128d_private) / sizeof(r_.m128d_private[0])) ; i++) { + r_.m128d_private[i].altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_round(a_.m128d_private[i].altivec_f64)); + } + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128d_private) / sizeof(r_.m128d_private[0])) ; i++) { + r_.m128d_private[i].neon_f64 = vrndiq_f64(a_.m128d_private[i].neon_f64); + } + #elif defined(simde_math_nearbyint) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_nearbyint(a_.f64[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm512_setzero_pd()); + #endif + break; + + case SIMDE_MM_FROUND_TO_NEAREST_INT: + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128d_private) / sizeof(r_.m128d_private[0])) ; i++) { + r_.m128d_private[i].altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_round(a_.m128d_private[i].altivec_f64)); + } + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128d_private) / sizeof(r_.m128d_private[0])) ; i++) { + r_.m128d_private[i].neon_f64 = vrndaq_f64(a_.m128d_private[i].neon_f64); + } + #elif defined(simde_math_roundeven) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_roundeven(a_.f64[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm512_setzero_pd()); + #endif + break; + + case SIMDE_MM_FROUND_TO_NEG_INF: + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128d_private) / sizeof(r_.m128d_private[0])) ; i++) { + r_.m128d_private[i].altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_floor(a_.m128d_private[i].altivec_f64)); + } + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128d_private) / sizeof(r_.m128d_private[0])) ; i++) { + r_.m128d_private[i].neon_f64 = vrndmq_f64(a_.m128d_private[i].neon_f64); + } + #elif defined(simde_math_floor) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_floor(a_.f64[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm512_setzero_pd()); + #endif + break; + + case SIMDE_MM_FROUND_TO_POS_INF: + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128d_private) / sizeof(r_.m128d_private[0])) ; i++) { + r_.m128d_private[i].altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_ceil(a_.m128d_private[i].altivec_f64)); + } + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128d_private) / sizeof(r_.m128d_private[0])) ; i++) { + r_.m128d_private[i].neon_f64 = vrndpq_f64(a_.m128d_private[i].neon_f64); + } + #elif defined(simde_math_ceil) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_ceil(a_.f64[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm512_setzero_pd()); + #endif + break; + + case SIMDE_MM_FROUND_TO_ZERO: + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128d_private) / sizeof(r_.m128d_private[0])) ; i++) { + r_.m128d_private[i].altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_trunc(a_.m128d_private[i].altivec_f64)); + } + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + for (size_t i = 0 ; i < (sizeof(r_.m128d_private) / sizeof(r_.m128d_private[0])) ; i++) { + r_.m128d_private[i].neon_f64 = vrndq_f64(a_.m128d_private[i].neon_f64); + } + #elif defined(simde_math_trunc) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_trunc(a_.f64[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm512_setzero_pd()); + #endif + break; + + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm512_setzero_pd()); + } + + return simde__m512d_from_private(r_); + } +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_ROUND_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/roundscale.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/roundscale.h new file mode 100644 index 00000000..80c9abf2 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/roundscale.h @@ -0,0 +1,616 @@ +#if !defined(SIMDE_X86_AVX512_ROUNDSCALE_H) +#define SIMDE_X86_AVX512_ROUNDSCALE_H + +#include "types.h" +#include "andnot.h" +#include "set1.h" +#include "mul.h" +#include "round.h" +#include "cmpeq.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_roundscale_ps(a, imm8) _mm_roundscale_ps((a), (imm8)) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_roundscale_ps_internal_ (simde__m128 result, simde__m128 a, int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + + simde__m128 r, clear_sign; + + clear_sign = simde_mm_andnot_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C(-0.0)), result); + r = simde_x_mm_select_ps(result, a, simde_mm_cmpeq_ps(clear_sign, simde_mm_set1_ps(SIMDE_MATH_INFINITYF))); + + return r; + } + #define simde_mm_roundscale_ps(a, imm8) \ + simde_mm_roundscale_ps_internal_( \ + simde_mm_mul_ps( \ + simde_mm_round_ps( \ + simde_mm_mul_ps( \ + a, \ + simde_mm_set1_ps(simde_math_exp2f(((imm8 >> 4) & 15)))), \ + ((imm8) & 15) \ + ), \ + simde_mm_set1_ps(simde_math_exp2f(-((imm8 >> 4) & 15))) \ + ), \ + (a), \ + (imm8) \ + ) +#endif +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_roundscale_ps + #define _mm_roundscale_ps(a, imm8) simde_mm_roundscale_ps(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_mask_roundscale_ps(src, k, a, imm8) _mm_mask_roundscale_ps(src, k, a, imm8) +#else + #define simde_mm_mask_roundscale_ps(src, k, a, imm8) simde_mm_mask_mov_ps(src, k, simde_mm_roundscale_ps(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_roundscale_ps + #define _mm_mask_roundscale_ps(src, k, a, imm8) simde_mm_mask_roundscale_ps(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_maskz_roundscale_ps(k, a, imm8) _mm_maskz_roundscale_ps(k, a, imm8) +#else + #define simde_mm_maskz_roundscale_ps(k, a, imm8) simde_mm_maskz_mov_ps(k, simde_mm_roundscale_ps(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_roundscale_ps + #define _mm_maskz_roundscale_ps(k, a, imm8) simde_mm_maskz_roundscale_ps(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm256_roundscale_ps(a, imm8) _mm256_roundscale_ps((a), (imm8)) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm256_roundscale_ps(a, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m256_private \ + simde_mm256_roundscale_ps_r_ = simde__m256_to_private(simde_mm256_setzero_ps()), \ + simde_mm256_roundscale_ps_a_ = simde__m256_to_private(a); \ + \ + for (size_t simde_mm256_roundscale_ps_i = 0 ; simde_mm256_roundscale_ps_i < (sizeof(simde_mm256_roundscale_ps_r_.m128) / sizeof(simde_mm256_roundscale_ps_r_.m128[0])) ; simde_mm256_roundscale_ps_i++) { \ + simde_mm256_roundscale_ps_r_.m128[simde_mm256_roundscale_ps_i] = simde_mm_roundscale_ps(simde_mm256_roundscale_ps_a_.m128[simde_mm256_roundscale_ps_i], imm8); \ + } \ + \ + simde__m256_from_private(simde_mm256_roundscale_ps_r_); \ + })) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m256 + simde_mm256_roundscale_ps_internal_ (simde__m256 result, simde__m256 a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + + simde__m256 r, clear_sign; + + clear_sign = simde_mm256_andnot_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(-0.0)), result); + r = simde_x_mm256_select_ps(result, a, simde_mm256_castsi256_ps(simde_mm256_cmpeq_epi32(simde_mm256_castps_si256(clear_sign), simde_mm256_castps_si256(simde_mm256_set1_ps(SIMDE_MATH_INFINITYF))))); + + return r; + } + #define simde_mm256_roundscale_ps(a, imm8) \ + simde_mm256_roundscale_ps_internal_( \ + simde_mm256_mul_ps( \ + simde_mm256_round_ps( \ + simde_mm256_mul_ps( \ + a, \ + simde_mm256_set1_ps(simde_math_exp2f(((imm8 >> 4) & 15)))), \ + ((imm8) & 15) \ + ), \ + simde_mm256_set1_ps(simde_math_exp2f(-((imm8 >> 4) & 15))) \ + ), \ + (a), \ + (imm8) \ + ) +#endif +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_roundscale_ps + #define _mm256_roundscale_ps(a, imm8) simde_mm256_roundscale_ps(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_roundscale_ps(src, k, a, imm8) _mm256_mask_roundscale_ps(src, k, a, imm8) +#else + #define simde_mm256_mask_roundscale_ps(src, k, a, imm8) simde_mm256_mask_mov_ps(src, k, simde_mm256_roundscale_ps(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_roundscale_ps + #define _mm256_mask_roundscale_ps(src, k, a, imm8) simde_mm256_mask_roundscale_ps(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_maskz_roundscale_ps(k, a, imm8) _mm256_maskz_roundscale_ps(k, a, imm8) +#else + #define simde_mm256_maskz_roundscale_ps(k, a, imm8) simde_mm256_maskz_mov_ps(k, simde_mm256_roundscale_ps(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_roundscale_ps + #define _mm256_maskz_roundscale_ps(k, a, imm8) simde_mm256_maskz_roundscale_ps(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_roundscale_ps(a, imm8) _mm512_roundscale_ps((a), (imm8)) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_roundscale_ps(a, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512_private \ + simde_mm512_roundscale_ps_r_ = simde__m512_to_private(simde_mm512_setzero_ps()), \ + simde_mm512_roundscale_ps_a_ = simde__m512_to_private(a); \ + \ + for (size_t simde_mm512_roundscale_ps_i = 0 ; simde_mm512_roundscale_ps_i < (sizeof(simde_mm512_roundscale_ps_r_.m256) / sizeof(simde_mm512_roundscale_ps_r_.m256[0])) ; simde_mm512_roundscale_ps_i++) { \ + simde_mm512_roundscale_ps_r_.m256[simde_mm512_roundscale_ps_i] = simde_mm256_roundscale_ps(simde_mm512_roundscale_ps_a_.m256[simde_mm512_roundscale_ps_i], imm8); \ + } \ + \ + simde__m512_from_private(simde_mm512_roundscale_ps_r_); \ + })) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512 + simde_mm512_roundscale_ps_internal_ (simde__m512 result, simde__m512 a, int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + + simde__m512 r, clear_sign; + + clear_sign = simde_mm512_andnot_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(-0.0)), result); + r = simde_mm512_mask_mov_ps(result, simde_mm512_cmpeq_epi32_mask(simde_mm512_castps_si512(clear_sign), simde_mm512_castps_si512(simde_mm512_set1_ps(SIMDE_MATH_INFINITYF))), a); + + return r; + } + #define simde_mm512_roundscale_ps(a, imm8) \ + simde_mm512_roundscale_ps_internal_( \ + simde_mm512_mul_ps( \ + simde_x_mm512_round_ps( \ + simde_mm512_mul_ps( \ + a, \ + simde_mm512_set1_ps(simde_math_exp2f(((imm8 >> 4) & 15)))), \ + ((imm8) & 15) \ + ), \ + simde_mm512_set1_ps(simde_math_exp2f(-((imm8 >> 4) & 15))) \ + ), \ + (a), \ + (imm8) \ + ) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_roundscale_ps + #define _mm512_roundscale_ps(a, imm8) simde_mm512_roundscale_ps(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_roundscale_ps(src, k, a, imm8) _mm512_mask_roundscale_ps(src, k, a, imm8) +#else + #define simde_mm512_mask_roundscale_ps(src, k, a, imm8) simde_mm512_mask_mov_ps(src, k, simde_mm512_roundscale_ps(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_roundscale_ps + #define _mm512_mask_roundscale_ps(src, k, a, imm8) simde_mm512_mask_roundscale_ps(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_maskz_roundscale_ps(k, a, imm8) _mm512_maskz_roundscale_ps(k, a, imm8) +#else + #define simde_mm512_maskz_roundscale_ps(k, a, imm8) simde_mm512_maskz_mov_ps(k, simde_mm512_roundscale_ps(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_roundscale_ps + #define _mm512_maskz_roundscale_ps(k, a, imm8) simde_mm512_maskz_roundscale_ps(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_roundscale_pd(a, imm8) _mm_roundscale_pd((a), (imm8)) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_roundscale_pd_internal_ (simde__m128d result, simde__m128d a, int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + + simde__m128d r, clear_sign; + + clear_sign = simde_mm_andnot_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(-0.0)), result); + r = simde_x_mm_select_pd(result, a, simde_mm_cmpeq_pd(clear_sign, simde_mm_set1_pd(SIMDE_MATH_INFINITY))); + + return r; + } + #define simde_mm_roundscale_pd(a, imm8) \ + simde_mm_roundscale_pd_internal_( \ + simde_mm_mul_pd( \ + simde_mm_round_pd( \ + simde_mm_mul_pd( \ + a, \ + simde_mm_set1_pd(simde_math_exp2(((imm8 >> 4) & 15)))), \ + ((imm8) & 15) \ + ), \ + simde_mm_set1_pd(simde_math_exp2(-((imm8 >> 4) & 15))) \ + ), \ + (a), \ + (imm8) \ + ) +#endif +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_roundscale_pd + #define _mm_roundscale_pd(a, imm8) simde_mm_roundscale_pd(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_mask_roundscale_pd(src, k, a, imm8) _mm_mask_roundscale_pd(src, k, a, imm8) +#else + #define simde_mm_mask_roundscale_pd(src, k, a, imm8) simde_mm_mask_mov_pd(src, k, simde_mm_roundscale_pd(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_roundscale_pd + #define _mm_mask_roundscale_pd(src, k, a, imm8) simde_mm_mask_roundscale_pd(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_maskz_roundscale_pd(k, a, imm8) _mm_maskz_roundscale_pd(k, a, imm8) +#else + #define simde_mm_maskz_roundscale_pd(k, a, imm8) simde_mm_maskz_mov_pd(k, simde_mm_roundscale_pd(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_roundscale_pd + #define _mm_maskz_roundscale_pd(k, a, imm8) simde_mm_maskz_roundscale_pd(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm256_roundscale_pd(a, imm8) _mm256_roundscale_pd((a), (imm8)) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm256_roundscale_pd(a, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m256d_private \ + simde_mm256_roundscale_pd_r_ = simde__m256d_to_private(simde_mm256_setzero_pd()), \ + simde_mm256_roundscale_pd_a_ = simde__m256d_to_private(a); \ + \ + for (size_t simde_mm256_roundscale_pd_i = 0 ; simde_mm256_roundscale_pd_i < (sizeof(simde_mm256_roundscale_pd_r_.m128d) / sizeof(simde_mm256_roundscale_pd_r_.m128d[0])) ; simde_mm256_roundscale_pd_i++) { \ + simde_mm256_roundscale_pd_r_.m128d[simde_mm256_roundscale_pd_i] = simde_mm_roundscale_pd(simde_mm256_roundscale_pd_a_.m128d[simde_mm256_roundscale_pd_i], imm8); \ + } \ + \ + simde__m256d_from_private(simde_mm256_roundscale_pd_r_); \ + })) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m256d + simde_mm256_roundscale_pd_internal_ (simde__m256d result, simde__m256d a, int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + + simde__m256d r, clear_sign; + + clear_sign = simde_mm256_andnot_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(-0.0)), result); + r = simde_x_mm256_select_pd(result, a, simde_mm256_castsi256_pd(simde_mm256_cmpeq_epi64(simde_mm256_castpd_si256(clear_sign), simde_mm256_castpd_si256(simde_mm256_set1_pd(SIMDE_MATH_INFINITY))))); + + return r; + } + #define simde_mm256_roundscale_pd(a, imm8) \ + simde_mm256_roundscale_pd_internal_( \ + simde_mm256_mul_pd( \ + simde_mm256_round_pd( \ + simde_mm256_mul_pd( \ + a, \ + simde_mm256_set1_pd(simde_math_exp2(((imm8 >> 4) & 15)))), \ + ((imm8) & 15) \ + ), \ + simde_mm256_set1_pd(simde_math_exp2(-((imm8 >> 4) & 15))) \ + ), \ + (a), \ + (imm8) \ + ) +#endif +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_roundscale_pd + #define _mm256_roundscale_pd(a, imm8) simde_mm256_roundscale_pd(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_roundscale_pd(src, k, a, imm8) _mm256_mask_roundscale_pd(src, k, a, imm8) +#else + #define simde_mm256_mask_roundscale_pd(src, k, a, imm8) simde_mm256_mask_mov_pd(src, k, simde_mm256_roundscale_pd(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_roundscale_pd + #define _mm256_mask_roundscale_pd(src, k, a, imm8) simde_mm256_mask_roundscale_pd(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_maskz_roundscale_pd(k, a, imm8) _mm256_maskz_roundscale_pd(k, a, imm8) +#else + #define simde_mm256_maskz_roundscale_pd(k, a, imm8) simde_mm256_maskz_mov_pd(k, simde_mm256_roundscale_pd(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_roundscale_pd + #define _mm256_maskz_roundscale_pd(k, a, imm8) simde_mm256_maskz_roundscale_pd(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_roundscale_pd(a, imm8) _mm512_roundscale_pd((a), (imm8)) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_roundscale_pd(a, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d_private \ + simde_mm512_roundscale_pd_r_ = simde__m512d_to_private(simde_mm512_setzero_pd()), \ + simde_mm512_roundscale_pd_a_ = simde__m512d_to_private(a); \ + \ + for (size_t simde_mm512_roundscale_pd_i = 0 ; simde_mm512_roundscale_pd_i < (sizeof(simde_mm512_roundscale_pd_r_.m256d) / sizeof(simde_mm512_roundscale_pd_r_.m256d[0])) ; simde_mm512_roundscale_pd_i++) { \ + simde_mm512_roundscale_pd_r_.m256d[simde_mm512_roundscale_pd_i] = simde_mm256_roundscale_pd(simde_mm512_roundscale_pd_a_.m256d[simde_mm512_roundscale_pd_i], imm8); \ + } \ + \ + simde__m512d_from_private(simde_mm512_roundscale_pd_r_); \ + })) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512d + simde_mm512_roundscale_pd_internal_ (simde__m512d result, simde__m512d a, int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + + simde__m512d r, clear_sign; + + clear_sign = simde_mm512_andnot_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(-0.0)), result); + r = simde_mm512_mask_mov_pd(result, simde_mm512_cmpeq_epi64_mask(simde_mm512_castpd_si512(clear_sign), simde_mm512_castpd_si512(simde_mm512_set1_pd(SIMDE_MATH_INFINITY))), a); + + return r; + } + #define simde_mm512_roundscale_pd(a, imm8) \ + simde_mm512_roundscale_pd_internal_( \ + simde_mm512_mul_pd( \ + simde_x_mm512_round_pd( \ + simde_mm512_mul_pd( \ + a, \ + simde_mm512_set1_pd(simde_math_exp2(((imm8 >> 4) & 15)))), \ + ((imm8) & 15) \ + ), \ + simde_mm512_set1_pd(simde_math_exp2(-((imm8 >> 4) & 15))) \ + ), \ + (a), \ + (imm8) \ + ) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_roundscale_pd + #define _mm512_roundscale_pd(a, imm8) simde_mm512_roundscale_pd(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_roundscale_pd(src, k, a, imm8) _mm512_mask_roundscale_pd(src, k, a, imm8) +#else + #define simde_mm512_mask_roundscale_pd(src, k, a, imm8) simde_mm512_mask_mov_pd(src, k, simde_mm512_roundscale_pd(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_roundscale_pd + #define _mm512_mask_roundscale_pd(src, k, a, imm8) simde_mm512_mask_roundscale_pd(src, k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_maskz_roundscale_pd(k, a, imm8) _mm512_maskz_roundscale_pd(k, a, imm8) +#else + #define simde_mm512_maskz_roundscale_pd(k, a, imm8) simde_mm512_maskz_mov_pd(k, simde_mm512_roundscale_pd(a, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_roundscale_pd + #define _mm512_maskz_roundscale_pd(k, a, imm8) simde_mm512_maskz_roundscale_pd(k, a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_roundscale_ss(a, b, imm8) _mm_roundscale_ss((a), (b), (imm8)) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_roundscale_ss_internal_ (simde__m128 result, simde__m128 b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + + simde__m128_private + r_ = simde__m128_to_private(result), + b_ = simde__m128_to_private(b); + + if(simde_math_isinff(r_.f32[0])) + r_.f32[0] = b_.f32[0]; + + return simde__m128_from_private(r_); + } + #define simde_mm_roundscale_ss(a, b, imm8) \ + simde_mm_roundscale_ss_internal_( \ + simde_mm_mul_ss( \ + simde_mm_round_ss( \ + a, \ + simde_mm_mul_ss( \ + b, \ + simde_mm_set1_ps(simde_math_exp2f(((imm8 >> 4) & 15)))), \ + ((imm8) & 15) \ + ), \ + simde_mm_set1_ps(simde_math_exp2f(-((imm8 >> 4) & 15))) \ + ), \ + (b), \ + (imm8) \ + ) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_roundscale_ss + #define _mm_roundscale_ss(a, b, imm8) simde_mm_roundscale_ss(a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_92035) + #define simde_mm_mask_roundscale_ss(src, k, a, b, imm8) _mm_mask_roundscale_ss((src), (k), (a), (b), (imm8)) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_mask_roundscale_ss_internal_ (simde__m128 a, simde__m128 b, simde__mmask8 k) { + simde__m128 r; + + if(k & 1) + r = a; + else + r = b; + + return r; + } + #define simde_mm_mask_roundscale_ss(src, k, a, b, imm8) \ + simde_mm_mask_roundscale_ss_internal_( \ + simde_mm_roundscale_ss( \ + a, \ + b, \ + imm8 \ + ), \ + simde_mm_move_ss( \ + (a), \ + (src) \ + ), \ + (k) \ + ) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_roundscale_ss + #define _mm_mask_roundscale_ss(src, k, a, b, imm8) simde_mm_mask_roundscale_ss(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_92035) + #define simde_mm_maskz_roundscale_ss(k, a, b, imm8) _mm_maskz_roundscale_ss((k), (a), (b), (imm8)) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_maskz_roundscale_ss_internal_ (simde__m128 a, simde__m128 b, simde__mmask8 k) { + simde__m128 r; + + if(k & 1) + r = a; + else + r = b; + + return r; + } + #define simde_mm_maskz_roundscale_ss(k, a, b, imm8) \ + simde_mm_maskz_roundscale_ss_internal_( \ + simde_mm_roundscale_ss( \ + a, \ + b, \ + imm8 \ + ), \ + simde_mm_move_ss( \ + (a), \ + simde_mm_setzero_ps() \ + ), \ + (k) \ + ) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_roundscale_ss + #define _mm_maskz_roundscale_ss(k, a, b, imm8) simde_mm_maskz_roundscale_ss(k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_roundscale_sd(a, b, imm8) _mm_roundscale_sd((a), (b), (imm8)) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_roundscale_sd_internal_ (simde__m128d result, simde__m128d b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + HEDLEY_STATIC_CAST(void, imm8); + + simde__m128d_private + r_ = simde__m128d_to_private(result), + b_ = simde__m128d_to_private(b); + + if(simde_math_isinf(r_.f64[0])) + r_.f64[0] = b_.f64[0]; + + return simde__m128d_from_private(r_); + } + #define simde_mm_roundscale_sd(a, b, imm8) \ + simde_mm_roundscale_sd_internal_( \ + simde_mm_mul_sd( \ + simde_mm_round_sd( \ + a, \ + simde_mm_mul_sd( \ + b, \ + simde_mm_set1_pd(simde_math_exp2(((imm8 >> 4) & 15)))), \ + ((imm8) & 15) \ + ), \ + simde_mm_set1_pd(simde_math_exp2(-((imm8 >> 4) & 15))) \ + ), \ + (b), \ + (imm8) \ + ) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_roundscale_sd + #define _mm_roundscale_sd(a, b, imm8) simde_mm_roundscale_sd(a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_92035) + #define simde_mm_mask_roundscale_sd(src, k, a, b, imm8) _mm_mask_roundscale_sd((src), (k), (a), (b), (imm8)) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_mask_roundscale_sd_internal_ (simde__m128d a, simde__m128d b, simde__mmask8 k) { + simde__m128d r; + + if(k & 1) + r = a; + else + r = b; + + return r; + } + #define simde_mm_mask_roundscale_sd(src, k, a, b, imm8) \ + simde_mm_mask_roundscale_sd_internal_( \ + simde_mm_roundscale_sd( \ + a, \ + b, \ + imm8 \ + ), \ + simde_mm_move_sd( \ + (a), \ + (src) \ + ), \ + (k) \ + ) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_roundscale_sd + #define _mm_mask_roundscale_sd(src, k, a, b, imm8) simde_mm_mask_roundscale_sd(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_92035) + #define simde_mm_maskz_roundscale_sd(k, a, b, imm8) _mm_maskz_roundscale_sd((k), (a), (b), (imm8)) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_maskz_roundscale_sd_internal_ (simde__m128d a, simde__m128d b, simde__mmask8 k) { + simde__m128d r; + + if(k & 1) + r = a; + else + r = b; + + return r; + } + #define simde_mm_maskz_roundscale_sd(k, a, b, imm8) \ + simde_mm_maskz_roundscale_sd_internal_( \ + simde_mm_roundscale_sd( \ + a, \ + b, \ + imm8 \ + ), \ + simde_mm_move_sd( \ + (a), \ + simde_mm_setzero_pd() \ + ), \ + (k) \ + ) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_roundscale_sd + #define _mm_maskz_roundscale_sd(k, a, b, imm8) simde_mm_maskz_roundscale_sd(k, a, b, imm8) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_ROUNDSCALE_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/roundscale_round.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/roundscale_round.h new file mode 100644 index 00000000..f941e48d --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/roundscale_round.h @@ -0,0 +1,690 @@ +#if !defined(SIMDE_X86_AVX512_ROUNDSCALE_ROUND_H) +#define SIMDE_X86_AVX512_ROUNDSCALE_ROUND_H + +#include "types.h" +#include "roundscale.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#if defined(HEDLEY_MSVC_VERSION) +#pragma warning( push ) +#pragma warning( disable : 4244 ) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_roundscale_round_ps(a, imm8, sae) _mm512_roundscale_round_ps(a, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_roundscale_round_ps(a, imm8, sae) simde_mm512_roundscale_ps(a, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_roundscale_round_ps(a,imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512 simde_mm512_roundscale_round_ps_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_roundscale_round_ps_envp; \ + int simde_mm512_roundscale_round_ps_x = feholdexcept(&simde_mm512_roundscale_round_ps_envp); \ + simde_mm512_roundscale_round_ps_r = simde_mm512_roundscale_ps(a, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_roundscale_round_ps_x == 0)) \ + fesetenv(&simde_mm512_roundscale_round_ps_envp); \ + } \ + else { \ + simde_mm512_roundscale_round_ps_r = simde_mm512_roundscale_ps(a, imm8); \ + } \ + \ + simde_mm512_roundscale_round_ps_r; \ + })) + #else + #define simde_mm512_roundscale_round_ps(a, imm8, sae) simde_mm512_roundscale_ps(a, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512 + simde_mm512_roundscale_round_ps (simde__m512 a, int imm8, int sae) + SIMDE_REQUIRE_RANGE(imm8, 0, 15) { + simde__m512 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_roundscale_ps(a, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_roundscale_ps(a, imm8); + #endif + } + else { + r = simde_mm512_roundscale_ps(a, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_roundscale_round_ps + #define _mm512_roundscale_round_ps(a, imm8, sae) simde_mm512_roundscale_round_ps(a, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_92035) + #define simde_mm512_mask_roundscale_round_ps(src, k, a, imm8, sae) _mm512_mask_roundscale_round_ps(src, k, a, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_mask_roundscale_round_ps(src, k, a, imm8, sae) simde_mm512_mask_roundscale_ps(src, k, a, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_mask_roundscale_round_ps(src, k, a, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512 simde_mm512_mask_roundscale_round_ps_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_mask_roundscale_round_ps_envp; \ + int simde_mm512_mask_roundscale_round_ps_x = feholdexcept(&simde_mm512_mask_roundscale_round_ps_envp); \ + simde_mm512_mask_roundscale_round_ps_r = simde_mm512_mask_roundscale_ps(src, k, a, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_mask_roundscale_round_ps_x == 0)) \ + fesetenv(&simde_mm512_mask_roundscale_round_ps_envp); \ + } \ + else { \ + simde_mm512_mask_roundscale_round_ps_r = simde_mm512_mask_roundscale_ps(src, k, a, imm8); \ + } \ + \ + simde_mm512_mask_roundscale_round_ps_r; \ + })) + #else + #define simde_mm512_mask_roundscale_round_ps(src, k, a, imm8, sae) simde_mm512_mask_roundscale_ps(src, k, a, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512 + simde_mm512_mask_roundscale_round_ps (simde__m512 src, simde__mmask8 k, simde__m512 a, int imm8, int sae) + SIMDE_REQUIRE_RANGE(imm8, 0, 15) { + simde__m512 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_mask_roundscale_ps(src, k, a, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_mask_roundscale_ps(src, k, a, imm8); + #endif + } + else { + r = simde_mm512_mask_roundscale_ps(src, k, a, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_roundscale_round_ps + #define _mm512_mask_roundscale_round_ps(src, k, a, imm8, sae) simde_mm512_mask_roundscale_round_ps(src, k, a, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_92035) + #define simde_mm512_maskz_roundscale_round_ps(k, a, imm8, sae) _mm512_maskz_roundscale_round_ps(k, a, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_maskz_roundscale_round_ps(k, a, imm8, sae) simde_mm512_maskz_roundscale_ps(k, a, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_maskz_roundscale_round_ps(k, a, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512 simde_mm512_maskz_roundscale_round_ps_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_maskz_roundscale_round_ps_envp; \ + int simde_mm512_maskz_roundscale_round_ps_x = feholdexcept(&simde_mm512_maskz_roundscale_round_ps_envp); \ + simde_mm512_maskz_roundscale_round_ps_r = simde_mm512_maskz_roundscale_ps(k, a, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_maskz_roundscale_round_ps_x == 0)) \ + fesetenv(&simde_mm512_maskz_roundscale_round_ps_envp); \ + } \ + else { \ + simde_mm512_maskz_roundscale_round_ps_r = simde_mm512_maskz_roundscale_ps(k, a, imm8); \ + } \ + \ + simde_mm512_maskz_roundscale_round_ps_r; \ + })) + #else + #define simde_mm512_maskz_roundscale_round_ps(src, k, a, imm8, sae) simde_mm512_maskz_roundscale_ps(k, a, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512 + simde_mm512_maskz_roundscale_round_ps (simde__mmask8 k, simde__m512 a, int imm8, int sae) + SIMDE_REQUIRE_RANGE(imm8, 0, 15) { + simde__m512 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_maskz_roundscale_ps(k, a, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_maskz_roundscale_ps(k, a, imm8); + #endif + } + else { + r = simde_mm512_maskz_roundscale_ps(k, a, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_roundscale_round_ps + #define _mm512_maskz_roundscale_round_ps(k, a, imm8, sae) simde_mm512_maskz_roundscale_round_ps(k, a, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_roundscale_round_pd(a, imm8, sae) _mm512_roundscale_round_pd(a, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_roundscale_round_pd(a, imm8, sae) simde_mm512_roundscale_pd(a, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_roundscale_round_pd(a, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d simde_mm512_roundscale_round_pd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_roundscale_round_pd_envp; \ + int simde_mm512_roundscale_round_pd_x = feholdexcept(&simde_mm512_roundscale_round_pd_envp); \ + simde_mm512_roundscale_round_pd_r = simde_mm512_roundscale_pd(a, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_roundscale_round_pd_x == 0)) \ + fesetenv(&simde_mm512_roundscale_round_pd_envp); \ + } \ + else { \ + simde_mm512_roundscale_round_pd_r = simde_mm512_roundscale_pd(a, imm8); \ + } \ + \ + simde_mm512_roundscale_round_pd_r; \ + })) + #else + #define simde_mm512_roundscale_round_pd(a, imm8, sae) simde_mm512_roundscale_pd(a, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512d + simde_mm512_roundscale_round_pd (simde__m512d a, int imm8, int sae) + SIMDE_REQUIRE_RANGE(imm8, 0, 15) { + simde__m512d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_roundscale_pd(a, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_roundscale_pd(a, imm8); + #endif + } + else { + r = simde_mm512_roundscale_pd(a, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_roundscale_round_pd + #define _mm512_roundscale_round_pd(a, imm8, sae) simde_mm512_roundscale_round_pd(a, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_92035) + #define simde_mm512_mask_roundscale_round_pd(src, k, a, imm8, sae) _mm512_mask_roundscale_round_pd(src, k, a, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_mask_roundscale_round_pd(src, k, a, imm8, sae) simde_mm512_mask_roundscale_pd(src, k, a, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_mask_roundscale_round_pd(src, k, a, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d simde_mm512_mask_roundscale_round_pd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_mask_roundscale_round_pd_envp; \ + int simde_mm512_mask_roundscale_round_pd_x = feholdexcept(&simde_mm512_mask_roundscale_round_pd_envp); \ + simde_mm512_mask_roundscale_round_pd_r = simde_mm512_mask_roundscale_pd(src, k, a, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_mask_roundscale_round_pd_x == 0)) \ + fesetenv(&simde_mm512_mask_roundscale_round_pd_envp); \ + } \ + else { \ + simde_mm512_mask_roundscale_round_pd_r = simde_mm512_mask_roundscale_pd(src, k, a, imm8); \ + } \ + \ + simde_mm512_mask_roundscale_round_pd_r; \ + })) + #else + #define simde_mm512_mask_roundscale_round_pd(src, k, a, imm8, sae) simde_mm512_mask_roundscale_pd(src, k, a, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512d + simde_mm512_mask_roundscale_round_pd (simde__m512d src, simde__mmask8 k, simde__m512d a, int imm8, int sae) + SIMDE_REQUIRE_RANGE(imm8, 0, 15) { + simde__m512d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_mask_roundscale_pd(src, k, a, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_mask_roundscale_pd(src, k, a, imm8); + #endif + } + else { + r = simde_mm512_mask_roundscale_pd(src, k, a, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_roundscale_round_pd + #define _mm512_mask_roundscale_round_pd(src, k, a, imm8, sae) simde_mm512_mask_roundscale_round_pd(src, k, a, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_92035) + #define simde_mm512_maskz_roundscale_round_pd(k, a, imm8, sae) _mm512_maskz_roundscale_round_pd(k, a, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm512_maskz_roundscale_round_pd(k, a, imm8, sae) simde_mm512_maskz_roundscale_pd(k, a, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm512_maskz_roundscale_round_pd(k, a, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512d simde_mm512_maskz_roundscale_round_pd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm512_maskz_roundscale_round_pd_envp; \ + int simde_mm512_maskz_roundscale_round_pd_x = feholdexcept(&simde_mm512_maskz_roundscale_round_pd_envp); \ + simde_mm512_maskz_roundscale_round_pd_r = simde_mm512_maskz_roundscale_pd(k, a, imm8); \ + if (HEDLEY_LIKELY(simde_mm512_maskz_roundscale_round_pd_x == 0)) \ + fesetenv(&simde_mm512_maskz_roundscale_round_pd_envp); \ + } \ + else { \ + simde_mm512_maskz_roundscale_round_pd_r = simde_mm512_maskz_roundscale_pd(k, a, imm8); \ + } \ + \ + simde_mm512_maskz_roundscale_round_pd_r; \ + })) + #else + #define simde_mm512_maskz_roundscale_round_pd(src, k, a, imm8, sae) simde_mm512_maskz_roundscale_pd(k, a, imm8) + #endif +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512d + simde_mm512_maskz_roundscale_round_pd (simde__mmask8 k, simde__m512d a, int imm8, int sae) + SIMDE_REQUIRE_RANGE(imm8, 0, 15) { + simde__m512d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm512_maskz_roundscale_pd(k, a, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm512_maskz_roundscale_pd(k, a, imm8); + #endif + } + else { + r = simde_mm512_maskz_roundscale_pd(k, a, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_roundscale_round_pd + #define _mm512_maskz_roundscale_round_pd(k, a, imm8, sae) simde_mm512_maskz_roundscale_round_pd(k, a, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_roundscale_round_ss(a, b, imm8, sae) _mm_roundscale_round_ss(a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_roundscale_round_ss(a, b, imm8, sae) simde_mm_roundscale_ss(a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_roundscale_round_ss(a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128 simde_mm_roundscale_round_ss_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_roundscale_round_ss_envp; \ + int simde_mm_roundscale_round_ss_x = feholdexcept(&simde_mm_roundscale_round_ss_envp); \ + simde_mm_roundscale_round_ss_r = simde_mm_roundscale_ss(a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm_roundscale_round_ss_x == 0)) \ + fesetenv(&simde_mm_roundscale_round_ss_envp); \ + } \ + else { \ + simde_mm_roundscale_round_ss_r = simde_mm_roundscale_ss(a, b, imm8); \ + } \ + \ + simde_mm_roundscale_round_ss_r; \ + })) + #else + #define simde_mm_roundscale_round_ss(a, b, imm8, sae) simde_mm_roundscale_ss(a, b, imm8) + #endif +#elif !(defined(HEDLEY_MSVC_VERSION) && defined(SIMDE_X86_AVX_NATIVE)) + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_roundscale_round_ss (simde__m128 a, simde__m128 b, const int imm8, const int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_roundscale_ss(a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_roundscale_ss(a, b, imm8); + #endif + } + else { + r = simde_mm_roundscale_ss(a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_roundscale_round_ss + #define _mm_roundscale_round_ss(a, b, imm8, sae) simde_mm_roundscale_round_ss(a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_92035) + #define simde_mm_mask_roundscale_round_ss(src, k, a, b, imm8, sae) _mm_mask_roundscale_round_ss(src, k, a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_mask_roundscale_round_ss(src, k, a, b, imm8, sae) simde_mm_mask_roundscale_ss(src, k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_mask_roundscale_round_ss(src, k, a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128 simde_mm_mask_roundscale_round_ss_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_mask_roundscale_round_ss_envp; \ + int simde_mm_mask_roundscale_round_ss_x = feholdexcept(&simde_mm_mask_roundscale_round_ss_envp); \ + simde_mm_mask_roundscale_round_ss_r = simde_mm_mask_roundscale_ss(src, k, a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm_mask_roundscale_round_ss_x == 0)) \ + fesetenv(&simde_mm_mask_roundscale_round_ss_envp); \ + } \ + else { \ + simde_mm_mask_roundscale_round_ss_r = simde_mm_mask_roundscale_ss(src, k, a, b, imm8); \ + } \ + \ + simde_mm_mask_roundscale_round_ss_r; \ + })) + #else + #define simde_mm_mask_roundscale_round_ss(src, k, a, b, imm8, sae) simde_mm_mask_roundscale_ss(src, k, a, b, imm8) + #endif +#elif !(defined(HEDLEY_MSVC_VERSION) && defined(SIMDE_X86_AVX_NATIVE)) + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_mask_roundscale_round_ss (simde__m128 src, simde__mmask8 k, simde__m128 a, simde__m128 b, const int imm8, const int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_mask_roundscale_ss(src, k, a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_mask_roundscale_ss(src, k, a, b, imm8); + #endif + } + else { + r = simde_mm_mask_roundscale_ss(src, k, a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_roundscale_round_ss + #define _mm_mask_roundscale_round_ss(src, k, a, b, imm8, sae) simde_mm_mask_roundscale_round_ss(src, k, a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_92035) + #define simde_mm_maskz_roundscale_round_ss(k, a, b, imm8, sae) _mm_maskz_roundscale_round_ss(k, a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_maskz_roundscale_round_ss(k, a, b, imm8, sae) simde_mm_maskz_roundscale_ss(k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_maskz_roundscale_round_ss(k, a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128 simde_mm_maskz_roundscale_round_ss_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_maskz_roundscale_round_ss_envp; \ + int simde_mm_maskz_roundscale_round_ss_x = feholdexcept(&simde_mm_maskz_roundscale_round_ss_envp); \ + simde_mm_maskz_roundscale_round_ss_r = simde_mm_maskz_roundscale_ss(k, a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm_maskz_roundscale_round_ss_x == 0)) \ + fesetenv(&simde_mm_maskz_roundscale_round_ss_envp); \ + } \ + else { \ + simde_mm_maskz_roundscale_round_ss_r = simde_mm_maskz_roundscale_ss(k, a, b, imm8); \ + } \ + \ + simde_mm_maskz_roundscale_round_ss_r; \ + })) + #else + #define simde_mm_maskz_roundscale_round_ss(k, a, b, imm8, sae) simde_mm_maskz_roundscale_ss(k, a, b, imm8) + #endif +#elif !(defined(HEDLEY_MSVC_VERSION) && defined(SIMDE_X86_AVX_NATIVE)) + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde_mm_maskz_roundscale_round_ss (simde__mmask8 k, simde__m128 a, simde__m128 b, const int imm8, const int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128 r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_maskz_roundscale_ss(k, a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_maskz_roundscale_ss(k, a, b, imm8); + #endif + } + else { + r = simde_mm_maskz_roundscale_ss(k, a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_roundscale_round_ss + #define _mm_maskz_roundscale_round_ss(k, a, b, imm8, sae) simde_mm_maskz_roundscale_round_ss(k, a, b, imm8, sae) +#endif + +#if defined(HEDLEY_MSVC_VERSION) +#pragma warning( pop ) +#endif + + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_roundscale_round_sd(a, b, imm8, sae) _mm_roundscale_round_sd(a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_roundscale_round_sd(a, b, imm8, sae) simde_mm_roundscale_sd(a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_roundscale_round_sd(a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128d simde_mm_roundscale_round_sd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_roundscale_round_sd_envp; \ + int simde_mm_roundscale_round_sd_x = feholdexcept(&simde_mm_roundscale_round_sd_envp); \ + simde_mm_roundscale_round_sd_r = simde_mm_roundscale_sd(a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm_roundscale_round_sd_x == 0)) \ + fesetenv(&simde_mm_roundscale_round_sd_envp); \ + } \ + else { \ + simde_mm_roundscale_round_sd_r = simde_mm_roundscale_sd(a, b, imm8); \ + } \ + \ + simde_mm_roundscale_round_sd_r; \ + })) + #else + #define simde_mm_roundscale_round_sd(a, b, imm8, sae) simde_mm_roundscale_sd(a, b, imm8) + #endif +#elif !(defined(HEDLEY_MSVC_VERSION) && defined(SIMDE_X86_AVX_NATIVE)) + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_roundscale_round_sd (simde__m128d a, simde__m128d b, const int imm8, const int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_roundscale_sd(a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_roundscale_sd(a, b, imm8); + #endif + } + else { + r = simde_mm_roundscale_sd(a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_roundscale_round_sd + #define _mm_roundscale_round_sd(a, b, imm8, sae) simde_mm_roundscale_round_sd(a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_92035) + #define simde_mm_mask_roundscale_round_sd(src, k, a, b, imm8, sae) _mm_mask_roundscale_round_sd(src, k, a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_mask_roundscale_round_sd(src, k, a, b, imm8, sae) simde_mm_mask_roundscale_sd(src, k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_mask_roundscale_round_sd(src, k, a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128d simde_mm_mask_roundscale_round_sd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_mask_roundscale_round_sd_envp; \ + int simde_mm_mask_roundscale_round_sd_x = feholdexcept(&simde_mm_mask_roundscale_round_sd_envp); \ + simde_mm_mask_roundscale_round_sd_r = simde_mm_mask_roundscale_sd(src, k, a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm_mask_roundscale_round_sd_x == 0)) \ + fesetenv(&simde_mm_mask_roundscale_round_sd_envp); \ + } \ + else { \ + simde_mm_mask_roundscale_round_sd_r = simde_mm_mask_roundscale_sd(src, k, a, b, imm8); \ + } \ + \ + simde_mm_mask_roundscale_round_sd_r; \ + })) + #else + #define simde_mm_mask_roundscale_round_sd(src, k, a, b, imm8, sae) simde_mm_mask_roundscale_sd(src, k, a, b, imm8) + #endif +#elif !(defined(HEDLEY_MSVC_VERSION) && defined(SIMDE_X86_AVX_NATIVE)) + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_mask_roundscale_round_sd (simde__m128d src, simde__mmask8 k, simde__m128d a, simde__m128d b, const int imm8, const int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_mask_roundscale_sd(src, k, a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_mask_roundscale_sd(src, k, a, b, imm8); + #endif + } + else { + r = simde_mm_mask_roundscale_sd(src, k, a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_roundscale_round_sd + #define _mm_mask_roundscale_round_sd(src, k, a, b, imm8, sae) simde_mm_mask_roundscale_round_sd(src, k, a, b, imm8, sae) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_92035) + #define simde_mm_maskz_roundscale_round_sd(k, a, b, imm8, sae) _mm_maskz_roundscale_round_sd(k, a, b, imm8, sae) +#elif defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_maskz_roundscale_round_sd(k, a, b, imm8, sae) simde_mm_maskz_roundscale_sd(k, a, b, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) + #if defined(SIMDE_HAVE_FENV_H) + #define simde_mm_maskz_roundscale_round_sd(k, a, b, imm8, sae) SIMDE_STATEMENT_EXPR_(({ \ + simde__m128d simde_mm_maskz_roundscale_round_sd_r; \ + \ + if (sae & SIMDE_MM_FROUND_NO_EXC) { \ + fenv_t simde_mm_maskz_roundscale_round_sd_envp; \ + int simde_mm_maskz_roundscale_round_sd_x = feholdexcept(&simde_mm_maskz_roundscale_round_sd_envp); \ + simde_mm_maskz_roundscale_round_sd_r = simde_mm_maskz_roundscale_sd(k, a, b, imm8); \ + if (HEDLEY_LIKELY(simde_mm_maskz_roundscale_round_sd_x == 0)) \ + fesetenv(&simde_mm_maskz_roundscale_round_sd_envp); \ + } \ + else { \ + simde_mm_maskz_roundscale_round_sd_r = simde_mm_maskz_roundscale_sd(k, a, b, imm8); \ + } \ + \ + simde_mm_maskz_roundscale_round_sd_r; \ + })) + #else + #define simde_mm_maskz_roundscale_round_sd(k, a, b, imm8, sae) simde_mm_maskz_roundscale_sd(k, a, b, imm8) + #endif +#elif !(defined(HEDLEY_MSVC_VERSION) && defined(SIMDE_X86_AVX_NATIVE)) + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde_mm_maskz_roundscale_round_sd (simde__mmask8 k, simde__m128d a, simde__m128d b, const int imm8, const int sae) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) + SIMDE_REQUIRE_CONSTANT(sae) { + simde__m128d r; + + if (sae & SIMDE_MM_FROUND_NO_EXC) { + #if defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = simde_mm_maskz_roundscale_sd(k, a, b, imm8); + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = simde_mm_maskz_roundscale_sd(k, a, b, imm8); + #endif + } + else { + r = simde_mm_maskz_roundscale_sd(k, a, b, imm8); + } + + return r; + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_roundscale_round_sd + #define _mm_maskz_roundscale_round_sd(k, a, b, imm8, sae) simde_mm_maskz_roundscale_round_sd(k, a, b, imm8, sae) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_ROUNDSCALE_ROUND_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sad.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sad.h new file mode 100644 index 00000000..60623e6b --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sad.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_SAD_H) +#define SIMDE_X86_AVX512_SAD_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_sad_epu8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_sad_epu8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_sad_epu8(a_.m256i[i], b_.m256i[i]); + } + #else + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + uint16_t tmp = 0; + SIMDE_VECTORIZE_REDUCTION(+:tmp) + for (size_t j = 0 ; j < ((sizeof(r_.u8) / sizeof(r_.u8[0])) / 8) ; j++) { + const size_t e = j + (i * 8); + tmp += (a_.u8[e] > b_.u8[e]) ? (a_.u8[e] - b_.u8[e]) : (b_.u8[e] - a_.u8[e]); + } + r_.i64[i] = tmp; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_sad_epu8 + #define _mm512_sad_epu8(a, b) simde_mm512_sad_epu8(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SAD_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/scalef.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/scalef.h new file mode 100644 index 00000000..11673317 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/scalef.h @@ -0,0 +1,389 @@ +#if !defined(SIMDE_X86_AVX512_SCALEF_H) +#define SIMDE_X86_AVX512_SCALEF_H + +#include "types.h" +#include "flushsubnormal.h" +#include "../svml.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_scalef_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_scalef_ps(a, b); + #else + return simde_mm_mul_ps(simde_x_mm_flushsubnormal_ps(a), simde_mm_exp2_ps(simde_mm_floor_ps(simde_x_mm_flushsubnormal_ps(b)))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_scalef_ps + #define _mm_scalef_ps(a, b) simde_mm_scalef_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mask_scalef_ps (simde__m128 src, simde__mmask8 k, simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_scalef_ps(src, k, a, b); + #else + return simde_mm_mask_mov_ps(src, k, simde_mm_scalef_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_scalef_ps + #define _mm_mask_scalef_ps(src, k, a, b) simde_mm_mask_scalef_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_maskz_scalef_ps (simde__mmask8 k, simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_scalef_ps(k, a, b); + #else + return simde_mm_maskz_mov_ps(k, simde_mm_scalef_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_scalef_ps + #define _mm_maskz_scalef_ps(k, a, b) simde_mm_maskz_scalef_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_scalef_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_scalef_ps(a, b); + #else + return simde_mm256_mul_ps(simde_x_mm256_flushsubnormal_ps(a), simde_mm256_exp2_ps(simde_mm256_floor_ps(simde_x_mm256_flushsubnormal_ps(b)))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_scalef_ps + #define _mm256_scalef_ps(a, b) simde_mm256_scalef_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask_scalef_ps (simde__m256 src, simde__mmask8 k, simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_scalef_ps(src, k, a, b); + #else + return simde_mm256_mask_mov_ps(src, k, simde_mm256_scalef_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_scalef_ps + #define _mm256_mask_scalef_ps(src, k, a, b) simde_mm256_mask_scalef_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_maskz_scalef_ps (simde__mmask8 k, simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_scalef_ps(k, a, b); + #else + return simde_mm256_maskz_mov_ps(k, simde_mm256_scalef_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_scalef_ps + #define _mm256_maskz_scalef_ps(k, a, b) simde_mm256_maskz_scalef_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_scalef_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_scalef_ps(a, b); + #else + return simde_mm512_mul_ps(simde_x_mm512_flushsubnormal_ps(a), simde_mm512_exp2_ps(simde_mm512_floor_ps(simde_x_mm512_flushsubnormal_ps(b)))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_scalef_ps + #define _mm512_scalef_ps(a, b) simde_mm512_scalef_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_scalef_ps (simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_scalef_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_scalef_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_scalef_ps + #define _mm512_mask_scalef_ps(src, k, a, b) simde_mm512_mask_scalef_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_scalef_ps (simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_scalef_ps(k, a, b); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_scalef_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_scalef_ps + #define _mm512_maskz_scalef_ps(k, a, b) simde_mm512_maskz_scalef_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_scalef_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_scalef_pd(a, b); + #else + return simde_mm_mul_pd(simde_x_mm_flushsubnormal_pd(a), simde_mm_exp2_pd(simde_mm_floor_pd(simde_x_mm_flushsubnormal_pd(b)))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_scalef_pd + #define _mm_scalef_pd(a, b) simde_mm_scalef_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mask_scalef_pd (simde__m128d src, simde__mmask8 k, simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_scalef_pd(src, k, a, b); + #else + return simde_mm_mask_mov_pd(src, k, simde_mm_scalef_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_scalef_pd + #define _mm_mask_scalef_pd(src, k, a, b) simde_mm_mask_scalef_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_maskz_scalef_pd (simde__mmask8 k, simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_scalef_pd(k, a, b); + #else + return simde_mm_maskz_mov_pd(k, simde_mm_scalef_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_scalef_pd + #define _mm_maskz_scalef_pd(k, a, b) simde_mm_maskz_scalef_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_scalef_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_scalef_pd(a, b); + #else + return simde_mm256_mul_pd(simde_x_mm256_flushsubnormal_pd(a), simde_mm256_exp2_pd(simde_mm256_floor_pd(simde_x_mm256_flushsubnormal_pd(b)))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_scalef_pd + #define _mm256_scalef_pd(a, b) simde_mm256_scalef_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mask_scalef_pd (simde__m256d src, simde__mmask8 k, simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_scalef_pd(src, k, a, b); + #else + return simde_mm256_mask_mov_pd(src, k, simde_mm256_scalef_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_scalef_pd + #define _mm256_mask_scalef_pd(src, k, a, b) simde_mm256_mask_scalef_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_maskz_scalef_pd (simde__mmask8 k, simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_scalef_pd(k, a, b); + #else + return simde_mm256_maskz_mov_pd(k, simde_mm256_scalef_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_scalef_pd + #define _mm256_maskz_scalef_pd(k, a, b) simde_mm256_maskz_scalef_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_scalef_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_scalef_pd(a, b); + #else + return simde_mm512_mul_pd(simde_x_mm512_flushsubnormal_pd(a), simde_mm512_exp2_pd(simde_mm512_floor_pd(simde_x_mm512_flushsubnormal_pd(b)))); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_scalef_pd + #define _mm512_scalef_pd(a, b) simde_mm512_scalef_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_scalef_pd (simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_scalef_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_scalef_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_scalef_pd + #define _mm512_mask_scalef_pd(src, k, a, b) simde_mm512_mask_scalef_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_scalef_pd (simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_scalef_pd(k, a, b); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_scalef_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_scalef_pd + #define _mm512_maskz_scalef_pd(k, a, b) simde_mm512_maskz_scalef_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_scalef_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm_scalef_ss(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + a_.f32[0] = (simde_math_issubnormalf(a_.f32[0]) ? 0 : a_.f32[0]) * simde_math_exp2f(simde_math_floorf((simde_math_issubnormalf(b_.f32[0]) ? 0 : b_.f32[0]))); + + return simde__m128_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_scalef_ss + #define _mm_scalef_ss(a, b) simde_mm_scalef_ss(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mask_scalef_ss (simde__m128 src, simde__mmask8 k, simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(HEDLEY_GCC_VERSION) + return _mm_mask_scalef_round_ss(src, k, a, b, _MM_FROUND_CUR_DIRECTION); + #else + simde__m128_private + src_ = simde__m128_to_private(src), + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + a_.f32[0] = ((k & 1) ? ((simde_math_issubnormalf(a_.f32[0]) ? 0 : a_.f32[0]) * simde_math_exp2f(simde_math_floorf((simde_math_issubnormalf(b_.f32[0]) ? 0 : b_.f32[0])))) : src_.f32[0]); + + return simde__m128_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_scalef_ss + #define _mm_mask_scalef_ss(src, k, a, b) simde_mm_mask_scalef_ss(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_maskz_scalef_ss (simde__mmask8 k, simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_95483) && !defined(SIMDE_BUG_GCC_105339) + return _mm_maskz_scalef_ss(k, a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + a_.f32[0] = ((k & 1) ? ((simde_math_issubnormalf(a_.f32[0]) ? 0 : a_.f32[0]) * simde_math_exp2f(simde_math_floorf((simde_math_issubnormalf(b_.f32[0]) ? 0 : b_.f32[0])))) : SIMDE_FLOAT32_C(0.0)); + + return simde__m128_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_scalef_ss + #define _mm_maskz_scalef_ss(k, a, b) simde_mm_maskz_scalef_ss(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_scalef_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm_scalef_sd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + a_.f64[0] = (simde_math_issubnormal(a_.f64[0]) ? 0 : a_.f64[0]) * simde_math_exp2(simde_math_floor((simde_math_issubnormal(b_.f64[0]) ? 0 : b_.f64[0]))); + + return simde__m128d_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_scalef_sd + #define _mm_scalef_sd(a, b) simde_mm_scalef_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mask_scalef_sd (simde__m128d src, simde__mmask8 k, simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_95483) && !defined(SIMDE_BUG_GCC_105339) + return _mm_mask_scalef_sd(src, k, a, b); + #else + simde__m128d_private + src_ = simde__m128d_to_private(src), + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + a_.f64[0] = ((k & 1) ? ((simde_math_issubnormal(a_.f64[0]) ? 0 : a_.f64[0]) * simde_math_exp2(simde_math_floor((simde_math_issubnormal(b_.f64[0]) ? 0 : b_.f64[0])))) : src_.f64[0]); + + return simde__m128d_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_scalef_sd + #define _mm_mask_scalef_sd(src, k, a, b) simde_mm_mask_scalef_sd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_maskz_scalef_sd (simde__mmask8 k, simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && !defined(SIMDE_BUG_GCC_95483) && !defined(SIMDE_BUG_GCC_105339) + return _mm_maskz_scalef_sd(k, a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + a_.f64[0] = ((k & 1) ? ((simde_math_issubnormal(a_.f64[0]) ? 0 : a_.f64[0]) * simde_math_exp2(simde_math_floor(simde_math_issubnormal(b_.f64[0]) ? 0 : b_.f64[0]))) : SIMDE_FLOAT64_C(0.0)); + + return simde__m128d_from_private(a_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_scalef_sd + #define _mm_maskz_scalef_sd(k, a, b) simde_mm_maskz_scalef_sd(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SCALEF_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/set.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/set.h new file mode 100644 index 00000000..d87a72ce --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/set.h @@ -0,0 +1,572 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_SET_H) +#define SIMDE_X86_AVX512_SET_H + +#include "types.h" +#include "load.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_set_epi16 (int16_t e31, int16_t e30, int16_t e29, int16_t e28, int16_t e27, int16_t e26, int16_t e25, int16_t e24, + int16_t e23, int16_t e22, int16_t e21, int16_t e20, int16_t e19, int16_t e18, int16_t e17, int16_t e16, + int16_t e15, int16_t e14, int16_t e13, int16_t e12, int16_t e11, int16_t e10, int16_t e9, int16_t e8, + int16_t e7, int16_t e6, int16_t e5, int16_t e4, int16_t e3, int16_t e2, int16_t e1, int16_t e0) { + simde__m512i_private r_; + + r_.i16[ 0] = e0; + r_.i16[ 1] = e1; + r_.i16[ 2] = e2; + r_.i16[ 3] = e3; + r_.i16[ 4] = e4; + r_.i16[ 5] = e5; + r_.i16[ 6] = e6; + r_.i16[ 7] = e7; + r_.i16[ 8] = e8; + r_.i16[ 9] = e9; + r_.i16[10] = e10; + r_.i16[11] = e11; + r_.i16[12] = e12; + r_.i16[13] = e13; + r_.i16[14] = e14; + r_.i16[15] = e15; + r_.i16[16] = e16; + r_.i16[17] = e17; + r_.i16[18] = e18; + r_.i16[19] = e19; + r_.i16[20] = e20; + r_.i16[21] = e21; + r_.i16[22] = e22; + r_.i16[23] = e23; + r_.i16[24] = e24; + r_.i16[25] = e25; + r_.i16[26] = e26; + r_.i16[27] = e27; + r_.i16[28] = e28; + r_.i16[29] = e29; + r_.i16[30] = e30; + r_.i16[31] = e31; + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set_epi16 + #define _mm512_set_epi16(e31, e30, e29, e28, e27, e26, e25, e24, e23, e22, e21, e20, e19, e18, e17, e16, e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) simde_mm512_set_epi16(e31, e30, e29, e28, e27, e26, e25, e24, e23, e22, e21, e20, e19, e18, e17, e16, e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_set_epi32 (int32_t e15, int32_t e14, int32_t e13, int32_t e12, int32_t e11, int32_t e10, int32_t e9, int32_t e8, + int32_t e7, int32_t e6, int32_t e5, int32_t e4, int32_t e3, int32_t e2, int32_t e1, int32_t e0) { + simde__m512i_private r_; + + r_.i32[ 0] = e0; + r_.i32[ 1] = e1; + r_.i32[ 2] = e2; + r_.i32[ 3] = e3; + r_.i32[ 4] = e4; + r_.i32[ 5] = e5; + r_.i32[ 6] = e6; + r_.i32[ 7] = e7; + r_.i32[ 8] = e8; + r_.i32[ 9] = e9; + r_.i32[10] = e10; + r_.i32[11] = e11; + r_.i32[12] = e12; + r_.i32[13] = e13; + r_.i32[14] = e14; + r_.i32[15] = e15; + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set_epi32 + #define _mm512_set_epi32(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) simde_mm512_set_epi32(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_set_epi64 (int64_t e7, int64_t e6, int64_t e5, int64_t e4, int64_t e3, int64_t e2, int64_t e1, int64_t e0) { + simde__m512i_private r_; + + r_.i64[0] = e0; + r_.i64[1] = e1; + r_.i64[2] = e2; + r_.i64[3] = e3; + r_.i64[4] = e4; + r_.i64[5] = e5; + r_.i64[6] = e6; + r_.i64[7] = e7; + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set_epi64 + #define _mm512_set_epi64(e7, e6, e5, e4, e3, e2, e1, e0) simde_mm512_set_epi64(e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_set_epu8 (uint8_t e63, uint8_t e62, uint8_t e61, uint8_t e60, uint8_t e59, uint8_t e58, uint8_t e57, uint8_t e56, + uint8_t e55, uint8_t e54, uint8_t e53, uint8_t e52, uint8_t e51, uint8_t e50, uint8_t e49, uint8_t e48, + uint8_t e47, uint8_t e46, uint8_t e45, uint8_t e44, uint8_t e43, uint8_t e42, uint8_t e41, uint8_t e40, + uint8_t e39, uint8_t e38, uint8_t e37, uint8_t e36, uint8_t e35, uint8_t e34, uint8_t e33, uint8_t e32, + uint8_t e31, uint8_t e30, uint8_t e29, uint8_t e28, uint8_t e27, uint8_t e26, uint8_t e25, uint8_t e24, + uint8_t e23, uint8_t e22, uint8_t e21, uint8_t e20, uint8_t e19, uint8_t e18, uint8_t e17, uint8_t e16, + uint8_t e15, uint8_t e14, uint8_t e13, uint8_t e12, uint8_t e11, uint8_t e10, uint8_t e9, uint8_t e8, + uint8_t e7, uint8_t e6, uint8_t e5, uint8_t e4, uint8_t e3, uint8_t e2, uint8_t e1, uint8_t e0) { + simde__m512i_private r_; + + r_.u8[ 0] = e0; + r_.u8[ 1] = e1; + r_.u8[ 2] = e2; + r_.u8[ 3] = e3; + r_.u8[ 4] = e4; + r_.u8[ 5] = e5; + r_.u8[ 6] = e6; + r_.u8[ 7] = e7; + r_.u8[ 8] = e8; + r_.u8[ 9] = e9; + r_.u8[10] = e10; + r_.u8[11] = e11; + r_.u8[12] = e12; + r_.u8[13] = e13; + r_.u8[14] = e14; + r_.u8[15] = e15; + r_.u8[16] = e16; + r_.u8[17] = e17; + r_.u8[18] = e18; + r_.u8[19] = e19; + r_.u8[20] = e20; + r_.u8[21] = e21; + r_.u8[22] = e22; + r_.u8[23] = e23; + r_.u8[24] = e24; + r_.u8[25] = e25; + r_.u8[26] = e26; + r_.u8[27] = e27; + r_.u8[28] = e28; + r_.u8[29] = e29; + r_.u8[30] = e30; + r_.u8[31] = e31; + r_.u8[32] = e32; + r_.u8[33] = e33; + r_.u8[34] = e34; + r_.u8[35] = e35; + r_.u8[36] = e36; + r_.u8[37] = e37; + r_.u8[38] = e38; + r_.u8[39] = e39; + r_.u8[40] = e40; + r_.u8[41] = e41; + r_.u8[42] = e42; + r_.u8[43] = e43; + r_.u8[44] = e44; + r_.u8[45] = e45; + r_.u8[46] = e46; + r_.u8[47] = e47; + r_.u8[48] = e48; + r_.u8[49] = e49; + r_.u8[50] = e50; + r_.u8[51] = e51; + r_.u8[52] = e52; + r_.u8[53] = e53; + r_.u8[54] = e54; + r_.u8[55] = e55; + r_.u8[56] = e56; + r_.u8[57] = e57; + r_.u8[58] = e58; + r_.u8[59] = e59; + r_.u8[60] = e60; + r_.u8[61] = e61; + r_.u8[62] = e62; + r_.u8[63] = e63; + + return simde__m512i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_set_epu16 (uint16_t e31, uint16_t e30, uint16_t e29, uint16_t e28, uint16_t e27, uint16_t e26, uint16_t e25, uint16_t e24, + uint16_t e23, uint16_t e22, uint16_t e21, uint16_t e20, uint16_t e19, uint16_t e18, uint16_t e17, uint16_t e16, + uint16_t e15, uint16_t e14, uint16_t e13, uint16_t e12, uint16_t e11, uint16_t e10, uint16_t e9, uint16_t e8, + uint16_t e7, uint16_t e6, uint16_t e5, uint16_t e4, uint16_t e3, uint16_t e2, uint16_t e1, uint16_t e0) { + simde__m512i_private r_; + + r_.u16[ 0] = e0; + r_.u16[ 1] = e1; + r_.u16[ 2] = e2; + r_.u16[ 3] = e3; + r_.u16[ 4] = e4; + r_.u16[ 5] = e5; + r_.u16[ 6] = e6; + r_.u16[ 7] = e7; + r_.u16[ 8] = e8; + r_.u16[ 9] = e9; + r_.u16[10] = e10; + r_.u16[11] = e11; + r_.u16[12] = e12; + r_.u16[13] = e13; + r_.u16[14] = e14; + r_.u16[15] = e15; + r_.u16[16] = e16; + r_.u16[17] = e17; + r_.u16[18] = e18; + r_.u16[19] = e19; + r_.u16[20] = e20; + r_.u16[21] = e21; + r_.u16[22] = e22; + r_.u16[23] = e23; + r_.u16[24] = e24; + r_.u16[25] = e25; + r_.u16[26] = e26; + r_.u16[27] = e27; + r_.u16[28] = e28; + r_.u16[29] = e29; + r_.u16[30] = e30; + r_.u16[31] = e31; + + return simde__m512i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_set_epu32 (uint32_t e15, uint32_t e14, uint32_t e13, uint32_t e12, uint32_t e11, uint32_t e10, uint32_t e9, uint32_t e8, + uint32_t e7, uint32_t e6, uint32_t e5, uint32_t e4, uint32_t e3, uint32_t e2, uint32_t e1, uint32_t e0) { + simde__m512i_private r_; + + r_.u32[ 0] = e0; + r_.u32[ 1] = e1; + r_.u32[ 2] = e2; + r_.u32[ 3] = e3; + r_.u32[ 4] = e4; + r_.u32[ 5] = e5; + r_.u32[ 6] = e6; + r_.u32[ 7] = e7; + r_.u32[ 8] = e8; + r_.u32[ 9] = e9; + r_.u32[10] = e10; + r_.u32[11] = e11; + r_.u32[12] = e12; + r_.u32[13] = e13; + r_.u32[14] = e14; + r_.u32[15] = e15; + + return simde__m512i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_set_epu64 (uint64_t e7, uint64_t e6, uint64_t e5, uint64_t e4, uint64_t e3, uint64_t e2, uint64_t e1, uint64_t e0) { + simde__m512i_private r_; + + r_.u64[ 0] = e0; + r_.u64[ 1] = e1; + r_.u64[ 2] = e2; + r_.u64[ 3] = e3; + r_.u64[ 4] = e4; + r_.u64[ 5] = e5; + r_.u64[ 6] = e6; + r_.u64[ 7] = e7; + + return simde__m512i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_set_epi8 (int8_t e63, int8_t e62, int8_t e61, int8_t e60, int8_t e59, int8_t e58, int8_t e57, int8_t e56, + int8_t e55, int8_t e54, int8_t e53, int8_t e52, int8_t e51, int8_t e50, int8_t e49, int8_t e48, + int8_t e47, int8_t e46, int8_t e45, int8_t e44, int8_t e43, int8_t e42, int8_t e41, int8_t e40, + int8_t e39, int8_t e38, int8_t e37, int8_t e36, int8_t e35, int8_t e34, int8_t e33, int8_t e32, + int8_t e31, int8_t e30, int8_t e29, int8_t e28, int8_t e27, int8_t e26, int8_t e25, int8_t e24, + int8_t e23, int8_t e22, int8_t e21, int8_t e20, int8_t e19, int8_t e18, int8_t e17, int8_t e16, + int8_t e15, int8_t e14, int8_t e13, int8_t e12, int8_t e11, int8_t e10, int8_t e9, int8_t e8, + int8_t e7, int8_t e6, int8_t e5, int8_t e4, int8_t e3, int8_t e2, int8_t e1, int8_t e0) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && (HEDLEY_GCC_VERSION_CHECK(10,0,0) || SIMDE_DETECT_CLANG_VERSION_CHECK(5,0,0)) + return _mm512_set_epi8( + e63, e62, e61, e60, e59, e58, e57, e56, + e55, e54, e53, e52, e51, e50, e49, e48, + e47, e46, e45, e44, e43, e42, e41, e40, + e39, e38, e37, e36, e35, e34, e33, e32, + e31, e30, e29, e28, e27, e26, e25, e24, + e23, e22, e21, e20, e19, e18, e17, e16, + e15, e14, e13, e12, e11, e10, e9, e8, + e7, e6, e5, e4, e3, e2, e1, e0 + ); + #else + simde__m512i_private r_; + + r_.i8[ 0] = e0; + r_.i8[ 1] = e1; + r_.i8[ 2] = e2; + r_.i8[ 3] = e3; + r_.i8[ 4] = e4; + r_.i8[ 5] = e5; + r_.i8[ 6] = e6; + r_.i8[ 7] = e7; + r_.i8[ 8] = e8; + r_.i8[ 9] = e9; + r_.i8[10] = e10; + r_.i8[11] = e11; + r_.i8[12] = e12; + r_.i8[13] = e13; + r_.i8[14] = e14; + r_.i8[15] = e15; + r_.i8[16] = e16; + r_.i8[17] = e17; + r_.i8[18] = e18; + r_.i8[19] = e19; + r_.i8[20] = e20; + r_.i8[21] = e21; + r_.i8[22] = e22; + r_.i8[23] = e23; + r_.i8[24] = e24; + r_.i8[25] = e25; + r_.i8[26] = e26; + r_.i8[27] = e27; + r_.i8[28] = e28; + r_.i8[29] = e29; + r_.i8[30] = e30; + r_.i8[31] = e31; + r_.i8[32] = e32; + r_.i8[33] = e33; + r_.i8[34] = e34; + r_.i8[35] = e35; + r_.i8[36] = e36; + r_.i8[37] = e37; + r_.i8[38] = e38; + r_.i8[39] = e39; + r_.i8[40] = e40; + r_.i8[41] = e41; + r_.i8[42] = e42; + r_.i8[43] = e43; + r_.i8[44] = e44; + r_.i8[45] = e45; + r_.i8[46] = e46; + r_.i8[47] = e47; + r_.i8[48] = e48; + r_.i8[49] = e49; + r_.i8[50] = e50; + r_.i8[51] = e51; + r_.i8[52] = e52; + r_.i8[53] = e53; + r_.i8[54] = e54; + r_.i8[55] = e55; + r_.i8[56] = e56; + r_.i8[57] = e57; + r_.i8[58] = e58; + r_.i8[59] = e59; + r_.i8[60] = e60; + r_.i8[61] = e61; + r_.i8[62] = e62; + r_.i8[63] = e63; + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set_epi8 + #define _mm512_set_epi8(e63, e62, e61, e60, e59, e58, e57, e56, e55, e54, e53, e52, e51, e50, e49, e48, e47, e46, e45, e44, e43, e42, e41, e40, e39, e38, e37, e36, e35, e34, e33, e32, e31, e30, e29, e28, e27, e26, e25, e24, e23, e22, e21, e20, e19, e18, e17, e16, e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) simde_mm512_set_epi8(e63, e62, e61, e60, e59, e58, e57, e56, e55, e54, e53, e52, e51, e50, e49, e48, e47, e46, e45, e44, e43, e42, e41, e40, e39, e38, e37, e36, e35, e34, e33, e32, e31, e30, e29, e28, e27, e26, e25, e24, e23, e22, e21, e20, e19, e18, e17, e16, e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_set_m128i (simde__m128i a, simde__m128i b, simde__m128i c, simde__m128i d) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + SIMDE_ALIGN_TO_64 simde__m128i v[] = { d, c, b, a }; + return simde_mm512_load_si512(HEDLEY_STATIC_CAST(__m512i *, HEDLEY_STATIC_CAST(void *, v))); + #else + simde__m512i_private r_; + + r_.m128i[0] = d; + r_.m128i[1] = c; + r_.m128i[2] = b; + r_.m128i[3] = a; + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_x_mm512_set_m256 (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + SIMDE_ALIGN_TO_64 simde__m256 v[] = { b, a }; + return simde_mm512_load_ps(HEDLEY_STATIC_CAST(__m512 *, HEDLEY_STATIC_CAST(void *, v))); + #else + simde__m512_private r_; + + r_.m256[0] = b; + r_.m256[1] = a; + + return simde__m512_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_set_m256i (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + SIMDE_ALIGN_TO_64 simde__m256i v[] = { b, a }; + return simde_mm512_load_si512(HEDLEY_STATIC_CAST(__m512i *, HEDLEY_STATIC_CAST(void *, v))); + #else + simde__m512i_private r_; + + r_.m256i[0] = b; + r_.m256i[1] = a; + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_x_mm512_set_m256d (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + SIMDE_ALIGN_TO_64 simde__m256d v[] = { b, a }; + return simde_mm512_load_pd(HEDLEY_STATIC_CAST(__m512d *, HEDLEY_STATIC_CAST(void *, v))); + #else + simde__m512d_private r_; + + r_.m256d[0] = b; + r_.m256d[1] = a; + + return simde__m512d_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_set_ps (simde_float32 e15, simde_float32 e14, simde_float32 e13, simde_float32 e12, + simde_float32 e11, simde_float32 e10, simde_float32 e9, simde_float32 e8, + simde_float32 e7, simde_float32 e6, simde_float32 e5, simde_float32 e4, + simde_float32 e3, simde_float32 e2, simde_float32 e1, simde_float32 e0) { + simde__m512_private r_; + + r_.f32[ 0] = e0; + r_.f32[ 1] = e1; + r_.f32[ 2] = e2; + r_.f32[ 3] = e3; + r_.f32[ 4] = e4; + r_.f32[ 5] = e5; + r_.f32[ 6] = e6; + r_.f32[ 7] = e7; + r_.f32[ 8] = e8; + r_.f32[ 9] = e9; + r_.f32[10] = e10; + r_.f32[11] = e11; + r_.f32[12] = e12; + r_.f32[13] = e13; + r_.f32[14] = e14; + r_.f32[15] = e15; + + return simde__m512_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set_ps + #define _mm512_set_ps(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) simde_mm512_set_ps(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_set_pd (simde_float64 e7, simde_float64 e6, simde_float64 e5, simde_float64 e4, simde_float64 e3, simde_float64 e2, simde_float64 e1, simde_float64 e0) { + simde__m512d_private r_; + + r_.f64[0] = e0; + r_.f64[1] = e1; + r_.f64[2] = e2; + r_.f64[3] = e3; + r_.f64[4] = e4; + r_.f64[5] = e5; + r_.f64[6] = e6; + r_.f64[7] = e7; + + return simde__m512d_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set_pd + #define _mm512_set_pd(e7, e6, e5, e4, e3, e2, e1, e0) simde_mm512_set_pd(e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512h +simde_mm512_set_ph (simde_float16 e31, simde_float16 e30, simde_float16 e29, simde_float16 e28, simde_float16 e27, simde_float16 e26, simde_float16 e25, simde_float16 e24, + simde_float16 e23, simde_float16 e22, simde_float16 e21, simde_float16 e20, simde_float16 e19, simde_float16 e18, simde_float16 e17, simde_float16 e16, + simde_float16 e15, simde_float16 e14, simde_float16 e13, simde_float16 e12, simde_float16 e11, simde_float16 e10, simde_float16 e9, simde_float16 e8, + simde_float16 e7, simde_float16 e6, simde_float16 e5, simde_float16 e4, simde_float16 e3, simde_float16 e2, simde_float16 e1, simde_float16 e0) { + simde__m512h_private r_; + + r_.f16[0] = e0; + r_.f16[1] = e1; + r_.f16[2] = e2; + r_.f16[3] = e3; + r_.f16[4] = e4; + r_.f16[5] = e5; + r_.f16[6] = e6; + r_.f16[7] = e7; + r_.f16[8] = e8; + r_.f16[9] = e9; + r_.f16[10] = e10; + r_.f16[11] = e11; + r_.f16[12] = e12; + r_.f16[13] = e13; + r_.f16[14] = e14; + r_.f16[15] = e15; + r_.f16[16] = e16; + r_.f16[17] = e17; + r_.f16[18] = e18; + r_.f16[19] = e19; + r_.f16[20] = e20; + r_.f16[21] = e21; + r_.f16[22] = e22; + r_.f16[23] = e23; + r_.f16[24] = e24; + r_.f16[25] = e25; + r_.f16[26] = e26; + r_.f16[27] = e27; + r_.f16[28] = e28; + r_.f16[29] = e29; + r_.f16[30] = e30; + r_.f16[31] = e31; + + return simde__m512h_from_private(r_); +} +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) + #undef _mm512_set_ph + #define _mm512_set_ph(e31, e30, e29, e28, e27, e26, e25, e24, e23, e22, e21, e20, e19, e18, e17, e16, e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) \ + simde_mm512_set_ph(e31, e30, e29, e28, e27, e26, e25, e24, e23, e22, e21, e20, e19, e18, e17, e16, e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) +#endif + + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SET_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/set1.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/set1.h new file mode 100644 index 00000000..33ae8418 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/set1.h @@ -0,0 +1,352 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + */ + +#if !defined(SIMDE_X86_AVX512_SET1_H) +#define SIMDE_X86_AVX512_SET1_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_set1_epi8 (int8_t a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_set1_epi8(a); + #else + simde__m512i_private r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set1_epi8 + #define _mm512_set1_epi8(a) simde_mm512_set1_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_set1_epi8(simde__m512i src, simde__mmask64 k, int8_t a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_set1_epi8(src, k, a); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_set1_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_set1_epi8 + #define _mm512_mask_set1_epi8(src, k, a) simde_mm512_mask_set1_epi8(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_set1_epi8(simde__mmask64 k, int8_t a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_set1_epi8(k, a); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_set1_epi8(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_set1_epi8 + #define _mm512_maskz_set1_epi8(k, a) simde_mm512_maskz_set1_epi8(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_set1_epi16 (int16_t a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_set1_epi16(a); + #else + simde__m512i_private r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set1_epi16 + #define _mm512_set1_epi16(a) simde_mm512_set1_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_set1_epi16(simde__m512i src, simde__mmask32 k, int16_t a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_set1_epi16(src, k, a); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_set1_epi16(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_set1_epi16 + #define _mm512_mask_set1_epi16(src, k, a) simde_mm512_mask_set1_epi16(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_set1_epi16(simde__mmask32 k, int16_t a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_set1_epi16(k, a); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_set1_epi16(a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_set1_epi16 + #define _mm512_maskz_set1_epi16(k, a) simde_mm512_maskz_set1_epi16(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_set1_epi32 (int32_t a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_set1_epi32(a); + #else + simde__m512i_private r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set1_epi32 + #define _mm512_set1_epi32(a) simde_mm512_set1_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_set1_epi32(simde__m512i src, simde__mmask16 k, int32_t a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_set1_epi32(src, k, a); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_set1_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_set1_epi32 + #define _mm512_mask_set1_epi32(src, k, a) simde_mm512_mask_set1_epi32(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_set1_epi32(simde__mmask16 k, int32_t a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_set1_epi32(k, a); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_set1_epi32(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_set1_epi32 + #define _mm512_maskz_set1_epi32(k, a) simde_mm512_maskz_set1_epi32(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_set1_epi64 (int64_t a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_set1_epi64(a); + #else + simde__m512i_private r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set1_epi64 + #define _mm512_set1_epi64(a) simde_mm512_set1_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_set1_epi64(simde__m512i src, simde__mmask8 k, int64_t a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_set1_epi64(src, k, a); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_set1_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_set1_epi64 + #define _mm512_mask_set1_epi64(src, k, a) simde_mm512_mask_set1_epi64(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_set1_epi64(simde__mmask8 k, int64_t a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_set1_epi64(k, a); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_set1_epi64(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_set1_epi64 + #define _mm512_maskz_set1_epi64(k, a) simde_mm512_maskz_set1_epi64(k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_set1_epu8 (uint8_t a) { + simde__m512i_private r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = a; + } + + return simde__m512i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_set1_epu16 (uint16_t a) { + simde__m512i_private r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = a; + } + + return simde__m512i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_set1_epu32 (uint32_t a) { + simde__m512i_private r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a; + } + + return simde__m512i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_set1_epu64 (uint64_t a) { + simde__m512i_private r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = a; + } + + return simde__m512i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_set1_ps (simde_float32 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_set1_ps(a); + #else + simde__m512_private r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a; + } + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set1_ps + #define _mm512_set1_ps(a) simde_mm512_set1_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_set1_pd (simde_float64 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_set1_pd(a); + #else + simde__m512d_private r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a; + } + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set1_pd + #define _mm512_set1_pd(a) simde_mm512_set1_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512h +simde_mm512_set1_ph (simde_float16 a) { + #if defined(SIMDE_X86_AVX512FP16_NATIVE) + return _mm512_set1_ph(a); + #else + simde__m512h_private r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f16) / sizeof(r_.f16[0])) ; i++) { + r_.f16[i] = a; + } + + return simde__m512h_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) + #undef _mm512_set1_ph + #define _mm512_set1_ph(a) simde_mm512_set1_ph(a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SET1_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/set4.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/set4.h new file mode 100644 index 00000000..379f829c --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/set4.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_SET4_H) +#define SIMDE_X86_AVX512_SET4_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_set4_epi32 (int32_t d, int32_t c, int32_t b, int32_t a) { + simde__m512i_private r_; + + r_.i32[ 0] = a; + r_.i32[ 1] = b; + r_.i32[ 2] = c; + r_.i32[ 3] = d; + r_.i32[ 4] = a; + r_.i32[ 5] = b; + r_.i32[ 6] = c; + r_.i32[ 7] = d; + r_.i32[ 8] = a; + r_.i32[ 9] = b; + r_.i32[10] = c; + r_.i32[11] = d; + r_.i32[12] = a; + r_.i32[13] = b; + r_.i32[14] = c; + r_.i32[15] = d; + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set4_epi32 + #define _mm512_set4_epi32(d,c,b,a) simde_mm512_set4_epi32(d,c,b,a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_set4_epi64 (int64_t d, int64_t c, int64_t b, int64_t a) { + simde__m512i_private r_; + + r_.i64[0] = a; + r_.i64[1] = b; + r_.i64[2] = c; + r_.i64[3] = d; + r_.i64[4] = a; + r_.i64[5] = b; + r_.i64[6] = c; + r_.i64[7] = d; + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set4_epi64 + #define _mm512_set4_epi64(d,c,b,a) simde_mm512_set4_epi64(d,c,b,a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_set4_ps (simde_float32 d, simde_float32 c, simde_float32 b, simde_float32 a) { + simde__m512_private r_; + + r_.f32[ 0] = a; + r_.f32[ 1] = b; + r_.f32[ 2] = c; + r_.f32[ 3] = d; + r_.f32[ 4] = a; + r_.f32[ 5] = b; + r_.f32[ 6] = c; + r_.f32[ 7] = d; + r_.f32[ 8] = a; + r_.f32[ 9] = b; + r_.f32[10] = c; + r_.f32[11] = d; + r_.f32[12] = a; + r_.f32[13] = b; + r_.f32[14] = c; + r_.f32[15] = d; + + return simde__m512_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set4_ps + #define _mm512_set4_ps(d,c,b,a) simde_mm512_set4_ps(d,c,b,a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_set4_pd (simde_float64 d, simde_float64 c, simde_float64 b, simde_float64 a) { + simde__m512d_private r_; + + r_.f64[0] = a; + r_.f64[1] = b; + r_.f64[2] = c; + r_.f64[3] = d; + r_.f64[4] = a; + r_.f64[5] = b; + r_.f64[6] = c; + r_.f64[7] = d; + + return simde__m512d_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_set4_pd + #define _mm512_set4_pd(d,c,b,a) simde_mm512_set4_pd(d,c,b,a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SET4_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setone.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setone.h new file mode 100644 index 00000000..df2f6e8b --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setone.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_SETONE_H) +#define SIMDE_X86_AVX512_SETONE_H + +#include "types.h" +#include "cast.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_setone_si512(void) { + simde__m512i_private r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = ~HEDLEY_STATIC_CAST(int_fast32_t, 0); + } + + return simde__m512i_from_private(r_); +} +#define simde_x_mm512_setone_epi32() simde_x_mm512_setone_si512() + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_x_mm512_setone_ps(void) { + return simde_mm512_castsi512_ps(simde_x_mm512_setone_si512()); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_x_mm512_setone_pd(void) { + return simde_mm512_castsi512_pd(simde_x_mm512_setone_si512()); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512h +simde_x_mm512_setone_ph(void) { + return simde_mm512_castsi512_ph(simde_x_mm512_setone_si512()); +} + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SETONE_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setr.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setr.h new file mode 100644 index 00000000..c44bed46 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setr.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + */ + +#if !defined(SIMDE_X86_AVX512_SETR_H) +#define SIMDE_X86_AVX512_SETR_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_setr_epi32 (int32_t e15, int32_t e14, int32_t e13, int32_t e12, int32_t e11, int32_t e10, int32_t e9, int32_t e8, + int32_t e7, int32_t e6, int32_t e5, int32_t e4, int32_t e3, int32_t e2, int32_t e1, int32_t e0) { + simde__m512i_private r_; + + r_.i32[ 0] = e15; + r_.i32[ 1] = e14; + r_.i32[ 2] = e13; + r_.i32[ 3] = e12; + r_.i32[ 4] = e11; + r_.i32[ 5] = e10; + r_.i32[ 6] = e9; + r_.i32[ 7] = e8; + r_.i32[ 8] = e7; + r_.i32[ 9] = e6; + r_.i32[10] = e5; + r_.i32[11] = e4; + r_.i32[12] = e3; + r_.i32[13] = e2; + r_.i32[14] = e1; + r_.i32[15] = e0; + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_setr_epi32 + #define _mm512_setr_epi32(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) simde_mm512_setr_epi32(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_setr_epi64 (int64_t e7, int64_t e6, int64_t e5, int64_t e4, int64_t e3, int64_t e2, int64_t e1, int64_t e0) { + simde__m512i_private r_; + + r_.i64[0] = e7; + r_.i64[1] = e6; + r_.i64[2] = e5; + r_.i64[3] = e4; + r_.i64[4] = e3; + r_.i64[5] = e2; + r_.i64[6] = e1; + r_.i64[7] = e0; + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_setr_epi64 + #define _mm512_setr_epi64(e7, e6, e5, e4, e3, e2, e1, e0) simde_mm512_setr_epi64(e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_setr_ps (simde_float32 e15, simde_float32 e14, simde_float32 e13, simde_float32 e12, + simde_float32 e11, simde_float32 e10, simde_float32 e9, simde_float32 e8, + simde_float32 e7, simde_float32 e6, simde_float32 e5, simde_float32 e4, + simde_float32 e3, simde_float32 e2, simde_float32 e1, simde_float32 e0) { + simde__m512_private r_; + + r_.f32[ 0] = e15; + r_.f32[ 1] = e14; + r_.f32[ 2] = e13; + r_.f32[ 3] = e12; + r_.f32[ 4] = e11; + r_.f32[ 5] = e10; + r_.f32[ 6] = e9; + r_.f32[ 7] = e8; + r_.f32[ 8] = e7; + r_.f32[ 9] = e6; + r_.f32[10] = e5; + r_.f32[11] = e4; + r_.f32[12] = e3; + r_.f32[13] = e2; + r_.f32[14] = e1; + r_.f32[15] = e0; + + return simde__m512_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_setr_ps + #define _mm512_setr_ps(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) simde_mm512_setr_ps(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_setr_pd (simde_float64 e7, simde_float64 e6, simde_float64 e5, simde_float64 e4, simde_float64 e3, simde_float64 e2, simde_float64 e1, simde_float64 e0) { + simde__m512d_private r_; + + r_.f64[0] = e7; + r_.f64[1] = e6; + r_.f64[2] = e5; + r_.f64[3] = e4; + r_.f64[4] = e3; + r_.f64[5] = e2; + r_.f64[6] = e1; + r_.f64[7] = e0; + + return simde__m512d_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_setr_pd + #define _mm512_setr_pd(e7, e6, e5, e4, e3, e2, e1, e0) simde_mm512_setr_pd(e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SETR_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setr4.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setr4.h new file mode 100644 index 00000000..7ee59214 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setr4.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + */ + +#if !defined(SIMDE_X86_AVX512_SETR4_H) +#define SIMDE_X86_AVX512_SETR4_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_setr4_epi32 (int32_t d, int32_t c, int32_t b, int32_t a) { + simde__m512i_private r_; + + r_.i32[ 0] = d; + r_.i32[ 1] = c; + r_.i32[ 2] = b; + r_.i32[ 3] = a; + r_.i32[ 4] = d; + r_.i32[ 5] = c; + r_.i32[ 6] = b; + r_.i32[ 7] = a; + r_.i32[ 8] = d; + r_.i32[ 9] = c; + r_.i32[10] = b; + r_.i32[11] = a; + r_.i32[12] = d; + r_.i32[13] = c; + r_.i32[14] = b; + r_.i32[15] = a; + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_setr4_epi32 + #define _mm512_setr4_epi32(d,c,b,a) simde_mm512_setr4_epi32(d,c,b,a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_setr4_epi64 (int64_t d, int64_t c, int64_t b, int64_t a) { + simde__m512i_private r_; + + r_.i64[0] = d; + r_.i64[1] = c; + r_.i64[2] = b; + r_.i64[3] = a; + r_.i64[4] = d; + r_.i64[5] = c; + r_.i64[6] = b; + r_.i64[7] = a; + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_setr4_epi64 + #define _mm512_setr4_epi64(d,c,b,a) simde_mm512_setr4_epi64(d,c,b,a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_setr4_ps (simde_float32 d, simde_float32 c, simde_float32 b, simde_float32 a) { + simde__m512_private r_; + + r_.f32[ 0] = d; + r_.f32[ 1] = c; + r_.f32[ 2] = b; + r_.f32[ 3] = a; + r_.f32[ 4] = d; + r_.f32[ 5] = c; + r_.f32[ 6] = b; + r_.f32[ 7] = a; + r_.f32[ 8] = d; + r_.f32[ 9] = c; + r_.f32[10] = b; + r_.f32[11] = a; + r_.f32[12] = d; + r_.f32[13] = c; + r_.f32[14] = b; + r_.f32[15] = a; + + return simde__m512_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_setr4_ps + #define _mm512_setr4_ps(d,c,b,a) simde_mm512_setr4_ps(d,c,b,a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_setr4_pd (simde_float64 d, simde_float64 c, simde_float64 b, simde_float64 a) { + simde__m512d_private r_; + + r_.f64[0] = d; + r_.f64[1] = c; + r_.f64[2] = b; + r_.f64[3] = a; + r_.f64[4] = d; + r_.f64[5] = c; + r_.f64[6] = b; + r_.f64[7] = a; + + return simde__m512d_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_setr4_pd + #define _mm512_setr4_pd(d,c,b,a) simde_mm512_setr4_pd(d,c,b,a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SETR4_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setzero.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setzero.h new file mode 100644 index 00000000..c5bfdc45 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/setzero.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + * 2020 Hidayat Khan + * 2020 Christopher Moore + */ + +#if !defined(SIMDE_X86_AVX512_SETZERO_H) +#define SIMDE_X86_AVX512_SETZERO_H + +#include "types.h" +#include "cast.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_setzero_si512(void) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_setzero_si512(); + #else + simde__m512i r; + simde_memset(&r, 0, sizeof(r)); + return r; + #endif +} +#define simde_mm512_setzero_epi32() simde_mm512_setzero_si512() +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_setzero_si512 + #define _mm512_setzero_si512() simde_mm512_setzero_si512() + #undef _mm512_setzero_epi32 + #define _mm512_setzero_epi32() simde_mm512_setzero_si512() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_setzero_ps(void) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_setzero_ps(); + #else + return simde_mm512_castsi512_ps(simde_mm512_setzero_si512()); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_setzero_ps + #define _mm512_setzero_ps() simde_mm512_setzero_ps() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_setzero_pd(void) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_setzero_pd(); + #else + return simde_mm512_castsi512_pd(simde_mm512_setzero_si512()); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_setzero_pd + #define _mm512_setzero_pd() simde_mm512_setzero_pd() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512h +simde_mm512_setzero_ph(void) { + #if defined(SIMDE_X86_AVX512FP16_NATIVE) + return _mm512_setzero_ph(); + #else + return simde_mm512_castsi512_ph(simde_mm512_setzero_si512()); + #endif +} +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) + #undef _mm512_setzero_ph + #define _mm512_setzero_ph() simde_mm512_setzero_ph() +#endif + + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SETZERO_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/shldv.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/shldv.h new file mode 100644 index 00000000..1cd38f1f --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/shldv.h @@ -0,0 +1,157 @@ +#if !defined(SIMDE_X86_AVX512_SHLDV_H) +#define SIMDE_X86_AVX512_SHLDV_H + +#include "types.h" +#include "../avx2.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_shldv_epi32(simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_AVX512VBMI2_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_shldv_epi32(a, b, c); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + uint64x2_t + values_lo = vreinterpretq_u64_u32(vzip1q_u32(b_.neon_u32, a_.neon_u32)), + values_hi = vreinterpretq_u64_u32(vzip2q_u32(b_.neon_u32, a_.neon_u32)); + + int32x4_t count = vandq_s32(c_.neon_i32, vdupq_n_s32(31)); + + values_lo = vshlq_u64(values_lo, vmovl_s32(vget_low_s32(count))); + values_hi = vshlq_u64(values_hi, vmovl_high_s32(count)); + + r_.neon_u32 = + vuzp2q_u32( + vreinterpretq_u32_u64(values_lo), + vreinterpretq_u32_u64(values_hi) + ); + #elif defined(SIMDE_X86_AVX2_NATIVE) + simde__m256i + tmp1, + lo = + simde_mm256_castps_si256( + simde_mm256_unpacklo_ps( + simde_mm256_castsi256_ps(simde_mm256_castsi128_si256(b)), + simde_mm256_castsi256_ps(simde_mm256_castsi128_si256(a)) + ) + ), + hi = + simde_mm256_castps_si256( + simde_mm256_unpackhi_ps( + simde_mm256_castsi256_ps(simde_mm256_castsi128_si256(b)), + simde_mm256_castsi256_ps(simde_mm256_castsi128_si256(a)) + ) + ), + tmp2 = + simde_mm256_castpd_si256( + simde_mm256_permute2f128_pd( + simde_mm256_castsi256_pd(lo), + simde_mm256_castsi256_pd(hi), + 32 + ) + ); + + tmp2 = + simde_mm256_sllv_epi64( + tmp2, + simde_mm256_cvtepi32_epi64( + simde_mm_and_si128( + c, + simde_mm_set1_epi32(31) + ) + ) + ); + + tmp1 = + simde_mm256_castpd_si256( + simde_mm256_permute2f128_pd( + simde_mm256_castsi256_pd(tmp2), + simde_mm256_castsi256_pd(tmp2), + 1 + ) + ); + + r_ = + simde__m128i_to_private( + simde_mm256_castsi256_si128( + simde_mm256_castps_si256( + simde_mm256_shuffle_ps( + simde_mm256_castsi256_ps(tmp2), + simde_mm256_castsi256_ps(tmp1), + 221 + ) + ) + ) + ); + #elif defined(SIMDE_X86_SSE2_NATIVE) + simde__m128i_private + c_ = simde__m128i_to_private(c), + lo = simde__m128i_to_private(simde_mm_unpacklo_epi32(b, a)), + hi = simde__m128i_to_private(simde_mm_unpackhi_epi32(b, a)); + + size_t halfway = (sizeof(r_.u32) / sizeof(r_.u32[0]) / 2); + SIMDE_VECTORIZE + for (size_t i = 0 ; i < halfway ; i++) { + lo.u64[i] <<= (c_.u32[i] & 31); + hi.u64[i] <<= (c_.u32[halfway + i] & 31); + } + + r_ = + simde__m128i_to_private( + simde_mm_castps_si128( + simde_mm_shuffle_ps( + simde_mm_castsi128_ps(simde__m128i_from_private(lo)), + simde_mm_castsi128_ps(simde__m128i_from_private(hi)), + 221) + ) + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_CONVERT_VECTOR_) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + simde__m128i_private + c_ = simde__m128i_to_private(c); + simde__m256i_private + a_ = simde__m256i_to_private(simde_mm256_castsi128_si256(a)), + b_ = simde__m256i_to_private(simde_mm256_castsi128_si256(b)), + tmp1, + tmp2; + + tmp1.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(tmp1.u64), SIMDE_SHUFFLE_VECTOR_(32, 32, b_.i32, a_.i32, 0, 8, 1, 9, 2, 10, 3, 11)); + SIMDE_CONVERT_VECTOR_(tmp2.u64, c_.u32); + + tmp1.u64 <<= (tmp2.u64 & 31); + + r_.i32 = SIMDE_SHUFFLE_VECTOR_(32, 16, tmp1.m128i_private[0].i32, tmp1.m128i_private[1].i32, 1, 3, 5, 7); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = HEDLEY_STATIC_CAST(uint32_t, (((HEDLEY_STATIC_CAST(uint64_t, a_.u32[i]) << 32) | b_.u32[i]) << (c_.u32[i] & 31)) >> 32); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512VBMI2_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_shldv_epi32 + #define _mm_shldv_epi32(a, b, c) simde_mm_shldv_epi32(a, b, c) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SHLDV_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/shuffle.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/shuffle.h new file mode 100644 index 00000000..d1c537f3 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/shuffle.h @@ -0,0 +1,417 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Christopher Moore + * 2023 Michael R. Crusoe + */ + +#if !defined(SIMDE_X86_AVX512_SHUFFLE_H) +#define SIMDE_X86_AVX512_SHUFFLE_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" +#include "extract.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_shuffle_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_shuffle_epi8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(a_.m256i) / sizeof(a_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_shuffle_epi8(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (b_.i8[i] & 0x80) ? 0 : a_.i8[(b_.i8[i] & 0x0f) + (i & 0x30)]; + } + #endif + + return simde__m512i_from_private(r_); +#endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_shuffle_epi8 + #define _mm512_shuffle_epi8(a, b) simde_mm512_shuffle_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_shuffle_epi8 (simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_shuffle_epi8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_shuffle_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_shuffle_epi8 + #define _mm512_mask_shuffle_epi8(src, k, a, b) simde_mm512_mask_shuffle_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_shuffle_epi8 (simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_shuffle_epi8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_shuffle_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_shuffle_epi8 + #define _mm512_maskz_shuffle_epi8(k, a, b) simde_mm512_maskz_shuffle_epi8(k, a, b) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) +# define simde_mm512_shuffle_epi32(a, imm8) _mm512_shuffle_epi32((a), (imm8)) +#elif defined(SIMDE_STATEMENT_EXPR_) +# define simde_mm512_shuffle_epi32(a, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512i_private simde_mm512_shuffle_epi32_r_, \ + simde_mm512_shuffle_epi32_a_ = simde__m512i_to_private((a)); \ + simde_mm512_shuffle_epi32_r_.m128i[0] = simde_mm_shuffle_epi32( \ + simde_mm512_shuffle_epi32_a_.m128i[0], (imm8)); \ + simde_mm512_shuffle_epi32_r_.m128i[1] = simde_mm_shuffle_epi32( \ + simde_mm512_shuffle_epi32_a_.m128i[1], (imm8)); \ + simde_mm512_shuffle_epi32_r_.m128i[2] = simde_mm_shuffle_epi32( \ + simde_mm512_shuffle_epi32_a_.m128i[2], (imm8)); \ + simde_mm512_shuffle_epi32_r_.m128i[3] = simde_mm_shuffle_epi32( \ + simde_mm512_shuffle_epi32_a_.m128i[3], (imm8)); \ + simde__m512i_from_private(simde_mm512_shuffle_epi32_r_); \ + })) +#else +# define simde_mm512_shuffle_epi32(a, imm8) \ + simde_x_mm512_set_m128i( \ + simde_mm_shuffle_epi32(simde_mm512_extracti32x4_epi32(a, 3), (imm8)), \ + simde_mm_shuffle_epi32(simde_mm512_extracti32x4_epi32(a, 2), (imm8)), \ + simde_mm_shuffle_epi32(simde_mm512_extracti32x4_epi32(a, 1), (imm8)), \ + simde_mm_shuffle_epi32(simde_mm512_extracti32x4_epi32(a, 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_shuffle_epi32 + #define _mm512_shuffle_epi32(a, imm8) simde_mm512_shuffle_epi32((a), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_shuffle_i32x4 (simde__m256i a, simde__m256i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + r_.m128i[0] = a_.m128i[ imm8 & 1]; + r_.m128i[1] = b_.m128i[(imm8 >> 1) & 1]; + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_shuffle_i32x4(a, b, imm8) _mm256_shuffle_i32x4(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_shuffle_i32x4 + #define _mm256_shuffle_i32x4(a, b, imm8) simde_mm256_shuffle_i32x4(a, b, imm8) +#endif + +#define simde_mm256_maskz_shuffle_i32x4(k, a, b, imm8) simde_mm256_maskz_mov_epi32(k, simde_mm256_shuffle_i32x4(a, b, imm8)) +#define simde_mm256_mask_shuffle_i32x4(src, k, a, b, imm8) simde_mm256_mask_mov_epi32(src, k, simde_mm256_shuffle_i32x4(a, b, imm8)) + +#define simde_mm256_shuffle_f32x4(a, b, imm8) simde_mm256_castsi256_ps(simde_mm256_shuffle_i32x4(simde_mm256_castps_si256(a), simde_mm256_castps_si256(b), imm8)) +#define simde_mm256_maskz_shuffle_f32x4(k, a, b, imm8) simde_mm256_maskz_mov_ps(k, simde_mm256_shuffle_f32x4(a, b, imm8)) +#define simde_mm256_mask_shuffle_f32x4(src, k, a, b, imm8) simde_mm256_mask_mov_ps(src, k, simde_mm256_shuffle_f32x4(a, b, imm8)) + +#define simde_mm256_shuffle_i64x2(a, b, imm8) simde_mm256_shuffle_i32x4(a, b, imm8) +#define simde_mm256_maskz_shuffle_i64x2(k, a, b, imm8) simde_mm256_maskz_mov_epi64(k, simde_mm256_shuffle_i64x2(a, b, imm8)) +#define simde_mm256_mask_shuffle_i64x2(src, k, a, b, imm8) simde_mm256_mask_mov_epi64(src, k, simde_mm256_shuffle_i64x2(a, b, imm8)) + +#define simde_mm256_shuffle_f64x2(a, b, imm8) simde_mm256_castsi256_pd(simde_mm256_shuffle_i64x2(simde_mm256_castpd_si256(a), simde_mm256_castpd_si256(b), imm8)) +#define simde_mm256_maskz_shuffle_f64x2(k, a, b, imm8) simde_mm256_maskz_mov_pd(k, simde_mm256_shuffle_f64x2(a, b, imm8)) +#define simde_mm256_mask_shuffle_f64x2(src, k, a, b, imm8) simde_mm256_mask_mov_pd(src, k, simde_mm256_shuffle_f64x2(a, b, imm8)) + +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_shuffle_i32x4 + #undef _mm256_mask_shuffle_i32x4 + #define _mm256_maskz_shuffle_i32x4(k, a, b, imm8) simde_mm256_maskz_shuffle_i32x4(k, a, b, imm8) + #define _mm256_mask_shuffle_i32x4(src, k, a, b, imm8) simde_mm256_mask_shuffle_i32x4(src, k, a, b, imm8) + + #undef _mm256_shuffle_f32x4 + #undef _mm256_maskz_shuffle_f32x4 + #undef _mm256_mask_shuffle_f32x4 + #define _mm256_shuffle_f32x4(a, b, imm8) simde_mm256_shuffle_f32x4(a, b, imm8) + #define _mm256_maskz_shuffle_f32x4(k, a, b, imm8) simde_mm256_maskz_shuffle_f32x4(k, a, b, imm8) + #define _mm256_mask_shuffle_f32x4(src, k, a, b, imm8) simde_mm256_mask_shuffle_f32x4(src, k, a, b, imm8) + + #undef _mm256_shuffle_i64x2 + #undef _mm256_maskz_shuffle_i64x2 + #undef _mm256_mask_shuffle_i64x2 + #define _mm256_shuffle_i64x2(a, b, imm8) simde_mm256_shuffle_i64x2(a, b, imm8) + #define _mm256_maskz_shuffle_i64x2(k, a, b, imm8) simde_mm256_maskz_shuffle_i64x2(k, a, b, imm8) + #define _mm256_mask_shuffle_i64x2(src, k, a, b, imm8) simde_mm256_mask_shuffle_i64x2(src, k, a, b, imm8) + + #undef _mm256_shuffle_f64x2 + #undef _mm256_maskz_shuffle_f64x2 + #undef _mm256_mask_shuffle_f64x2 + #define _mm256_shuffle_f64x2(a, b, imm8) simde_mm256_shuffle_f64x2(a, b, imm8) + #define _mm256_maskz_shuffle_f64x2(k, a, b, imm8) simde_mm256_maskz_shuffle_f64x2(k, a, b, imm8) + #define _mm256_mask_shuffle_f64x2(src, k, a, b, imm8) simde_mm256_mask_shuffle_f64x2(src, k, a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_shuffle_i32x4 (simde__m512i a, simde__m512i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + r_.m128i[0] = a_.m128i[ imm8 & 3]; + r_.m128i[1] = a_.m128i[(imm8 >> 2) & 3]; + r_.m128i[2] = b_.m128i[(imm8 >> 4) & 3]; + r_.m128i[3] = b_.m128i[(imm8 >> 6) & 3]; + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_shuffle_i32x4(a, b, imm8) _mm512_shuffle_i32x4(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_shuffle_i32x4 + #define _mm512_shuffle_i32x4(a, b, imm8) simde_mm512_shuffle_i32x4(a, b, imm8) +#endif + +#define simde_mm512_maskz_shuffle_i32x4(k, a, b, imm8) simde_mm512_maskz_mov_epi32(k, simde_mm512_shuffle_i32x4(a, b, imm8)) +#define simde_mm512_mask_shuffle_i32x4(src, k, a, b, imm8) simde_mm512_mask_mov_epi32(src, k, simde_mm512_shuffle_i32x4(a, b, imm8)) + +#define simde_mm512_shuffle_f32x4(a, b, imm8) simde_mm512_castsi512_ps(simde_mm512_shuffle_i32x4(simde_mm512_castps_si512(a), simde_mm512_castps_si512(b), imm8)) +#define simde_mm512_maskz_shuffle_f32x4(k, a, b, imm8) simde_mm512_maskz_mov_ps(k, simde_mm512_shuffle_f32x4(a, b, imm8)) +#define simde_mm512_mask_shuffle_f32x4(src, k, a, b, imm8) simde_mm512_mask_mov_ps(src, k, simde_mm512_shuffle_f32x4(a, b, imm8)) + +#define simde_mm512_shuffle_i64x2(a, b, imm8) simde_mm512_shuffle_i32x4(a, b, imm8) +#define simde_mm512_maskz_shuffle_i64x2(k, a, b, imm8) simde_mm512_maskz_mov_epi64(k, simde_mm512_shuffle_i64x2(a, b, imm8)) +#define simde_mm512_mask_shuffle_i64x2(src, k, a, b, imm8) simde_mm512_mask_mov_epi64(src, k, simde_mm512_shuffle_i64x2(a, b, imm8)) + +#define simde_mm512_shuffle_f64x2(a, b, imm8) simde_mm512_castsi512_pd(simde_mm512_shuffle_i64x2(simde_mm512_castpd_si512(a), simde_mm512_castpd_si512(b), imm8)) +#define simde_mm512_maskz_shuffle_f64x2(k, a, b, imm8) simde_mm512_maskz_mov_pd(k, simde_mm512_shuffle_f64x2(a, b, imm8)) +#define simde_mm512_mask_shuffle_f64x2(src, k, a, b, imm8) simde_mm512_mask_mov_pd(src, k, simde_mm512_shuffle_f64x2(a, b, imm8)) + +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_shuffle_i32x4 + #undef _mm512_mask_shuffle_i32x4 + #define _mm512_maskz_shuffle_i32x4(k, a, b, imm8) simde_mm512_maskz_shuffle_i32x4(k, a, b, imm8) + #define _mm512_mask_shuffle_i32x4(src, k, a, b, imm8) simde_mm512_mask_shuffle_i32x4(src, k, a, b, imm8) + + #undef _mm512_shuffle_f32x4 + #undef _mm512_maskz_shuffle_f32x4 + #undef _mm512_mask_shuffle_f32x4 + #define _mm512_shuffle_f32x4(a, b, imm8) simde_mm512_shuffle_f32x4(a, b, imm8) + #define _mm512_maskz_shuffle_f32x4(k, a, b, imm8) simde_mm512_maskz_shuffle_f32x4(k, a, b, imm8) + #define _mm512_mask_shuffle_f32x4(src, k, a, b, imm8) simde_mm512_mask_shuffle_f32x4(src, k, a, b, imm8) + + #undef _mm512_shuffle_i64x2 + #undef _mm512_maskz_shuffle_i64x2 + #undef _mm512_mask_shuffle_i64x2 + #define _mm512_shuffle_i64x2(a, b, imm8) simde_mm512_shuffle_i64x2(a, b, imm8) + #define _mm512_maskz_shuffle_i64x2(k, a, b, imm8) simde_mm512_maskz_shuffle_i64x2(k, a, b, imm8) + #define _mm512_mask_shuffle_i64x2(src, k, a, b, imm8) simde_mm512_mask_shuffle_i64x2(src, k, a, b, imm8) + + #undef _mm512_shuffle_f64x2 + #undef _mm512_maskz_shuffle_f64x2 + #undef _mm512_mask_shuffle_f64x2 + #define _mm512_shuffle_f64x2(a, b, imm8) simde_mm512_shuffle_f64x2(a, b, imm8) + #define _mm512_maskz_shuffle_f64x2(k, a, b, imm8) simde_mm512_maskz_shuffle_f64x2(k, a, b, imm8) + #define _mm512_mask_shuffle_f64x2(src, k, a, b, imm8) simde_mm512_mask_shuffle_f64x2(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_shuffle_ps(a, b, imm8) _mm512_shuffle_ps(a, b, imm8) +#elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_shuffle_ps(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512_private \ + simde_mm512_shuffle_ps_a_ = simde__m512_to_private(a), \ + simde_mm512_shuffle_ps_b_ = simde__m512_to_private(b); \ + \ + simde_mm512_shuffle_ps_a_.m256[0] = simde_mm256_shuffle_ps(simde_mm512_shuffle_ps_a_.m256[0], simde_mm512_shuffle_ps_b_.m256[0], imm8); \ + simde_mm512_shuffle_ps_a_.m256[1] = simde_mm256_shuffle_ps(simde_mm512_shuffle_ps_a_.m256[1], simde_mm512_shuffle_ps_b_.m256[1], imm8); \ + \ + simde__m512_from_private(simde_mm512_shuffle_ps_a_); \ + })) +#elif defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm512_shuffle_ps(a, b, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512_private \ + simde_mm512_shuffle_ps_a_ = simde__m512_to_private(a), \ + simde_mm512_shuffle_ps_b_ = simde__m512_to_private(b); \ + \ + simde_mm512_shuffle_ps_a_.f32 = \ + SIMDE_SHUFFLE_VECTOR_( \ + 32, 64, \ + simde_mm512_shuffle_ps_a_.f32, \ + simde_mm512_shuffle_ps_b_.f32, \ + (((imm8) ) & 3), \ + (((imm8) >> 2) & 3), \ + (((imm8) >> 4) & 3) + 16, \ + (((imm8) >> 6) & 3) + 16, \ + (((imm8) ) & 3) + 4, \ + (((imm8) >> 2) & 3) + 4, \ + (((imm8) >> 4) & 3) + 20, \ + (((imm8) >> 6) & 3) + 20, \ + (((imm8) ) & 3) + 8, \ + (((imm8) >> 2) & 3) + 8, \ + (((imm8) >> 4) & 3) + 24, \ + (((imm8) >> 6) & 3) + 24, \ + (((imm8) ) & 3) + 12, \ + (((imm8) >> 2) & 3) + 12, \ + (((imm8) >> 4) & 3) + 28, \ + (((imm8) >> 6) & 3) + 28 \ + ); \ + \ + simde__m512_from_private(simde_mm512_shuffle_ps_a_); \ + })) +#else + SIMDE_FUNCTION_ATTRIBUTES + simde__m512 + simde_mm512_shuffle_ps(simde__m512 a, simde__m512 b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE (imm8, 0, 255) { + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + const size_t halfway = (sizeof(r_.m128_private[0].f32) / sizeof(r_.m128_private[0].f32[0]) / 2); + for (size_t i = 0 ; i < (sizeof(r_.m128_private) / sizeof(r_.m128_private[0])) ; i++) { + SIMDE_VECTORIZE + for (size_t j = 0 ; j < halfway ; j++) { + r_.m128_private[i].f32[j] = a_.m128_private[i].f32[(imm8 >> (j * 2)) & 3]; + r_.m128_private[i].f32[halfway + j] = b_.m128_private[i].f32[(imm8 >> ((halfway + j) * 2)) & 3]; + } + } + + return simde__m512_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_shuffle_ps + #define _mm512_shuffle_ps(a, b, imm8) simde_mm512_shuffle_ps(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_shuffle_pd(simde__m512d a, simde__m512d b, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE (imm8, 0, 255) { + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < ((sizeof(r_.f64) / sizeof(r_.f64[0])) / 2) ; i++) { + r_.f64[i * 2] = (imm8 & ( 1 << (i*2) )) ? a_.f64[i * 2 + 1]: a_.f64[i * 2]; + r_.f64[i * 2 + 1] = (imm8 & ( 1 << (i*2+1) )) ? b_.f64[i * 2 + 1]: b_.f64[i * 2]; + } + + return simde__m512d_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_shuffle_pd(a, b, imm8) _mm512_shuffle_pd(a, b, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_shuffle_pd + #define _mm512_shuffle_pd(a, b, imm8) simde_mm512_shuffle_pd(a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512BW_NATIVE) +# define simde_mm512_shufflehi_epi16(a, imm8) _mm512_shufflehi_epi16(a, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) +# define simde_mm512_shufflehi_epi16(a, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512i_private simde_mm512_shufflehi_epi16_r_, \ + simde_mm512_shufflehi_epi16_a_ = simde__m512i_to_private((a)); \ + simde_mm512_shufflehi_epi16_r_.m128i[0] = simde_mm_shufflehi_epi16( \ + simde_mm512_shufflehi_epi16_a_.m128i[0], (imm8)); \ + simde_mm512_shufflehi_epi16_r_.m128i[1] = simde_mm_shufflehi_epi16( \ + simde_mm512_shufflehi_epi16_a_.m128i[1], (imm8)); \ + simde_mm512_shufflehi_epi16_r_.m128i[2] = simde_mm_shufflehi_epi16( \ + simde_mm512_shufflehi_epi16_a_.m128i[2], (imm8)); \ + simde_mm512_shufflehi_epi16_r_.m128i[3] = simde_mm_shufflehi_epi16( \ + simde_mm512_shufflehi_epi16_a_.m128i[3], (imm8)); \ + simde__m512i_from_private(simde_mm512_shufflehi_epi16_r_); \ + })) +#else +# define simde_mm512_shufflehi_epi16(a, imm8) \ + simde_x_mm512_set_m128i( \ + simde_mm_shufflehi_epi16(simde_mm512_extracti32x4_epi32((a), 3), (imm8)), \ + simde_mm_shufflehi_epi16(simde_mm512_extracti32x4_epi32((a), 2), (imm8)), \ + simde_mm_shufflehi_epi16(simde_mm512_extracti32x4_epi32((a), 1), (imm8)), \ + simde_mm_shufflehi_epi16(simde_mm512_extracti32x4_epi32((a), 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_shufflehi_epi16 + #define _mm512_shufflehi_epi16(a, imm8) simde_mm512_shufflehi_epi16(a, imm8) +#endif + +#if defined(SIMDE_X86_AVX512BW_NATIVE) +# define simde_mm512_shufflelo_epi16(a, imm8) _mm512_shufflelo_epi16(a, imm8) +#elif defined(SIMDE_STATEMENT_EXPR_) +# define simde_mm512_shufflelo_epi16(a, imm8) SIMDE_STATEMENT_EXPR_(({ \ + simde__m512i_private simde_mm512_shufflelo_epi16_r_, \ + simde_mm512_shufflelo_epi16_a_ = simde__m512i_to_private((a)); \ + simde_mm512_shufflelo_epi16_r_.m128i[0] = simde_mm_shufflelo_epi16( \ + simde_mm512_shufflelo_epi16_a_.m128i[0], (imm8)); \ + simde_mm512_shufflelo_epi16_r_.m128i[1] = simde_mm_shufflelo_epi16( \ + simde_mm512_shufflelo_epi16_a_.m128i[1], (imm8)); \ + simde_mm512_shufflelo_epi16_r_.m128i[2] = simde_mm_shufflelo_epi16( \ + simde_mm512_shufflelo_epi16_a_.m128i[2], (imm8)); \ + simde_mm512_shufflelo_epi16_r_.m128i[3] = simde_mm_shufflelo_epi16( \ + simde_mm512_shufflelo_epi16_a_.m128i[3], (imm8)); \ + simde__m512i_from_private(simde_mm512_shufflelo_epi16_r_); \ + })) +#else +# define simde_mm512_shufflelo_epi16(a, imm8) \ + simde_x_mm512_set_m128i( \ + simde_mm_shufflelo_epi16(simde_mm512_extracti32x4_epi32((a), 3), (imm8)), \ + simde_mm_shufflelo_epi16(simde_mm512_extracti32x4_epi32((a), 2), (imm8)), \ + simde_mm_shufflelo_epi16(simde_mm512_extracti32x4_epi32((a), 1), (imm8)), \ + simde_mm_shufflelo_epi16(simde_mm512_extracti32x4_epi32((a), 0), (imm8))) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_shufflelo_epi16 + #define _mm512_shufflelo_epi16(a, imm8) simde_mm512_shufflelo_epi16(a, imm8) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SHUFFLE_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sll.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sll.h new file mode 100644 index 00000000..18fbbb8c --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sll.h @@ -0,0 +1,247 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_SLL_H) +#define SIMDE_X86_AVX512_SLL_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" +#include "setzero.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_sll_epi16 (simde__m512i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_sll_epi16(a, count); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_sll_epi16(a_.m256i[i], count); + } + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + + uint64_t shift = HEDLEY_STATIC_CAST(uint64_t, count_.i64[0]); + if (shift > 15) + return simde_mm512_setzero_si512(); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i16 = a_.i16 << HEDLEY_STATIC_CAST(int16_t, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, a_.i16[i] << (shift)); + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_sll_epi16 + #define _mm512_sll_epi16(a, count) simde_mm512_sll_epi16(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_sll_epi16 (simde__m512i src, simde__mmask32 k, simde__m512i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX51BW_NATIVE) + return _mm512_mask_sll_epi16(src, k, a, count); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_sll_epi16(a, count)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sll_epi16 + #define _mm512_mask_sll_epi16(src, k, a, count) simde_mm512_mask_sll_epi16(src, k, a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_sll_epi16 (simde__mmask32 k, simde__m512i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_sll_epi16(k, a, count); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_sll_epi16(a, count)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_sll_epi16 + #define _mm512_maskz_sll_epi16(k, a, count) simde_mm512_maskz_sll_epi16(k, a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_sll_epi32 (simde__m512i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sll_epi32(a, count); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_sll_epi32(a_.m256i[i], count); + } + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + + uint64_t shift = HEDLEY_STATIC_CAST(uint64_t, count_.i64[0]); + if (shift > 31) + return simde_mm512_setzero_si512(); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i32 = a_.i32 << HEDLEY_STATIC_CAST(int32_t, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = HEDLEY_STATIC_CAST(int32_t, a_.i32[i] << (shift)); + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_sll_epi32 + #define _mm512_sll_epi32(a, count) simde_mm512_sll_epi32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_sll_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sll_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_sll_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sll_epi32 + #define _mm512_mask_sll_epi32(src, k, a, b) simde_mm512_mask_sll_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_sll_epi32(simde__mmask16 k, simde__m512i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_sll_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_sll_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_sll_epi32 + #define _mm512_maskz_sll_epi32(k, a, b) simde_mm512_maskz_sll_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_sll_epi64 (simde__m512i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sll_epi64(a, count); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_sll_epi64(a_.m256i[i], count); + } + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + + uint64_t shift = HEDLEY_STATIC_CAST(uint64_t, count_.i64[0]); + if (shift > 63) + return simde_mm512_setzero_si512(); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i64 = a_.i64 << HEDLEY_STATIC_CAST(int64_t, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = HEDLEY_STATIC_CAST(int64_t, a_.i64[i] << (shift)); + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_sll_epi64 + #define _mm512_sll_epi64(a, count) simde_mm512_sll_epi64(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_sll_epi64(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sll_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_sll_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sll_epi64 + #define _mm512_mask_sll_epi64(src, k, a, b) simde_mm512_mask_sll_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_sll_epi64(simde__mmask8 k, simde__m512i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_sll_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_sll_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_sll_epi64 + #define _mm512_maskz_sll_epi64(k, a, b) simde_mm512_maskz_sll_epi64(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SLL_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/slli.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/slli.h new file mode 100644 index 00000000..d2ad75b7 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/slli.h @@ -0,0 +1,179 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + * 2020 Christopher Moore + */ + +#if !defined(SIMDE_X86_AVX512_SLLI_H) +#define SIMDE_X86_AVX512_SLLI_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" +#include "setzero.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_slli_epi16 (simde__m512i a, const unsigned int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && (defined(HEDLEY_GCC_VERSION) && ((__GNUC__ == 5 && __GNUC_MINOR__ == 5) || (__GNUC__ == 6 && __GNUC_MINOR__ >= 4))) + simde__m512i r; + + SIMDE_CONSTIFY_16_(_mm512_slli_epi16, r, simde_mm512_setzero_si512(), imm8, a); + + return r; + #elif defined(SIMDE_X86_AVX512BW_NATIVE) + return SIMDE_BUG_IGNORE_SIGN_CONVERSION(_mm512_slli_epi16(a, imm8)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + if(imm8 < 16) + r_.i16 = HEDLEY_STATIC_CAST(__typeof__(r_.i16), (a_.i16 << HEDLEY_STATIC_CAST(int16_t, imm8))); + else + return simde_mm512_setzero_si512(); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (imm8 < 16) ? HEDLEY_STATIC_CAST(int16_t, a_.i16[i] << (imm8 & 0xff)) : 0; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_slli_epi16 + #define _mm512_slli_epi16(a, imm8) simde_mm512_slli_epi16(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_slli_epi32 (simde__m512i a, unsigned int imm8) { + /* I guess the restriction was added in 6.4, back-ported to 5.5, then + * removed (fixed) in 7? */ + #if defined(SIMDE_X86_AVX512F_NATIVE) && (defined(HEDLEY_GCC_VERSION) && ((__GNUC__ == 5 && __GNUC_MINOR__ == 5) || (__GNUC__ == 6 && __GNUC_MINOR__ >= 4))) + simde__m512i r; + + SIMDE_CONSTIFY_32_(_mm512_slli_epi32, r, simde_mm512_setzero_si512(), imm8, a); + + return r; + #elif defined(SIMDE_X86_AVX512F_NATIVE) + return SIMDE_BUG_IGNORE_SIGN_CONVERSION(_mm512_slli_epi32(a, imm8)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + /* The Intel Intrinsics Guide says that only the 8 LSBits of imm8 are + * used. In this case we should do "imm8 &= 0xff". However in + * practice all bits are used. */ + if (imm8 > 31) { + simde_memset(&r_, 0, sizeof(r_)); + } else { + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256i[0] = simde_mm256_slli_epi32(a_.m256i[0], HEDLEY_STATIC_CAST(int, imm8)); + r_.m256i[1] = simde_mm256_slli_epi32(a_.m256i[1], HEDLEY_STATIC_CAST(int, imm8)); + #elif defined(SIMDE_X86_SSE2_NATIVE) + r_.m128i[0] = simde_mm_slli_epi32(a_.m128i[0], HEDLEY_STATIC_CAST(int, imm8)); + r_.m128i[1] = simde_mm_slli_epi32(a_.m128i[1], HEDLEY_STATIC_CAST(int, imm8)); + r_.m128i[2] = simde_mm_slli_epi32(a_.m128i[2], HEDLEY_STATIC_CAST(int, imm8)); + r_.m128i[3] = simde_mm_slli_epi32(a_.m128i[3], HEDLEY_STATIC_CAST(int, imm8)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = a_.u32 << imm8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] << imm8; + } + #endif + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_slli_epi32 + #define _mm512_slli_epi32(a, imm8) simde_mm512_slli_epi32(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_slli_epi64 (simde__m512i a, unsigned int imm8) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && (defined(HEDLEY_GCC_VERSION) && ((__GNUC__ == 5 && __GNUC_MINOR__ == 5) || (__GNUC__ == 6 && __GNUC_MINOR__ >= 4))) + simde__m512i r; + + SIMDE_CONSTIFY_64_(_mm512_slli_epi64, r, simde_mm512_setzero_si512(), imm8, a); + + return r; + #elif defined(SIMDE_X86_AVX512F_NATIVE) + return SIMDE_BUG_IGNORE_SIGN_CONVERSION(_mm512_slli_epi64(a, imm8)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + /* The Intel Intrinsics Guide says that only the 8 LSBits of imm8 are + * used. In this case we should do "imm8 &= 0xff". However in + * practice all bits are used. */ + if (imm8 > 63) { + simde_memset(&r_, 0, sizeof(r_)); + } else { + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256i[0] = simde_mm256_slli_epi64(a_.m256i[0], HEDLEY_STATIC_CAST(int, imm8)); + r_.m256i[1] = simde_mm256_slli_epi64(a_.m256i[1], HEDLEY_STATIC_CAST(int, imm8)); + #elif defined(SIMDE_X86_SSE2_NATIVE) + r_.m128i[0] = simde_mm_slli_epi64(a_.m128i[0], HEDLEY_STATIC_CAST(int, imm8)); + r_.m128i[1] = simde_mm_slli_epi64(a_.m128i[1], HEDLEY_STATIC_CAST(int, imm8)); + r_.m128i[2] = simde_mm_slli_epi64(a_.m128i[2], HEDLEY_STATIC_CAST(int, imm8)); + r_.m128i[3] = simde_mm_slli_epi64(a_.m128i[3], HEDLEY_STATIC_CAST(int, imm8)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && !defined(SIMDE_BUG_GCC_97248) + r_.u64 = a_.u64 << imm8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = a_.u64[i] << imm8; + } + #endif + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_slli_epi64 + #define _mm512_slli_epi64(a, imm8) simde_mm512_slli_epi64(a, imm8) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SLLI_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sllv.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sllv.h new file mode 100644 index 00000000..f4caa6ee --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sllv.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_SLLV_H) +#define SIMDE_X86_AVX512_SLLV_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_sllv_epi16 (simde__m512i a, simde__m512i b) { + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b), + r_; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), (b_.u16 < 16)) & (a_.u16 << b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (b_.u16[i] < 16) ? HEDLEY_STATIC_CAST(uint16_t, (a_.u16[i] << b_.u16[i])) : 0; + } + #endif + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512BW_NATIVE) + #define simde_mm512_sllv_epi16(a, b) _mm512_sllv_epi16(a, b) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_sllv_epi16 + #define _mm512_sllv_epi16(a, b) simde_mm512_sllv_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_sllv_epi32 (simde__m512i a, simde__m512i b) { + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b), + r_; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), (b_.u32 < 32)) & (a_.u32 << b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (b_.u32[i] < 32) ? HEDLEY_STATIC_CAST(uint32_t, (a_.u32[i] << b_.u32[i])) : 0; + } + #endif + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_sllv_epi32(a, b) _mm512_sllv_epi32(a, b) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_sllv_epi32 + #define _mm512_sllv_epi32(a, b) simde_mm512_sllv_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_sllv_epi64 (simde__m512i a, simde__m512i b) { + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b), + r_; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), (b_.u64 < 64)) & (a_.u64 << b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (b_.u64[i] < 64) ? HEDLEY_STATIC_CAST(uint64_t, (a_.u64[i] << b_.u64[i])) : 0; + } + #endif + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_sllv_epi64(a, b) _mm512_sllv_epi64(a, b) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_sllv_epi64 + #define _mm512_sllv_epi64(a, b) simde_mm512_sllv_epi64(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SLLV_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sqrt.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sqrt.h new file mode 100644 index 00000000..cdc18ae5 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sqrt.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + */ + +#if !defined(SIMDE_X86_AVX512_SQRT_H) +#define SIMDE_X86_AVX512_SQRT_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_sqrt_ps (simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sqrt_ps(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if defined(SIMDE_X86_AVX_NATIVE) + r_.m256[0] = simde_mm256_sqrt_ps(a_.m256[0]); + r_.m256[1] = simde_mm256_sqrt_ps(a_.m256[1]); + #elif defined(simde_math_sqrtf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_sqrtf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) +# define _mm512_sqrt_ps(a) simde_mm512_sqrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_sqrt_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sqrt_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_sqrt_ps(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sqrt_ps + #define _mm512_mask_sqrt_ps(src, k, a) simde_mm512_mask_sqrt_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_sqrt_pd (simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sqrt_pd(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if defined(SIMDE_X86_AVX_NATIVE) + r_.m256d[0] = simde_mm256_sqrt_pd(a_.m256d[0]); + r_.m256d[1] = simde_mm256_sqrt_pd(a_.m256d[1]); + #elif defined(simde_math_sqrt) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_sqrt(a_.f64[i]); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) +# define _mm512_sqrt_pd(a) simde_mm512_sqrt_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_sqrt_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sqrt_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_sqrt_pd(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sqrt_pd + #define _mm512_mask_sqrt_pd(src, k, a) simde_mm512_mask_sqrt_pd(src, k, a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SQRT_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sra.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sra.h new file mode 100644 index 00000000..3a7512dd --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sra.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_SRA_H) +#define SIMDE_X86_AVX512_SRA_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_sra_epi16 (simde__m512i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_sra_epi16(a, count); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_sra_epi16(a_.m256i[i], count); + } + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + uint64_t shift = HEDLEY_STATIC_CAST(uint64_t, count_.i64[0]); + + if (shift > 15) shift = 15; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i16 = a_.i16 >> HEDLEY_STATIC_CAST(int16_t, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] >> shift; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_sra_epi16 + #define _mm512_sra_epi16(a, count) simde_mm512_sra_epi16(a, count) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SRA_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srai.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srai.h new file mode 100644 index 00000000..4fcbd95c --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srai.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_SRAI_H) +#define SIMDE_X86_AVX512_SRAI_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_srai_epi16 (simde__m512i a, const int imm8) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + unsigned int shift = HEDLEY_STATIC_CAST(unsigned int, imm8); + + if (shift > 15) shift = 15; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i16 = a_.i16 >> HEDLEY_STATIC_CAST(int16_t, shift); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] >> shift; + } + #endif + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512BW_NATIVE) +# define simde_mm512_srai_epi16(a, imm8) _mm512_srai_epi16(a, imm8) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_srai_epi16 + #define _mm512_srai_epi16(a, imm8) simde_mm512_srai_epi16(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_srai_epi32 (simde__m512i a, const unsigned int imm8) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i32 = a_.i32 >> HEDLEY_STATIC_CAST(int32_t, imm8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] >> imm8; + } + #endif + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_AVX512F_NATIVE) +# define simde_mm512_srai_epi32(a, imm8) _mm512_srai_epi32(a, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_srai_epi32 + #define _mm512_srai_epi32(a, imm8) simde_mm512_srai_epi32(a, imm8) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SRAI_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srav.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srav.h new file mode 100644 index 00000000..9c811f54 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srav.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_SRAV_H) +#define SIMDE_X86_AVX512_SRAV_H + +#include "types.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_srav_epi16 (simde__m512i a, simde__m512i count) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_srav_epi16(a, count); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + count_ = simde__m512i_to_private(count); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + uint32_t shift = HEDLEY_STATIC_CAST(uint32_t, count_.i16[i]); + if (shift > 16) shift = 15; + r_.i16[i] = a_.i16[i] >> shift; + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_srav_epi16 + #define _mm512_srav_epi16(a, count) simde_mm512_srav_epi16(a, count) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SRAV_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srl.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srl.h new file mode 100644 index 00000000..31e3fa17 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srl.h @@ -0,0 +1,216 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_SRL_H) +#define SIMDE_X86_AVX512_SRL_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" +#include "setzero.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_srl_epi16 (simde__m512i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_srl_epi16(a, count); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_srl_epi16(a_.m256i[i], count); + } + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + + if (HEDLEY_STATIC_CAST(uint64_t, count_.i64[0]) > 15) + return simde_mm512_setzero_si512(); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u16 = a_.u16 >> count_.i64[0]; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.u16[i] = a_.u16[i] >> count_.i64[0]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_srl_epi16 + #define _mm512_srl_epi16(a, count) simde_mm512_srl_epi16(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_srl_epi32 (simde__m512i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_srl_epi32(a, count); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_srl_epi32(a_.m256i[i], count); + } + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + + if (HEDLEY_STATIC_CAST(uint64_t, count_.i64[0]) > 31) + return simde_mm512_setzero_si512(); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = a_.u32 >> count_.i64[0]; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.u32[i] = a_.u32[i] >> count_.i64[0]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_srl_epi32 + #define _mm512_srl_epi32(a, count) simde_mm512_srl_epi32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_srl_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_srl_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_srl_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_srl_epi32 + #define _mm512_mask_srl_epi32(src, k, a, b) simde_mm512_mask_srl_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_srl_epi32(simde__mmask16 k, simde__m512i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_srl_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_srl_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_srl_epi32 + #define _mm512_maskz_srl_epi32(k, a, b) simde_mm512_maskz_srl_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_srl_epi64 (simde__m512i a, simde__m128i count) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_srl_epi64(a, count); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_srl_epi64(a_.m256i[i], count); + } + #else + simde__m128i_private + count_ = simde__m128i_to_private(count); + + if (HEDLEY_STATIC_CAST(uint64_t, count_.i64[0]) > 63) + return simde_mm512_setzero_si512(); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u64 = a_.u64 >> count_.i64[0]; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.u64[i] = a_.u64[i] >> count_.i64[0]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_srl_epi64 + #define _mm512_srl_epi64(a, count) simde_mm512_srl_epi64(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_srl_epi64(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_srl_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_srl_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_srl_epi64 + #define _mm512_mask_srl_epi64(src, k, a, b) simde_mm512_mask_srl_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_srl_epi64(simde__mmask8 k, simde__m512i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_srl_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_srl_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_srl_epi64 + #define _mm512_maskz_srl_epi64(k, a, b) simde_mm512_maskz_srl_epi64(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SRL_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srli.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srli.h new file mode 100644 index 00000000..f240693b --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srli.h @@ -0,0 +1,180 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_SRLI_H) +#define SIMDE_X86_AVX512_SRLI_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" +#include "setzero.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_srli_epi16 (simde__m512i a, const unsigned int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && (defined(HEDLEY_GCC_VERSION) && ((__GNUC__ == 5 && __GNUC_MINOR__ == 5) || (__GNUC__ == 6 && __GNUC_MINOR__ >= 4))) + simde__m512i r; + + SIMDE_CONSTIFY_16_(_mm512_srli_epi16, r, simde_mm512_setzero_si512(), imm8, a); + + return r; + #elif defined(SIMDE_X86_AVX512BW_NATIVE) + return SIMDE_BUG_IGNORE_SIGN_CONVERSION(_mm512_srli_epi16(a, imm8)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + if (HEDLEY_STATIC_CAST(unsigned int, imm8) > 15) + return simde_mm512_setzero_si512(); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u16 = a_.u16 >> SIMDE_CAST_VECTOR_SHIFT_COUNT(16, imm8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = a_.u16[i] >> imm8; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_NATIVE) + #define simde_mm512_srli_epi16(a, imm8) _mm512_srli_epi16(a, imm8) +#endif +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_srli_epi16 + #define _mm512_srli_epi16(a, imm8) simde_mm512_srli_epi16(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_srli_epi32 (simde__m512i a, unsigned int imm8) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && (defined(HEDLEY_GCC_VERSION) && ((__GNUC__ == 5 && __GNUC_MINOR__ == 5) || (__GNUC__ == 6 && __GNUC_MINOR__ >= 4))) + simde__m512i r; + + SIMDE_CONSTIFY_32_(_mm512_srli_epi32, r, simde_mm512_setzero_si512(), imm8, a); + + return r; + #elif defined(SIMDE_X86_AVX512F_NATIVE) + return SIMDE_BUG_IGNORE_SIGN_CONVERSION(_mm512_srli_epi32(a, imm8)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256i[0] = simde_mm256_srli_epi32(a_.m256i[0], HEDLEY_STATIC_CAST(int, imm8)); + r_.m256i[1] = simde_mm256_srli_epi32(a_.m256i[1], HEDLEY_STATIC_CAST(int, imm8)); + #elif defined(SIMDE_X86_SSE2_NATIVE) + r_.m128i[0] = simde_mm_srli_epi32(a_.m128i[0], HEDLEY_STATIC_CAST(int, imm8)); + r_.m128i[1] = simde_mm_srli_epi32(a_.m128i[1], HEDLEY_STATIC_CAST(int, imm8)); + r_.m128i[2] = simde_mm_srli_epi32(a_.m128i[2], HEDLEY_STATIC_CAST(int, imm8)); + r_.m128i[3] = simde_mm_srli_epi32(a_.m128i[3], HEDLEY_STATIC_CAST(int, imm8)); + #else + if (imm8 > 31) { + simde_memset(&r_, 0, sizeof(r_)); + } else { + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = a_.u32 >> imm8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] >> imm8; + } + #endif + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_srli_epi32 + #define _mm512_srli_epi32(a, imm8) simde_mm512_srli_epi32(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_srli_epi64 (simde__m512i a, unsigned int imm8) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && (defined(HEDLEY_GCC_VERSION) && ((__GNUC__ == 5 && __GNUC_MINOR__ == 5) || (__GNUC__ == 6 && __GNUC_MINOR__ >= 4))) + simde__m512i r; + + SIMDE_CONSTIFY_64_(_mm512_srli_epi64, r, simde_mm512_setzero_si512(), imm8, a); + + return r; + #elif defined(SIMDE_X86_AVX512F_NATIVE) + return SIMDE_BUG_IGNORE_SIGN_CONVERSION(_mm512_srli_epi64(a, imm8)); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a); + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256i[0] = simde_mm256_srli_epi64(a_.m256i[0], HEDLEY_STATIC_CAST(int, imm8)); + r_.m256i[1] = simde_mm256_srli_epi64(a_.m256i[1], HEDLEY_STATIC_CAST(int, imm8)); + #elif defined(SIMDE_X86_SSE2_NATIVE) + r_.m128i[0] = simde_mm_srli_epi64(a_.m128i[0], HEDLEY_STATIC_CAST(int, imm8)); + r_.m128i[1] = simde_mm_srli_epi64(a_.m128i[1], HEDLEY_STATIC_CAST(int, imm8)); + r_.m128i[2] = simde_mm_srli_epi64(a_.m128i[2], HEDLEY_STATIC_CAST(int, imm8)); + r_.m128i[3] = simde_mm_srli_epi64(a_.m128i[3], HEDLEY_STATIC_CAST(int, imm8)); + #else + /* The Intel Intrinsics Guide says that only the 8 LSBits of imm8 are + * used. In this case we should do "imm8 &= 0xff" here. However in + * practice all bits are used. */ + if (imm8 > 63) { + simde_memset(&r_, 0, sizeof(r_)); + } else { + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && !defined(SIMDE_BUG_GCC_97248) + r_.u64 = a_.u64 >> imm8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = a_.u64[i] >> imm8; + } + #endif + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_srli_epi64 + #define _mm512_srli_epi64(a, imm8) simde_mm512_srli_epi64(a, imm8) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SRLI_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srlv.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srlv.h new file mode 100644 index 00000000..7b7f7747 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/srlv.h @@ -0,0 +1,282 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_SRLV_H) +#define SIMDE_X86_AVX512_SRLV_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_srlv_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_srlv_epi16(a, b); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + r_; + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), (b_.u16 < 16)) & (a_.u16 >> b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (b_.u16[i] < 16) ? (a_.u16[i] >> b_.u16[i]) : 0; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm_srlv_epi16 + #define _mm_srlv_epi16(a, b) simde_mm_srlv_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_srlv_epi16(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_mask_srlv_epi16(src, k, a, b); + #else + return simde_mm_mask_mov_epi16(src, k, simde_mm_srlv_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_srlv_epi16 + #define _mm_mask_srlv_epi16(src, k, a, b) simde_mm_mask_srlv_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_srlv_epi16(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm_maskz_srlv_epi16(k, a, b); + #else + return simde_mm_maskz_mov_epi16(k, simde_mm_srlv_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_srlv_epi16 + #define _mm_maskz_srlv_epi16(k, a, b) simde_mm_maskz_srlv_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_srlv_epi32(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_srlv_epi32(src, k, a, b); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_srlv_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_srlv_epi32 + #define _mm_mask_srlv_epi32(src, k, a, b) simde_mm_mask_srlv_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_srlv_epi32(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_srlv_epi32(k, a, b); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_srlv_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_srlv_epi32 + #define _mm_maskz_srlv_epi32(k, a, b) simde_mm_maskz_srlv_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_srlv_epi64(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_srlv_epi64(src, k, a, b); + #else + return simde_mm_mask_mov_epi64(src, k, simde_mm_srlv_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_srlv_epi64 + #define _mm_mask_srlv_epi64(src, k, a, b) simde_mm_mask_srlv_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_srlv_epi64(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_srlv_epi64(k, a, b); + #else + return simde_mm_maskz_mov_epi64(k, simde_mm_srlv_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_srlv_epi64 + #define _mm_maskz_srlv_epi64(k, a, b) simde_mm_maskz_srlv_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_srlv_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm256_srlv_epi16(a, b); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b), + r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_srlv_epi16(a_.m128i[i], b_.m128i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), (b_.u16 < 16)) & (a_.u16 >> b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (b_.u16[i] < 16) ? (a_.u16[i] >> b_.u16[i]) : 0; + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm256_srlv_epi16 + #define _mm256_srlv_epi16(a, b) simde_mm256_srlv_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_srlv_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_srlv_epi16(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b), + r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_srlv_epi16(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), (b_.u16 < 16)) & (a_.u16 >> b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (b_.u16[i] < 16) ? (a_.u16[i] >> b_.u16[i]) : 0; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_srlv_epi16 + #define _mm512_srlv_epi16(a, b) simde_mm512_srlv_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_srlv_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_srlv_epi32(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b), + r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_srlv_epi32(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), (b_.u32 < 32)) & (a_.u32 >> b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (b_.u32[i] < 32) ? (a_.u32[i] >> b_.u32[i]) : 0; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_srlv_epi32 + #define _mm512_srlv_epi32(a, b) simde_mm512_srlv_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_srlv_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_srlv_epi64(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b), + r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_srlv_epi64(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), (b_.u64 < 64)) & (a_.u64 >> b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (b_.u64[i] < 64) ? (a_.u64[i] >> b_.u64[i]) : 0; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_srlv_epi64 + #define _mm512_srlv_epi64(a, b) simde_mm512_srlv_epi64(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SRLV_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/store.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/store.h new file mode 100644 index 00000000..1f1538bc --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/store.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_STORE_H) +#define SIMDE_X86_AVX512_STORE_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_store_ps (void * mem_addr, simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + _mm512_store_ps(mem_addr, a); + #else + simde_memcpy(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m512), &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_store_ps + #define _mm512_store_ps(mem_addr, a) simde_mm512_store_ps(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_store_pd (void * mem_addr, simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + _mm512_store_pd(mem_addr, a); + #else + simde_memcpy(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m512d), &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_store_pd + #define _mm512_store_pd(mem_addr, a) simde_mm512_store_pd(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_store_si512 (void * mem_addr, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + _mm512_store_si512(HEDLEY_REINTERPRET_CAST(void*, mem_addr), a); + #else + simde_memcpy(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m512i), &a, sizeof(a)); + #endif +} +#define simde_mm512_store_epi8(mem_addr, a) simde_mm512_store_si512(mem_addr, a) +#define simde_mm512_store_epi16(mem_addr, a) simde_mm512_store_si512(mem_addr, a) +#define simde_mm512_store_epi32(mem_addr, a) simde_mm512_store_si512(mem_addr, a) +#define simde_mm512_store_epi64(mem_addr, a) simde_mm512_store_si512(mem_addr, a) +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_store_epi8 + #undef _mm512_store_epi16 + #undef _mm512_store_epi32 + #undef _mm512_store_epi64 + #undef _mm512_store_si512 + #define _mm512_store_si512(mem_addr, a) simde_mm512_store_si512(mem_addr, a) + #define _mm512_store_epi8(mem_addr, a) simde_mm512_store_si512(mem_addr, a) + #define _mm512_store_epi16(mem_addr, a) simde_mm512_store_si512(mem_addr, a) + #define _mm512_store_epi32(mem_addr, a) simde_mm512_store_si512(mem_addr, a) + #define _mm512_store_epi64(mem_addr, a) simde_mm512_store_si512(mem_addr, a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_STORE_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/storeu.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/storeu.h new file mode 100644 index 00000000..e00801fa --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/storeu.h @@ -0,0 +1,218 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_STOREU_H) +#define SIMDE_X86_AVX512_STOREU_H + +#include "types.h" +#include "mov.h" +#include "setzero.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#define simde_mm256_storeu_epi8(mem_addr, a) simde_mm256_storeu_si256(mem_addr, a) +#define simde_mm256_storeu_epi16(mem_addr, a) simde_mm256_storeu_si256(mem_addr, a) +#define simde_mm256_storeu_epi32(mem_addr, a) simde_mm256_storeu_si256(mem_addr, a) +#define simde_mm256_storeu_epi64(mem_addr, a) simde_mm256_storeu_si256(mem_addr, a) +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_storeu_epi8 + #undef _mm256_storeu_epi16 + #define _mm256_storeu_epi8(mem_addr, a) simde_mm256_storeu_si256(mem_addr, a) + #define _mm256_storeu_epi16(mem_addr, a) simde_mm256_storeu_si256(mem_addr, a) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_storeu_epi32 + #undef _mm256_storeu_epi64 + #define _mm256_storeu_epi32(mem_addr, a) simde_mm256_storeu_si256(mem_addr, a) + #define _mm256_storeu_epi64(mem_addr, a) simde_mm256_storeu_si256(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm256_mask_storeu_epi16 (void * mem_addr, simde__mmask16 k, simde__m256i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + _mm256_mask_storeu_epi16(HEDLEY_REINTERPRET_CAST(void*, mem_addr), k, a); + #else + const simde__m256i zero = simde_mm256_setzero_si256(); + simde_mm256_storeu_epi16(mem_addr, simde_mm256_mask_mov_epi16(zero, k, a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_storeu_epi16 + #define _mm256_mask_storeu_epi16(mem_addr, k, a) simde_mm256_mask_storeu_epi16(mem_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_storeu_ps (void * mem_addr, simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + _mm512_storeu_ps(mem_addr, a); + #else + simde_memcpy(mem_addr, &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_storeu_ps + #define _mm512_storeu_ps(mem_addr, a) simde_mm512_storeu_ps(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_storeu_pd (void * mem_addr, simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + _mm512_storeu_pd(mem_addr, a); + #else + simde_memcpy(mem_addr, &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_storeu_pd + #define _mm512_storeu_pd(mem_addr, a) simde_mm512_storeu_pd(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_storeu_ph (void * mem_addr, simde__m512h a) { + #if defined(SIMDE_X86_AVX512FP16_NATIVE) + _mm512_storeu_ph(mem_addr, a); + #else + simde_memcpy(mem_addr, &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_AVX512FP16_ENABLE_NATIVE_ALIASES) + #undef _mm512_storeu_ph + #define _mm512_storeu_ph(mem_addr, a) simde_mm512_storeu_ph(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_storeu_si512 (void * mem_addr, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + _mm512_storeu_si512(HEDLEY_REINTERPRET_CAST(void*, mem_addr), a); + #else + simde_memcpy(mem_addr, &a, sizeof(a)); + #endif +} +#define simde_mm512_storeu_epi8(mem_addr, a) simde_mm512_storeu_si512(mem_addr, a) +#define simde_mm512_storeu_epi16(mem_addr, a) simde_mm512_storeu_si512(mem_addr, a) +#define simde_mm512_storeu_epi32(mem_addr, a) simde_mm512_storeu_si512(mem_addr, a) +#define simde_mm512_storeu_epi64(mem_addr, a) simde_mm512_storeu_si512(mem_addr, a) +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_storeu_epi8 + #undef _mm512_storeu_epi16 + #define _mm512_storeu_epi16(mem_addr, a) simde_mm512_storeu_si512(mem_addr, a) + #define _mm512_storeu_epi8(mem_addr, a) simde_mm512_storeu_si512(mem_addr, a) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_storeu_epi32 + #undef _mm512_storeu_epi64 + #undef _mm512_storeu_si512 + #define _mm512_storeu_si512(mem_addr, a) simde_mm512_storeu_si512(mem_addr, a) + #define _mm512_storeu_epi32(mem_addr, a) simde_mm512_storeu_si512(mem_addr, a) + #define _mm512_storeu_epi64(mem_addr, a) simde_mm512_storeu_si512(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_mask_storeu_epi16 (void * mem_addr, simde__mmask32 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + _mm512_mask_storeu_epi16(HEDLEY_REINTERPRET_CAST(void*, mem_addr), k, a); + #else + const simde__m512i zero = simde_mm512_setzero_si512(); + simde_mm512_storeu_epi16(mem_addr, simde_mm512_mask_mov_epi16(zero, k, a)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_storeu_epi16 + #define _mm512_mask_storeu_epi16(mem_addr, k, a) simde_mm512_mask_storeu_epi16(mem_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_mask_storeu_epi32 (void * mem_addr, simde__mmask16 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + _mm512_mask_storeu_epi32(HEDLEY_REINTERPRET_CAST(void*, mem_addr), k, a); + #else + const simde__m512i zero = simde_mm512_setzero_si512(); + simde_mm512_storeu_epi32(mem_addr, simde_mm512_mask_mov_epi32(zero, k, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_storeu_epi32 + #define _mm512_mask_storeu_epi32(mem_addr, k, a) simde_mm512_mask_storeu_epi32(mem_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_mask_storeu_epi64 (void * mem_addr, simde__mmask8 k, simde__m512i a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + _mm512_mask_storeu_epi64(HEDLEY_REINTERPRET_CAST(void*, mem_addr), k, a); + #else + const simde__m512i zero = simde_mm512_setzero_si512(); + simde_mm512_storeu_epi64(mem_addr, simde_mm512_mask_mov_epi64(zero, k, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_storeu_epi64 + #define _mm512_mask_storeu_epi64(mem_addr, k, a) simde_mm512_mask_storeu_epi64(mem_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_mask_storeu_ps (void * mem_addr, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + _mm512_mask_storeu_ps(HEDLEY_REINTERPRET_CAST(void*, mem_addr), k, a); + #else + const simde__m512 zero = simde_mm512_setzero_ps(); + simde_mm512_storeu_ps(mem_addr, simde_mm512_mask_mov_ps(zero, k, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_storeu_ps + #define _mm512_mask_storeu_ps(mem_addr, k, a) simde_mm512_mask_storeu_ps(mem_addr, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm512_mask_storeu_pd (void * mem_addr, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + _mm512_mask_storeu_pd(HEDLEY_REINTERPRET_CAST(void*, mem_addr), k, a); + #else + const simde__m512d zero = simde_mm512_setzero_pd(); + simde_mm512_storeu_pd(mem_addr, simde_mm512_mask_mov_pd(zero, k, a)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_storeu_pd + #define _mm512_mask_storeu_pd(mem_addr, k, a) simde_mm512_mask_storeu_pd(mem_addr, k, a) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_STOREU_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sub.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sub.h new file mode 100644 index 00000000..6e44d85a --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/sub.h @@ -0,0 +1,351 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_SUB_H) +#define SIMDE_X86_AVX512_SUB_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_sub_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_sub_epi8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = a_.i8 - b_.i8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_sub_epi8(a_.m256i[i], b_.m256i[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_sub_epi8 + #define _mm512_sub_epi8(a, b) simde_mm512_sub_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_sub_epi8 (simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_sub_epi8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_sub_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sub_epi8 + #define _mm512_mask_sub_epi8(src, k, a, b) simde_mm512_mask_sub_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_sub_epi8 (simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_sub_epi8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_sub_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_sub_epi8 + #define _mm512_maskz_sub_epi8(k, a, b) simde_mm512_maskz_sub_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_sub_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_sub_epi16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = a_.i16 - b_.i16; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_sub_epi16(a_.m256i[i], b_.m256i[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_sub_epi16 + #define _mm512_sub_epi16(a, b) simde_mm512_sub_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_sub_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sub_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 - b_.i32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_sub_epi32(a_.m256i[i], b_.m256i[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_sub_epi32 + #define _mm512_sub_epi32(a, b) simde_mm512_sub_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_sub_epi32 (simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sub_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_sub_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sub_epi32 + #define _mm512_mask_sub_epi32(src, k, a, b) simde_mm512_mask_sub_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_sub_epi32(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_sub_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_sub_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_sub_epi32 + #define _mm512_maskz_sub_epi32(k, a, b) simde_mm512_maskz_sub_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_sub_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sub_epi64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 - b_.i64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_sub_epi64(a_.m256i[i], b_.m256i[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_sub_epi64 + #define _mm512_sub_epi64(a, b) simde_mm512_sub_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_sub_epi64 (simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sub_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_sub_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sub_epi64 + #define _mm512_mask_sub_epi64(src, k, a, b) simde_mm512_mask_sub_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_sub_epi64(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_sub_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_sub_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_sub_epi64 + #define _mm512_maskz_sub_epi64(k, a, b) simde_mm512_maskz_sub_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_sub_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sub_ps(a, b); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = a_.f32 - b_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_sub_ps(a_.m256[i], b_.m256[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_sub_ps + #define _mm512_sub_ps(a, b) simde_mm512_sub_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_sub_ps (simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sub_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_sub_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sub_ps + #define _mm512_mask_sub_ps(src, k, a, b) simde_mm512_mask_sub_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_sub_ps(simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_sub_ps(k, a, b); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_sub_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_sub_ps + #define _mm512_maskz_sub_ps(k, a, b) simde_mm512_maskz_sub_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_sub_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sub_pd(a, b); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = a_.f64 - b_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_sub_pd(a_.m256d[i], b_.m256d[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_sub_pd + #define _mm512_sub_pd(a, b) simde_mm512_sub_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_sub_pd (simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sub_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_sub_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sub_pd + #define _mm512_mask_sub_pd(src, k, a, b) simde_mm512_mask_sub_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_sub_pd(simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_sub_pd(k, a, b); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_sub_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_sub_pd + #define _mm512_maskz_sub_pd(k, a, b) simde_mm512_maskz_sub_pd(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SUB_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/subs.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/subs.h new file mode 100644 index 00000000..114ecf1c --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/subs.h @@ -0,0 +1,222 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_SUBS_H) +#define SIMDE_X86_AVX512_SUBS_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_subs_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_subs_epi8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if !defined(HEDLEY_INTEL_VERSION) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_subs_epi8(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = simde_math_subs_i8(a_.i8[i], b_.i8[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_subs_epi8 + #define _mm512_subs_epi8(a, b) simde_mm512_subs_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_subs_epi8 (simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_subs_epi8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_subs_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_subs_epi8 + #define _mm512_mask_subs_epi8(src, k, a, b) simde_mm512_mask_subs_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_subs_epi8 (simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_subs_epi8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_subs_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_subs_epi8 + #define _mm512_maskz_subs_epi8(k, a, b) simde_mm512_maskz_subs_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_subs_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_subs_epi16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if !defined(HEDLEY_INTEL_VERSION) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_subs_epi16(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = simde_math_subs_i16(a_.i16[i], b_.i16[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_subs_epi16 + #define _mm512_subs_epi16(a, b) simde_mm512_subs_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_subs_epu8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_subs_epu8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if !defined(HEDLEY_INTEL_VERSION) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_subs_epu8(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = simde_math_subs_u8(a_.u8[i], b_.u8[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_subs_epu8 + #define _mm512_subs_epu8(a, b) simde_mm512_subs_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_subs_epu8 (simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_subs_epu8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_subs_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_subs_epu8 + #define _mm512_mask_subs_epu8(src, k, a, b) simde_mm512_mask_subs_epu8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_subs_epu8 (simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_subs_epu8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_subs_epu8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_subs_epu8 + #define _mm512_maskz_subs_epu8(k, a, b) simde_mm512_maskz_subs_epu8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_subs_epu16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_subs_epu16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if !defined(HEDLEY_INTEL_VERSION) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_subs_epu16(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = simde_math_subs_u16(a_.u16[i], b_.u16[i]); + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_subs_epu16 + #define _mm512_subs_epu16(a, b) simde_mm512_subs_epu16(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_SUBS_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/ternarylogic.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/ternarylogic.h new file mode 100644 index 00000000..c9a2f67c --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/ternarylogic.h @@ -0,0 +1,3769 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2021 Kunwar Maheep Singh + * 2021 Christopher Moore + */ + +/* The ternarylogic implementation is based on Wojciech Muła's work at + * https://github.com/WojciechMula/ternary-logic */ + +#if !defined(SIMDE_X86_AVX512_TERNARYLOGIC_H) +#define SIMDE_X86_AVX512_TERNARYLOGIC_H + +#include "types.h" +#include "movm.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x00_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + HEDLEY_STATIC_CAST(void, b); + HEDLEY_STATIC_CAST(void, c); + const uint_fast32_t c0 = 0; + return c0; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x01_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = a | t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x02_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | a; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = c & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x03_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, c); + const uint_fast32_t t0 = b | a; + const uint_fast32_t t1 = ~t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x04_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a | c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = b & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x05_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, b); + const uint_fast32_t t0 = c | a; + const uint_fast32_t t1 = ~t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x06_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = b ^ c; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x07_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = a | t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x08_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 & b; + const uint_fast32_t t2 = t1 & c; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x09_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = a | t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x0a_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, b); + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = c & t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x0b_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = ~b; + const uint_fast32_t t2 = t1 | c; + const uint_fast32_t t3 = t0 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x0c_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, c); + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = b & t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x0d_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = ~c; + const uint_fast32_t t2 = t1 | b; + const uint_fast32_t t3 = t0 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x0e_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = b | c; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x0f_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, b); + HEDLEY_STATIC_CAST(void, c); + const uint_fast32_t t0 = ~a; + return t0; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x10_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x11_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + const uint_fast32_t t0 = c | b; + const uint_fast32_t t1 = ~t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x12_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = a ^ c; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x13_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = b | t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x14_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = a ^ b; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x15_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & a; + const uint_fast32_t t1 = c | t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x16_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a & t1; + const uint_fast32_t t3 = ~a; + const uint_fast32_t t4 = b ^ c; + const uint_fast32_t t5 = t3 & t4; + const uint_fast32_t t6 = t2 | t5; + return t6; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x17_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = b & c; + const uint_fast32_t t2 = (a & t0) | (~a & t1); + const uint_fast32_t t3 = ~t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x18_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ b; + const uint_fast32_t t1 = a ^ c; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x19_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = b & c; + const uint_fast32_t t2 = a & t1; + const uint_fast32_t t3 = t0 ^ t2; + const uint_fast32_t t4 = ~t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x1a_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & b; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a ^ c; + const uint_fast32_t t3 = t1 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x1b_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = ~b; + const uint_fast32_t t2 = t1 | c; + const uint_fast32_t t3 = t0 ^ t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x1c_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a ^ b; + const uint_fast32_t t3 = t1 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x1d_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & b; + const uint_fast32_t t1 = ~c; + const uint_fast32_t t2 = t1 | b; + const uint_fast32_t t3 = t0 ^ t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x1e_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = a ^ t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x1f_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = a & t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x20_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 & a; + const uint_fast32_t t2 = t1 & c; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x21_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = b | t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x22_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = c & t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x23_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = ~a; + const uint_fast32_t t2 = t1 | c; + const uint_fast32_t t3 = t0 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x24_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ b; + const uint_fast32_t t1 = b ^ c; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x25_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & b; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~c; + const uint_fast32_t t3 = a ^ t2; + const uint_fast32_t t4 = t1 & t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x26_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & b; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = b ^ c; + const uint_fast32_t t3 = t1 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x27_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = ~a; + const uint_fast32_t t2 = t1 | c; + const uint_fast32_t t3 = t0 ^ t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x28_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ a; + const uint_fast32_t t1 = c & t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x29_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 | c; + const uint_fast32_t t2 = ~a; + const uint_fast32_t t3 = b ^ c; + const uint_fast32_t t4 = t2 ^ t3; + const uint_fast32_t t5 = t1 & t4; + return t5; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x2a_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & a; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = c & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x2b_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & a; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = c & t1; + const uint_fast32_t t3 = ~c; + const uint_fast32_t t4 = b | a; + const uint_fast32_t t5 = ~t4; + const uint_fast32_t t6 = t3 & t5; + const uint_fast32_t t7 = t2 | t6; + return t7; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x2c_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = a ^ b; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x2d_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = b | t0; + const uint_fast32_t t2 = a ^ t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x2e_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = a & b; + const uint_fast32_t t2 = t0 ^ t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x2f_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = ~b; + const uint_fast32_t t2 = t1 & c; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x30_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, c); + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = a & t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x31_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = ~c; + const uint_fast32_t t2 = t1 | a; + const uint_fast32_t t3 = t0 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x32_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = a | c; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x33_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + HEDLEY_STATIC_CAST(void, c); + const uint_fast32_t t0 = ~b; + return t0; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x34_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a ^ b; + const uint_fast32_t t3 = t1 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x35_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & b; + const uint_fast32_t t1 = ~c; + const uint_fast32_t t2 = t1 | a; + const uint_fast32_t t3 = t0 ^ t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x36_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a | c; + const uint_fast32_t t1 = b ^ t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x37_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a | c; + const uint_fast32_t t1 = b & t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x38_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a | c; + const uint_fast32_t t1 = a ^ b; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x39_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = t0 | a; + const uint_fast32_t t2 = b ^ t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x3a_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = a & t0; + const uint_fast32_t t2 = ~a; + const uint_fast32_t t3 = t2 & c; + const uint_fast32_t t4 = t1 | t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x3b_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = ~a; + const uint_fast32_t t2 = t1 & c; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x3c_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, c); + const uint_fast32_t t0 = b ^ a; + return t0; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x3d_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ b; + const uint_fast32_t t1 = a | c; + const uint_fast32_t t2 = ~t1; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x3e_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 & c; + const uint_fast32_t t2 = a ^ b; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x3f_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, c); + const uint_fast32_t t0 = b & a; + const uint_fast32_t t1 = ~t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x40_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = t0 & a; + const uint_fast32_t t2 = t1 & b; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x41_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ a; + const uint_fast32_t t1 = c | t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x42_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = b ^ c; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x43_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~b; + const uint_fast32_t t3 = a ^ t2; + const uint_fast32_t t4 = t1 & t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x44_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = b & t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x45_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = ~a; + const uint_fast32_t t2 = t1 | b; + const uint_fast32_t t3 = t0 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x46_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = b ^ c; + const uint_fast32_t t3 = t1 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x47_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = ~a; + const uint_fast32_t t2 = t1 | b; + const uint_fast32_t t3 = t0 ^ t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x48_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = b & t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x49_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 | b; + const uint_fast32_t t2 = ~a; + const uint_fast32_t t3 = b ^ c; + const uint_fast32_t t4 = t2 ^ t3; + const uint_fast32_t t5 = t1 & t4; + return t5; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x4a_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = a ^ c; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x4b_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 | c; + const uint_fast32_t t2 = a ^ t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x4c_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = b & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x4d_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = b & t1; + const uint_fast32_t t3 = ~b; + const uint_fast32_t t4 = a | c; + const uint_fast32_t t5 = ~t4; + const uint_fast32_t t6 = t3 & t5; + const uint_fast32_t t7 = t2 | t6; + return t7; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x4e_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = c & t0; + const uint_fast32_t t2 = ~c; + const uint_fast32_t t3 = t2 & b; + const uint_fast32_t t4 = t1 | t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x4f_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = ~c; + const uint_fast32_t t2 = b & t1; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x50_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, b); + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = a & t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x51_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = ~b; + const uint_fast32_t t2 = t1 | a; + const uint_fast32_t t3 = t0 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x52_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a ^ c; + const uint_fast32_t t3 = t1 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x53_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = ~b; + const uint_fast32_t t2 = t1 | a; + const uint_fast32_t t3 = t0 ^ t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x54_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = a | b; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x55_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + HEDLEY_STATIC_CAST(void, b); + const uint_fast32_t t0 = ~c; + return t0; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x56_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | a; + const uint_fast32_t t1 = c ^ t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x57_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | a; + const uint_fast32_t t1 = c & t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x58_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a | b; + const uint_fast32_t t1 = a ^ c; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x59_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 | a; + const uint_fast32_t t2 = c ^ t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x5a_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, b); + const uint_fast32_t t0 = c ^ a; + return t0; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x5b_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a | b; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a ^ c; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x5c_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = a & t0; + const uint_fast32_t t2 = ~a; + const uint_fast32_t t3 = t2 & b; + const uint_fast32_t t4 = t1 | t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x5d_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = ~a; + const uint_fast32_t t2 = t1 & b; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x5e_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = t0 & b; + const uint_fast32_t t2 = a ^ c; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x5f_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, b); + const uint_fast32_t t0 = c & a; + const uint_fast32_t t1 = ~t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x60_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = a & t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x61_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 | a; + const uint_fast32_t t2 = ~b; + const uint_fast32_t t3 = a ^ c; + const uint_fast32_t t4 = t2 ^ t3; + const uint_fast32_t t5 = t1 & t4; + return t5; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x62_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a | c; + const uint_fast32_t t1 = b ^ c; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x63_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 | c; + const uint_fast32_t t2 = b ^ t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x64_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a | b; + const uint_fast32_t t1 = b ^ c; + const uint_fast32_t t2 = t0 & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x65_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 | b; + const uint_fast32_t t2 = c ^ t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x66_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + const uint_fast32_t t0 = c ^ b; + return t0; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x67_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = a | b; + const uint_fast32_t t2 = ~t1; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x68_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = a & t0; + const uint_fast32_t t2 = ~a; + const uint_fast32_t t3 = b & c; + const uint_fast32_t t4 = t2 & t3; + const uint_fast32_t t5 = t1 | t4; + return t5; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x69_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = a ^ t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x6a_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & a; + const uint_fast32_t t1 = c ^ t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x6b_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 & c; + const uint_fast32_t c1 = ~HEDLEY_STATIC_CAST(uint_fast32_t, 0); + const uint_fast32_t t2 = a ^ c1; + const uint_fast32_t t3 = b ^ c; + const uint_fast32_t t4 = t2 ^ t3; + const uint_fast32_t t5 = t1 | t4; + return t5; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x6c_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = b ^ t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x6d_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 & b; + const uint_fast32_t c1 = ~HEDLEY_STATIC_CAST(uint_fast32_t, 0); + const uint_fast32_t t2 = a ^ c1; + const uint_fast32_t t3 = b ^ c; + const uint_fast32_t t4 = t2 ^ t3; + const uint_fast32_t t5 = t1 | t4; + return t5; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x6e_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 & b; + const uint_fast32_t t2 = b ^ c; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x6f_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = b ^ c; + const uint_fast32_t t2 = t0 | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x70_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x71_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = b ^ c; + const uint_fast32_t t3 = a & t2; + const uint_fast32_t t4 = t1 | t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x72_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = c & t0; + const uint_fast32_t t2 = ~c; + const uint_fast32_t t3 = t2 & a; + const uint_fast32_t t4 = t1 | t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x73_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = ~c; + const uint_fast32_t t2 = a & t1; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x74_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = b & t0; + const uint_fast32_t t2 = ~b; + const uint_fast32_t t3 = t2 & a; + const uint_fast32_t t4 = t1 | t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x75_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = ~b; + const uint_fast32_t t2 = a & t1; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x76_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 & a; + const uint_fast32_t t2 = b ^ c; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x77_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + const uint_fast32_t t0 = c & b; + const uint_fast32_t t1 = ~t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x78_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = a ^ t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x79_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 & a; + const uint_fast32_t c1 = ~HEDLEY_STATIC_CAST(uint_fast32_t, 0); + const uint_fast32_t t2 = b ^ c1; + const uint_fast32_t t3 = a ^ c; + const uint_fast32_t t4 = t2 ^ t3; + const uint_fast32_t t5 = t1 | t4; + return t5; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x7a_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 & a; + const uint_fast32_t t2 = a ^ c; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x7b_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = a ^ c; + const uint_fast32_t t2 = t0 | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x7c_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = t0 & a; + const uint_fast32_t t2 = a ^ b; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x7d_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = a ^ b; + const uint_fast32_t t2 = t0 | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x7e_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ b; + const uint_fast32_t t1 = a ^ c; + const uint_fast32_t t2 = t0 | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x7f_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & b; + const uint_fast32_t t1 = t0 & c; + const uint_fast32_t c1 = ~HEDLEY_STATIC_CAST(uint_fast32_t, 0); + const uint_fast32_t t2 = t1 ^ c1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x80_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = a & t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x81_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~b; + const uint_fast32_t t3 = a ^ t2; + const uint_fast32_t t4 = t1 & t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x82_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ a; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = c & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x83_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ b; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~a; + const uint_fast32_t t3 = t2 | c; + const uint_fast32_t t4 = t1 & t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x84_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = b & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x85_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~c; + const uint_fast32_t t3 = t2 | b; + const uint_fast32_t t4 = t1 & t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x86_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = a ^ b; + const uint_fast32_t t2 = c ^ t1; + const uint_fast32_t t3 = t0 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x87_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = a ^ t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x88_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + const uint_fast32_t t0 = c & b; + return t0; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x89_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~a; + const uint_fast32_t t3 = t2 | b; + const uint_fast32_t t4 = t1 & t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x8a_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 | b; + const uint_fast32_t t2 = c & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x8b_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 | b; + const uint_fast32_t t2 = ~b; + const uint_fast32_t t3 = t2 | c; + const uint_fast32_t t4 = t1 & t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x8c_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 | c; + const uint_fast32_t t2 = b & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x8d_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = t0 | b; + const uint_fast32_t t2 = ~a; + const uint_fast32_t t3 = t2 | c; + const uint_fast32_t t4 = t1 & t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x8e_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = ~a; + const uint_fast32_t t2 = b ^ c; + const uint_fast32_t t3 = t1 & t2; + const uint_fast32_t t4 = t0 | t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x8f_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = b & c; + const uint_fast32_t t2 = t0 | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x90_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x91_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~b; + const uint_fast32_t t3 = t2 | a; + const uint_fast32_t t4 = t1 & t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x92_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a | c; + const uint_fast32_t t1 = a ^ b; + const uint_fast32_t t2 = c ^ t1; + const uint_fast32_t t3 = t0 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x93_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = b ^ t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x94_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a | b; + const uint_fast32_t t1 = a ^ c; + const uint_fast32_t t2 = b ^ t1; + const uint_fast32_t t3 = t0 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x95_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & a; + const uint_fast32_t t1 = c ^ t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x96_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = a ^ t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x97_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 | a; + const uint_fast32_t t2 = t1 ^ a; + const uint_fast32_t t3 = b ^ c; + const uint_fast32_t t4 = a ^ t3; + const uint_fast32_t t5 = t2 | t4; + return t5; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x98_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a | b; + const uint_fast32_t t3 = t1 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x99_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + const uint_fast32_t t0 = c ^ b; + const uint_fast32_t t1 = ~t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x9a_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 & a; + const uint_fast32_t t2 = t1 ^ c; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x9b_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~a; + const uint_fast32_t t3 = t2 & c; + const uint_fast32_t t4 = t1 | t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x9c_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = t0 & a; + const uint_fast32_t t2 = t1 ^ b; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x9d_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~a; + const uint_fast32_t t3 = t2 & b; + const uint_fast32_t t4 = t1 | t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x9e_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = a ^ b; + const uint_fast32_t t2 = c ^ t1; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0x9f_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = a & t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xa0_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, b); + const uint_fast32_t t0 = c & a; + return t0; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xa1_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~b; + const uint_fast32_t t3 = t2 | a; + const uint_fast32_t t4 = t1 & t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xa2_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = a | t0; + const uint_fast32_t t2 = c & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xa3_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 | a; + const uint_fast32_t t2 = ~a; + const uint_fast32_t t3 = t2 | c; + const uint_fast32_t t4 = t1 & t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xa4_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a | b; + const uint_fast32_t t3 = t1 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xa5_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, b); + const uint_fast32_t t0 = c ^ a; + const uint_fast32_t t1 = ~t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xa6_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 & b; + const uint_fast32_t t2 = t1 ^ c; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xa7_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~b; + const uint_fast32_t t3 = t2 & c; + const uint_fast32_t t4 = t1 | t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xa8_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a | b; + const uint_fast32_t t1 = c & t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xa9_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | a; + const uint_fast32_t t1 = c ^ t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xaa_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + HEDLEY_STATIC_CAST(void, b); + return c; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xab_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | a; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = c | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xac_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = ~a; + const uint_fast32_t t2 = t1 & b; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xad_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = b & c; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xae_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 & b; + const uint_fast32_t t2 = t1 | c; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xaf_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, b); + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = c | t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xb0_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 | c; + const uint_fast32_t t2 = a & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xb1_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = t0 | a; + const uint_fast32_t t2 = ~b; + const uint_fast32_t t3 = t2 | c; + const uint_fast32_t t4 = t1 & t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xb2_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = b & t0; + const uint_fast32_t t2 = ~b; + const uint_fast32_t t3 = a | c; + const uint_fast32_t t4 = t2 & t3; + const uint_fast32_t t5 = t1 | t4; + return t5; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xb3_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = a & c; + const uint_fast32_t t2 = t0 | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xb4_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = t0 & b; + const uint_fast32_t t2 = t1 ^ a; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xb5_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~b; + const uint_fast32_t t3 = t2 & a; + const uint_fast32_t t4 = t1 | t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xb6_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = a ^ b; + const uint_fast32_t t2 = c ^ t1; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xb7_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = b & t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xb8_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = ~b; + const uint_fast32_t t2 = t1 & a; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xb9_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a & c; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xba_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 & a; + const uint_fast32_t t2 = t1 | c; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xbb_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = c | t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xbc_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = a ^ b; + const uint_fast32_t t2 = t0 | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xbd_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a ^ b; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xbe_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ a; + const uint_fast32_t t1 = c | t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xbf_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & a; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = c | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xc0_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, c); + const uint_fast32_t t0 = b & a; + return t0; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xc1_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ b; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~c; + const uint_fast32_t t3 = t2 | a; + const uint_fast32_t t4 = t1 & t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xc2_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ b; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a | c; + const uint_fast32_t t3 = t1 & t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xc3_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, c); + const uint_fast32_t t0 = b ^ a; + const uint_fast32_t t1 = ~t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xc4_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = t0 | a; + const uint_fast32_t t2 = b & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xc5_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = t0 | a; + const uint_fast32_t t2 = ~a; + const uint_fast32_t t3 = t2 | b; + const uint_fast32_t t4 = t1 & t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xc6_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 & c; + const uint_fast32_t t2 = t1 ^ b; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xc7_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ b; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~c; + const uint_fast32_t t3 = t2 & b; + const uint_fast32_t t4 = t1 | t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xc8_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a | c; + const uint_fast32_t t1 = b & t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xc9_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a | c; + const uint_fast32_t t1 = b ^ t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xca_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & b; + const uint_fast32_t t1 = ~a; + const uint_fast32_t t2 = t1 & c; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xcb_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ b; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = b & c; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xcc_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + HEDLEY_STATIC_CAST(void, c); + return b; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xcd_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a | c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = b | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xce_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = t0 & c; + const uint_fast32_t t2 = t1 | b; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xcf_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, c); + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = b | t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xd0_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = b | t0; + const uint_fast32_t t2 = a & t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xd1_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a & b; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xd2_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 & c; + const uint_fast32_t t2 = t1 ^ a; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xd3_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ b; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = ~c; + const uint_fast32_t t3 = t2 & a; + const uint_fast32_t t4 = t1 | t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xd4_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = b & t0; + const uint_fast32_t t2 = b ^ c; + const uint_fast32_t t3 = ~t2; + const uint_fast32_t t4 = a & t3; + const uint_fast32_t t5 = t1 | t4; + return t5; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xd5_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = a & b; + const uint_fast32_t t2 = t0 | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xd6_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & b; + const uint_fast32_t t1 = a ^ c; + const uint_fast32_t t2 = b ^ t1; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xd7_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ a; + const uint_fast32_t t1 = c & t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xd8_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = c & b; + const uint_fast32_t t1 = ~c; + const uint_fast32_t t2 = t1 & a; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xd9_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a & b; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xda_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & b; + const uint_fast32_t t1 = a ^ c; + const uint_fast32_t t2 = t0 | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xdb_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ b; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a ^ c; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xdc_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = t0 & a; + const uint_fast32_t t2 = t1 | b; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xdd_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = b | t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xde_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = b | t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xdf_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = b | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xe0_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = a & t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xe1_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = a ^ t0; + const uint_fast32_t t2 = ~t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xe2_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & a; + const uint_fast32_t t1 = ~b; + const uint_fast32_t t2 = t1 & c; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xe3_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ b; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a & c; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xe4_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = c & a; + const uint_fast32_t t1 = ~c; + const uint_fast32_t t2 = t1 & b; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xe5_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a & b; + const uint_fast32_t t3 = t1 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xe6_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & b; + const uint_fast32_t t1 = b ^ c; + const uint_fast32_t t2 = t0 | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xe7_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = ~a; + const uint_fast32_t t2 = t1 ^ c; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xe8_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = b ^ c; + const uint_fast32_t t2 = a & t1; + const uint_fast32_t t3 = t0 | t2; + return t3; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xe9_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = b ^ c; + const uint_fast32_t t2 = t0 ^ t1; + const uint_fast32_t t3 = a & b; + const uint_fast32_t t4 = t2 | t3; + return t4; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xea_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & a; + const uint_fast32_t t1 = c | t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xeb_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ a; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = c | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xec_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a & c; + const uint_fast32_t t1 = b | t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xed_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = a ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = b | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xee_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + const uint_fast32_t t0 = c | b; + return t0; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xef_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~a; + const uint_fast32_t t1 = b | c; + const uint_fast32_t t2 = t0 | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xf0_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, b); + HEDLEY_STATIC_CAST(void, c); + return a; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xf1_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xf2_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 & c; + const uint_fast32_t t2 = t1 | a; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xf3_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, c); + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = a | t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xf4_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = t0 & b; + const uint_fast32_t t2 = t1 | a; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xf5_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, b); + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = a | t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xf6_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = a | t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xf7_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xf8_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b & c; + const uint_fast32_t t1 = a | t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xf9_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b ^ c; + const uint_fast32_t t1 = ~t0; + const uint_fast32_t t2 = a | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xfa_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, b); + const uint_fast32_t t0 = c | a; + return t0; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xfb_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~b; + const uint_fast32_t t1 = t0 | c; + const uint_fast32_t t2 = a | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xfc_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, c); + const uint_fast32_t t0 = b | a; + return t0; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xfd_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = ~c; + const uint_fast32_t t1 = a | b; + const uint_fast32_t t2 = t0 | t1; + return t2; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xfe_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + const uint_fast32_t t0 = b | c; + const uint_fast32_t t1 = a | t0; + return t1; +} + +SIMDE_FUNCTION_ATTRIBUTES +uint_fast32_t +simde_x_ternarylogic_0xff_impl_(uint_fast32_t a, uint_fast32_t b, uint_fast32_t c) { + HEDLEY_STATIC_CAST(void, a); + HEDLEY_STATIC_CAST(void, b); + HEDLEY_STATIC_CAST(void, c); + const uint_fast32_t c1 = ~HEDLEY_STATIC_CAST(uint_fast32_t, 0); + return c1; +} + +#define SIMDE_X_TERNARYLOGIC_CASE(value) \ + case value: \ + SIMDE_VECTORIZE \ + for (size_t i = 0 ; i < (sizeof(r_.u32f) / sizeof(r_.u32f[0])) ; i++) { \ + r_.u32f[i] = HEDLEY_CONCAT3(simde_x_ternarylogic_, value, _impl_)(a_.u32f[i], b_.u32f[i], c_.u32f[i]); \ + } \ + break; + +#define SIMDE_X_TERNARYLOGIC_SWITCH(value) \ + switch(value) { \ + SIMDE_X_TERNARYLOGIC_CASE(0x00) \ + SIMDE_X_TERNARYLOGIC_CASE(0x01) \ + SIMDE_X_TERNARYLOGIC_CASE(0x02) \ + SIMDE_X_TERNARYLOGIC_CASE(0x03) \ + SIMDE_X_TERNARYLOGIC_CASE(0x04) \ + SIMDE_X_TERNARYLOGIC_CASE(0x05) \ + SIMDE_X_TERNARYLOGIC_CASE(0x06) \ + SIMDE_X_TERNARYLOGIC_CASE(0x07) \ + SIMDE_X_TERNARYLOGIC_CASE(0x08) \ + SIMDE_X_TERNARYLOGIC_CASE(0x09) \ + SIMDE_X_TERNARYLOGIC_CASE(0x0a) \ + SIMDE_X_TERNARYLOGIC_CASE(0x0b) \ + SIMDE_X_TERNARYLOGIC_CASE(0x0c) \ + SIMDE_X_TERNARYLOGIC_CASE(0x0d) \ + SIMDE_X_TERNARYLOGIC_CASE(0x0e) \ + SIMDE_X_TERNARYLOGIC_CASE(0x0f) \ + SIMDE_X_TERNARYLOGIC_CASE(0x10) \ + SIMDE_X_TERNARYLOGIC_CASE(0x11) \ + SIMDE_X_TERNARYLOGIC_CASE(0x12) \ + SIMDE_X_TERNARYLOGIC_CASE(0x13) \ + SIMDE_X_TERNARYLOGIC_CASE(0x14) \ + SIMDE_X_TERNARYLOGIC_CASE(0x15) \ + SIMDE_X_TERNARYLOGIC_CASE(0x16) \ + SIMDE_X_TERNARYLOGIC_CASE(0x17) \ + SIMDE_X_TERNARYLOGIC_CASE(0x18) \ + SIMDE_X_TERNARYLOGIC_CASE(0x19) \ + SIMDE_X_TERNARYLOGIC_CASE(0x1a) \ + SIMDE_X_TERNARYLOGIC_CASE(0x1b) \ + SIMDE_X_TERNARYLOGIC_CASE(0x1c) \ + SIMDE_X_TERNARYLOGIC_CASE(0x1d) \ + SIMDE_X_TERNARYLOGIC_CASE(0x1e) \ + SIMDE_X_TERNARYLOGIC_CASE(0x1f) \ + SIMDE_X_TERNARYLOGIC_CASE(0x20) \ + SIMDE_X_TERNARYLOGIC_CASE(0x21) \ + SIMDE_X_TERNARYLOGIC_CASE(0x22) \ + SIMDE_X_TERNARYLOGIC_CASE(0x23) \ + SIMDE_X_TERNARYLOGIC_CASE(0x24) \ + SIMDE_X_TERNARYLOGIC_CASE(0x25) \ + SIMDE_X_TERNARYLOGIC_CASE(0x26) \ + SIMDE_X_TERNARYLOGIC_CASE(0x27) \ + SIMDE_X_TERNARYLOGIC_CASE(0x28) \ + SIMDE_X_TERNARYLOGIC_CASE(0x29) \ + SIMDE_X_TERNARYLOGIC_CASE(0x2a) \ + SIMDE_X_TERNARYLOGIC_CASE(0x2b) \ + SIMDE_X_TERNARYLOGIC_CASE(0x2c) \ + SIMDE_X_TERNARYLOGIC_CASE(0x2d) \ + SIMDE_X_TERNARYLOGIC_CASE(0x2e) \ + SIMDE_X_TERNARYLOGIC_CASE(0x2f) \ + SIMDE_X_TERNARYLOGIC_CASE(0x30) \ + SIMDE_X_TERNARYLOGIC_CASE(0x31) \ + SIMDE_X_TERNARYLOGIC_CASE(0x32) \ + SIMDE_X_TERNARYLOGIC_CASE(0x33) \ + SIMDE_X_TERNARYLOGIC_CASE(0x34) \ + SIMDE_X_TERNARYLOGIC_CASE(0x35) \ + SIMDE_X_TERNARYLOGIC_CASE(0x36) \ + SIMDE_X_TERNARYLOGIC_CASE(0x37) \ + SIMDE_X_TERNARYLOGIC_CASE(0x38) \ + SIMDE_X_TERNARYLOGIC_CASE(0x39) \ + SIMDE_X_TERNARYLOGIC_CASE(0x3a) \ + SIMDE_X_TERNARYLOGIC_CASE(0x3b) \ + SIMDE_X_TERNARYLOGIC_CASE(0x3c) \ + SIMDE_X_TERNARYLOGIC_CASE(0x3d) \ + SIMDE_X_TERNARYLOGIC_CASE(0x3e) \ + SIMDE_X_TERNARYLOGIC_CASE(0x3f) \ + SIMDE_X_TERNARYLOGIC_CASE(0x40) \ + SIMDE_X_TERNARYLOGIC_CASE(0x41) \ + SIMDE_X_TERNARYLOGIC_CASE(0x42) \ + SIMDE_X_TERNARYLOGIC_CASE(0x43) \ + SIMDE_X_TERNARYLOGIC_CASE(0x44) \ + SIMDE_X_TERNARYLOGIC_CASE(0x45) \ + SIMDE_X_TERNARYLOGIC_CASE(0x46) \ + SIMDE_X_TERNARYLOGIC_CASE(0x47) \ + SIMDE_X_TERNARYLOGIC_CASE(0x48) \ + SIMDE_X_TERNARYLOGIC_CASE(0x49) \ + SIMDE_X_TERNARYLOGIC_CASE(0x4a) \ + SIMDE_X_TERNARYLOGIC_CASE(0x4b) \ + SIMDE_X_TERNARYLOGIC_CASE(0x4c) \ + SIMDE_X_TERNARYLOGIC_CASE(0x4d) \ + SIMDE_X_TERNARYLOGIC_CASE(0x4e) \ + SIMDE_X_TERNARYLOGIC_CASE(0x4f) \ + SIMDE_X_TERNARYLOGIC_CASE(0x50) \ + SIMDE_X_TERNARYLOGIC_CASE(0x51) \ + SIMDE_X_TERNARYLOGIC_CASE(0x52) \ + SIMDE_X_TERNARYLOGIC_CASE(0x53) \ + SIMDE_X_TERNARYLOGIC_CASE(0x54) \ + SIMDE_X_TERNARYLOGIC_CASE(0x55) \ + SIMDE_X_TERNARYLOGIC_CASE(0x56) \ + SIMDE_X_TERNARYLOGIC_CASE(0x57) \ + SIMDE_X_TERNARYLOGIC_CASE(0x58) \ + SIMDE_X_TERNARYLOGIC_CASE(0x59) \ + SIMDE_X_TERNARYLOGIC_CASE(0x5a) \ + SIMDE_X_TERNARYLOGIC_CASE(0x5b) \ + SIMDE_X_TERNARYLOGIC_CASE(0x5c) \ + SIMDE_X_TERNARYLOGIC_CASE(0x5d) \ + SIMDE_X_TERNARYLOGIC_CASE(0x5e) \ + SIMDE_X_TERNARYLOGIC_CASE(0x5f) \ + SIMDE_X_TERNARYLOGIC_CASE(0x60) \ + SIMDE_X_TERNARYLOGIC_CASE(0x61) \ + SIMDE_X_TERNARYLOGIC_CASE(0x62) \ + SIMDE_X_TERNARYLOGIC_CASE(0x63) \ + SIMDE_X_TERNARYLOGIC_CASE(0x64) \ + SIMDE_X_TERNARYLOGIC_CASE(0x65) \ + SIMDE_X_TERNARYLOGIC_CASE(0x66) \ + SIMDE_X_TERNARYLOGIC_CASE(0x67) \ + SIMDE_X_TERNARYLOGIC_CASE(0x68) \ + SIMDE_X_TERNARYLOGIC_CASE(0x69) \ + SIMDE_X_TERNARYLOGIC_CASE(0x6a) \ + SIMDE_X_TERNARYLOGIC_CASE(0x6b) \ + SIMDE_X_TERNARYLOGIC_CASE(0x6c) \ + SIMDE_X_TERNARYLOGIC_CASE(0x6d) \ + SIMDE_X_TERNARYLOGIC_CASE(0x6e) \ + SIMDE_X_TERNARYLOGIC_CASE(0x6f) \ + SIMDE_X_TERNARYLOGIC_CASE(0x70) \ + SIMDE_X_TERNARYLOGIC_CASE(0x71) \ + SIMDE_X_TERNARYLOGIC_CASE(0x72) \ + SIMDE_X_TERNARYLOGIC_CASE(0x73) \ + SIMDE_X_TERNARYLOGIC_CASE(0x74) \ + SIMDE_X_TERNARYLOGIC_CASE(0x75) \ + SIMDE_X_TERNARYLOGIC_CASE(0x76) \ + SIMDE_X_TERNARYLOGIC_CASE(0x77) \ + SIMDE_X_TERNARYLOGIC_CASE(0x78) \ + SIMDE_X_TERNARYLOGIC_CASE(0x79) \ + SIMDE_X_TERNARYLOGIC_CASE(0x7a) \ + SIMDE_X_TERNARYLOGIC_CASE(0x7b) \ + SIMDE_X_TERNARYLOGIC_CASE(0x7c) \ + SIMDE_X_TERNARYLOGIC_CASE(0x7d) \ + SIMDE_X_TERNARYLOGIC_CASE(0x7e) \ + SIMDE_X_TERNARYLOGIC_CASE(0x7f) \ + SIMDE_X_TERNARYLOGIC_CASE(0x80) \ + SIMDE_X_TERNARYLOGIC_CASE(0x81) \ + SIMDE_X_TERNARYLOGIC_CASE(0x82) \ + SIMDE_X_TERNARYLOGIC_CASE(0x83) \ + SIMDE_X_TERNARYLOGIC_CASE(0x84) \ + SIMDE_X_TERNARYLOGIC_CASE(0x85) \ + SIMDE_X_TERNARYLOGIC_CASE(0x86) \ + SIMDE_X_TERNARYLOGIC_CASE(0x87) \ + SIMDE_X_TERNARYLOGIC_CASE(0x88) \ + SIMDE_X_TERNARYLOGIC_CASE(0x89) \ + SIMDE_X_TERNARYLOGIC_CASE(0x8a) \ + SIMDE_X_TERNARYLOGIC_CASE(0x8b) \ + SIMDE_X_TERNARYLOGIC_CASE(0x8c) \ + SIMDE_X_TERNARYLOGIC_CASE(0x8d) \ + SIMDE_X_TERNARYLOGIC_CASE(0x8e) \ + SIMDE_X_TERNARYLOGIC_CASE(0x8f) \ + SIMDE_X_TERNARYLOGIC_CASE(0x90) \ + SIMDE_X_TERNARYLOGIC_CASE(0x91) \ + SIMDE_X_TERNARYLOGIC_CASE(0x92) \ + SIMDE_X_TERNARYLOGIC_CASE(0x93) \ + SIMDE_X_TERNARYLOGIC_CASE(0x94) \ + SIMDE_X_TERNARYLOGIC_CASE(0x95) \ + SIMDE_X_TERNARYLOGIC_CASE(0x96) \ + SIMDE_X_TERNARYLOGIC_CASE(0x97) \ + SIMDE_X_TERNARYLOGIC_CASE(0x98) \ + SIMDE_X_TERNARYLOGIC_CASE(0x99) \ + SIMDE_X_TERNARYLOGIC_CASE(0x9a) \ + SIMDE_X_TERNARYLOGIC_CASE(0x9b) \ + SIMDE_X_TERNARYLOGIC_CASE(0x9c) \ + SIMDE_X_TERNARYLOGIC_CASE(0x9d) \ + SIMDE_X_TERNARYLOGIC_CASE(0x9e) \ + SIMDE_X_TERNARYLOGIC_CASE(0x9f) \ + SIMDE_X_TERNARYLOGIC_CASE(0xa0) \ + SIMDE_X_TERNARYLOGIC_CASE(0xa1) \ + SIMDE_X_TERNARYLOGIC_CASE(0xa2) \ + SIMDE_X_TERNARYLOGIC_CASE(0xa3) \ + SIMDE_X_TERNARYLOGIC_CASE(0xa4) \ + SIMDE_X_TERNARYLOGIC_CASE(0xa5) \ + SIMDE_X_TERNARYLOGIC_CASE(0xa6) \ + SIMDE_X_TERNARYLOGIC_CASE(0xa7) \ + SIMDE_X_TERNARYLOGIC_CASE(0xa8) \ + SIMDE_X_TERNARYLOGIC_CASE(0xa9) \ + SIMDE_X_TERNARYLOGIC_CASE(0xaa) \ + SIMDE_X_TERNARYLOGIC_CASE(0xab) \ + SIMDE_X_TERNARYLOGIC_CASE(0xac) \ + SIMDE_X_TERNARYLOGIC_CASE(0xad) \ + SIMDE_X_TERNARYLOGIC_CASE(0xae) \ + SIMDE_X_TERNARYLOGIC_CASE(0xaf) \ + SIMDE_X_TERNARYLOGIC_CASE(0xb0) \ + SIMDE_X_TERNARYLOGIC_CASE(0xb1) \ + SIMDE_X_TERNARYLOGIC_CASE(0xb2) \ + SIMDE_X_TERNARYLOGIC_CASE(0xb3) \ + SIMDE_X_TERNARYLOGIC_CASE(0xb4) \ + SIMDE_X_TERNARYLOGIC_CASE(0xb5) \ + SIMDE_X_TERNARYLOGIC_CASE(0xb6) \ + SIMDE_X_TERNARYLOGIC_CASE(0xb7) \ + SIMDE_X_TERNARYLOGIC_CASE(0xb8) \ + SIMDE_X_TERNARYLOGIC_CASE(0xb9) \ + SIMDE_X_TERNARYLOGIC_CASE(0xba) \ + SIMDE_X_TERNARYLOGIC_CASE(0xbb) \ + SIMDE_X_TERNARYLOGIC_CASE(0xbc) \ + SIMDE_X_TERNARYLOGIC_CASE(0xbd) \ + SIMDE_X_TERNARYLOGIC_CASE(0xbe) \ + SIMDE_X_TERNARYLOGIC_CASE(0xbf) \ + SIMDE_X_TERNARYLOGIC_CASE(0xc0) \ + SIMDE_X_TERNARYLOGIC_CASE(0xc1) \ + SIMDE_X_TERNARYLOGIC_CASE(0xc2) \ + SIMDE_X_TERNARYLOGIC_CASE(0xc3) \ + SIMDE_X_TERNARYLOGIC_CASE(0xc4) \ + SIMDE_X_TERNARYLOGIC_CASE(0xc5) \ + SIMDE_X_TERNARYLOGIC_CASE(0xc6) \ + SIMDE_X_TERNARYLOGIC_CASE(0xc7) \ + SIMDE_X_TERNARYLOGIC_CASE(0xc8) \ + SIMDE_X_TERNARYLOGIC_CASE(0xc9) \ + SIMDE_X_TERNARYLOGIC_CASE(0xca) \ + SIMDE_X_TERNARYLOGIC_CASE(0xcb) \ + SIMDE_X_TERNARYLOGIC_CASE(0xcc) \ + SIMDE_X_TERNARYLOGIC_CASE(0xcd) \ + SIMDE_X_TERNARYLOGIC_CASE(0xce) \ + SIMDE_X_TERNARYLOGIC_CASE(0xcf) \ + SIMDE_X_TERNARYLOGIC_CASE(0xd0) \ + SIMDE_X_TERNARYLOGIC_CASE(0xd1) \ + SIMDE_X_TERNARYLOGIC_CASE(0xd2) \ + SIMDE_X_TERNARYLOGIC_CASE(0xd3) \ + SIMDE_X_TERNARYLOGIC_CASE(0xd4) \ + SIMDE_X_TERNARYLOGIC_CASE(0xd5) \ + SIMDE_X_TERNARYLOGIC_CASE(0xd6) \ + SIMDE_X_TERNARYLOGIC_CASE(0xd7) \ + SIMDE_X_TERNARYLOGIC_CASE(0xd8) \ + SIMDE_X_TERNARYLOGIC_CASE(0xd9) \ + SIMDE_X_TERNARYLOGIC_CASE(0xda) \ + SIMDE_X_TERNARYLOGIC_CASE(0xdb) \ + SIMDE_X_TERNARYLOGIC_CASE(0xdc) \ + SIMDE_X_TERNARYLOGIC_CASE(0xdd) \ + SIMDE_X_TERNARYLOGIC_CASE(0xde) \ + SIMDE_X_TERNARYLOGIC_CASE(0xdf) \ + SIMDE_X_TERNARYLOGIC_CASE(0xe0) \ + SIMDE_X_TERNARYLOGIC_CASE(0xe1) \ + SIMDE_X_TERNARYLOGIC_CASE(0xe2) \ + SIMDE_X_TERNARYLOGIC_CASE(0xe3) \ + SIMDE_X_TERNARYLOGIC_CASE(0xe4) \ + SIMDE_X_TERNARYLOGIC_CASE(0xe5) \ + SIMDE_X_TERNARYLOGIC_CASE(0xe6) \ + SIMDE_X_TERNARYLOGIC_CASE(0xe7) \ + SIMDE_X_TERNARYLOGIC_CASE(0xe8) \ + SIMDE_X_TERNARYLOGIC_CASE(0xe9) \ + SIMDE_X_TERNARYLOGIC_CASE(0xea) \ + SIMDE_X_TERNARYLOGIC_CASE(0xeb) \ + SIMDE_X_TERNARYLOGIC_CASE(0xec) \ + SIMDE_X_TERNARYLOGIC_CASE(0xed) \ + SIMDE_X_TERNARYLOGIC_CASE(0xee) \ + SIMDE_X_TERNARYLOGIC_CASE(0xef) \ + SIMDE_X_TERNARYLOGIC_CASE(0xf0) \ + SIMDE_X_TERNARYLOGIC_CASE(0xf1) \ + SIMDE_X_TERNARYLOGIC_CASE(0xf2) \ + SIMDE_X_TERNARYLOGIC_CASE(0xf3) \ + SIMDE_X_TERNARYLOGIC_CASE(0xf4) \ + SIMDE_X_TERNARYLOGIC_CASE(0xf5) \ + SIMDE_X_TERNARYLOGIC_CASE(0xf6) \ + SIMDE_X_TERNARYLOGIC_CASE(0xf7) \ + SIMDE_X_TERNARYLOGIC_CASE(0xf8) \ + SIMDE_X_TERNARYLOGIC_CASE(0xf9) \ + SIMDE_X_TERNARYLOGIC_CASE(0xfa) \ + SIMDE_X_TERNARYLOGIC_CASE(0xfb) \ + SIMDE_X_TERNARYLOGIC_CASE(0xfc) \ + SIMDE_X_TERNARYLOGIC_CASE(0xfd) \ + SIMDE_X_TERNARYLOGIC_CASE(0xfe) \ + SIMDE_X_TERNARYLOGIC_CASE(0xff) \ + } + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_ternarylogic_epi32(a, b, c, imm8) _mm_ternarylogic_epi32(a, b, c, imm8) +#else + SIMDE_HUGE_FUNCTION_ATTRIBUTES + simde__m128i + simde_mm_ternarylogic_epi32(simde__m128i a, simde__m128i b, simde__m128i c, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + #if defined(SIMDE_TERNARYLOGIC_COMPRESSION) + int to_do, mask; + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + simde__m128i_private t_; + to_do = imm8; + + r_.u64 = a_.u64 ^ a_.u64; + + mask = 0xFF; + if ((to_do & mask) == mask) { + r_.u64 = ~r_.u64; + to_do &= ~mask; + } + + mask = 0xF0; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 = a_.u64; + to_do &= ~mask; + } + + mask = 0xCC; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= b_.u64; + to_do &= ~mask; + } + + mask = 0xAA; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= c_.u64; + to_do &= ~mask; + } + + mask = 0x0F; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~a_.u64; + to_do &= ~mask; + } + + mask = 0x33; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~b_.u64; + to_do &= ~mask; + } + + mask = 0x55; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~c_.u64; + to_do &= ~mask; + } + + mask = 0x3C; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= a_.u64 ^ b_.u64; + to_do &= ~mask; + } + + mask = 0x5A; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= a_.u64 ^ c_.u64; + to_do &= ~mask; + } + + mask = 0x66; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= b_.u64 ^ c_.u64; + to_do &= ~mask; + } + + mask = 0xA0; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= a_.u64 & c_.u64; + to_do &= ~mask; + } + + mask = 0x50; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~c_.u64 & a_.u64; + to_do &= ~mask; + } + + mask = 0x0A; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~a_.u64 & c_.u64; + to_do &= ~mask; + } + + mask = 0x88; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= b_.u64 & c_.u64; + to_do &= ~mask; + } + + mask = 0x44; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~c_.u64 & b_.u64; + to_do &= ~mask; + } + + mask = 0x22; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~b_.u64 & c_.u64; + to_do &= ~mask; + } + + if (to_do & 0xc0) { + t_.u64 = a_.u64 & b_.u64; + if ((to_do & 0xc0) == 0xc0) r_.u64 |= t_.u64; + else if (to_do & 0x80) r_.u64 |= c_.u64 & t_.u64; + else r_.u64 |= ~c_.u64 & t_.u64; + } + + if (to_do & 0x30) { + t_.u64 = ~b_.u64 & a_.u64; + if ((to_do & 0x30) == 0x30) r_.u64 |= t_.u64; + else if (to_do & 0x20) r_.u64 |= c_.u64 & t_.u64; + else r_.u64 |= ~c_.u64 & t_.u64; + } + + if (to_do & 0x0c) { + t_.u64 = ~a_.u64 & b_.u64; + if ((to_do & 0x0c) == 0x0c) r_.u64 |= t_.u64; + else if (to_do & 0x08) r_.u64 |= c_.u64 & t_.u64; + else r_.u64 |= ~c_.u64 & t_.u64; + } + + if (to_do & 0x03) { + t_.u64 = ~(a_.u64 | b_.u64); + if ((to_do & 0x03) == 0x03) r_.u64 |= t_.u64; + else if (to_do & 0x02) r_.u64 |= c_.u64 & t_.u64; + else r_.u64 |= ~c_.u64 & t_.u64; + } + #else + uint64_t t; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + to_do = imm8; + + mask = 0xFF; + if ((to_do & mask) == mask) { + r_.u64[i] = UINT64_MAX; + to_do &= ~mask; + } + else r_.u64[i] = 0; + + mask = 0xF0; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] = a_.u64[i]; + to_do &= ~mask; + } + + mask = 0xCC; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= b_.u64[i]; + to_do &= ~mask; + } + + mask = 0xAA; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x0F; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~a_.u64[i]; + to_do &= ~mask; + } + + mask = 0x33; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~b_.u64[i]; + to_do &= ~mask; + } + + mask = 0x55; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x3C; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= a_.u64[i] ^ b_.u64[i]; + to_do &= ~mask; + } + + mask = 0x5A; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= a_.u64[i] ^ c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x66; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= b_.u64[i] ^ c_.u64[i]; + to_do &= ~mask; + } + + mask = 0xA0; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= a_.u64[i] & c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x50; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~c_.u64[i] & a_.u64[i]; + to_do &= ~mask; + } + + mask = 0x0A; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~a_.u64[i] & c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x88; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= b_.u64[i] & c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x44; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~c_.u64[i] & b_.u64[i]; + to_do &= ~mask; + } + + mask = 0x22; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~b_.u64[i] & c_.u64[i]; + to_do &= ~mask; + } + + if (to_do & 0xc0) { + t = a_.u64[i] & b_.u64[i]; + if ((to_do & 0xc0) == 0xc0) r_.u64[i] |= t; + else if (to_do & 0x80) r_.u64[i] |= c_.u64[i] & t; + else r_.u64[i] |= ~c_.u64[i] & t; + } + + if (to_do & 0x30) { + t = ~b_.u64[i] & a_.u64[i]; + if ((to_do & 0x30) == 0x30) r_.u64[i] |= t; + else if (to_do & 0x20) r_.u64[i] |= c_.u64[i] & t; + else r_.u64[i] |= ~c_.u64[i] & t; + } + + if (to_do & 0x0c) { + t = ~a_.u64[i] & b_.u64[i]; + if ((to_do & 0x0c) == 0x0c) r_.u64[i] |= t; + else if (to_do & 0x08) r_.u64[i] |= c_.u64[i] & t; + else r_.u64[i] |= ~c_.u64[i] & t; + } + + if (to_do & 0x03) { + t = ~(a_.u64[i] | b_.u64[i]); + if ((to_do & 0x03) == 0x03) r_.u64[i] |= t; + else if (to_do & 0x02) r_.u64[i] |= c_.u64[i] & t; + else r_.u64[i] |= ~c_.u64[i] & t; + } + } + #endif + #else + SIMDE_X_TERNARYLOGIC_SWITCH(imm8 & 255) + #endif + + return simde__m128i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_ternarylogic_epi32 + #define _mm_ternarylogic_epi32(a, b, c, imm8) simde_mm_ternarylogic_epi32(a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_mask_ternarylogic_epi32(src, k, a, b, imm8) _mm_mask_ternarylogic_epi32(src, k, a, b, imm8) +#else + #define simde_mm_mask_ternarylogic_epi32(src, k, a, b, imm8) simde_mm_mask_mov_epi32(src, k, simde_mm_ternarylogic_epi32(src, a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_ternarylogic_epi32 + #define _mm_mask_ternarylogic_epi32(src, k, a, b, imm8) simde_mm_mask_ternarylogic_epi32(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm_maskz_ternarylogic_epi32(k, a, b, c, imm8) _mm_maskz_ternarylogic_epi32(k, a, b, c, imm8) +#else + #define simde_mm_maskz_ternarylogic_epi32(k, a, b, c, imm8) simde_mm_maskz_mov_epi32(k, simde_mm_ternarylogic_epi32(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_ternarylogic_epi32 + #define _mm_maskz_ternarylogic_epi32(k, a, b, c, imm8) simde_mm_maskz_ternarylogic_epi32(k, a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm256_ternarylogic_epi32(a, b, c, imm8) _mm256_ternarylogic_epi32(a, b, c, imm8) +#else + SIMDE_HUGE_FUNCTION_ATTRIBUTES + simde__m256i + simde_mm256_ternarylogic_epi32(simde__m256i a, simde__m256i b, simde__m256i c, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b), + c_ = simde__m256i_to_private(c); + + #if defined(SIMDE_TERNARYLOGIC_COMPRESSION) + int to_do, mask; + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + simde__m256i_private t_; + to_do = imm8; + + r_.u64 = a_.u64 ^ a_.u64; + + mask = 0xFF; + if ((to_do & mask) == mask) { + r_.u64 = ~r_.u64; + to_do &= ~mask; + } + + mask = 0xF0; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 = a_.u64; + to_do &= ~mask; + } + + mask = 0xCC; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= b_.u64; + to_do &= ~mask; + } + + mask = 0xAA; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= c_.u64; + to_do &= ~mask; + } + + mask = 0x0F; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~a_.u64; + to_do &= ~mask; + } + + mask = 0x33; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~b_.u64; + to_do &= ~mask; + } + + mask = 0x55; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~c_.u64; + to_do &= ~mask; + } + + mask = 0x3C; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= a_.u64 ^ b_.u64; + to_do &= ~mask; + } + + mask = 0x5A; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= a_.u64 ^ c_.u64; + to_do &= ~mask; + } + + mask = 0x66; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= b_.u64 ^ c_.u64; + to_do &= ~mask; + } + + mask = 0xA0; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= a_.u64 & c_.u64; + to_do &= ~mask; + } + + mask = 0x50; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~c_.u64 & a_.u64; + to_do &= ~mask; + } + + mask = 0x0A; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~a_.u64 & c_.u64; + to_do &= ~mask; + } + + mask = 0x88; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= b_.u64 & c_.u64; + to_do &= ~mask; + } + + mask = 0x44; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~c_.u64 & b_.u64; + to_do &= ~mask; + } + + mask = 0x22; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~b_.u64 & c_.u64; + to_do &= ~mask; + } + + if (to_do & 0xc0) { + t_.u64 = a_.u64 & b_.u64; + if ((to_do & 0xc0) == 0xc0) r_.u64 |= t_.u64; + else if (to_do & 0x80) r_.u64 |= c_.u64 & t_.u64; + else r_.u64 |= ~c_.u64 & t_.u64; + } + + if (to_do & 0x30) { + t_.u64 = ~b_.u64 & a_.u64; + if ((to_do & 0x30) == 0x30) r_.u64 |= t_.u64; + else if (to_do & 0x20) r_.u64 |= c_.u64 & t_.u64; + else r_.u64 |= ~c_.u64 & t_.u64; + } + + if (to_do & 0x0c) { + t_.u64 = ~a_.u64 & b_.u64; + if ((to_do & 0x0c) == 0x0c) r_.u64 |= t_.u64; + else if (to_do & 0x08) r_.u64 |= c_.u64 & t_.u64; + else r_.u64 |= ~c_.u64 & t_.u64; + } + + if (to_do & 0x03) { + t_.u64 = ~(a_.u64 | b_.u64); + if ((to_do & 0x03) == 0x03) r_.u64 |= t_.u64; + else if (to_do & 0x02) r_.u64 |= c_.u64 & t_.u64; + else r_.u64 |= ~c_.u64 & t_.u64; + } + #else + uint64_t t; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + to_do = imm8; + + mask = 0xFF; + if ((to_do & mask) == mask) { + r_.u64[i] = UINT64_MAX; + to_do &= ~mask; + } + else r_.u64[i] = 0; + + mask = 0xF0; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] = a_.u64[i]; + to_do &= ~mask; + } + + mask = 0xCC; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= b_.u64[i]; + to_do &= ~mask; + } + + mask = 0xAA; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x0F; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~a_.u64[i]; + to_do &= ~mask; + } + + mask = 0x33; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~b_.u64[i]; + to_do &= ~mask; + } + + mask = 0x55; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x3C; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= a_.u64[i] ^ b_.u64[i]; + to_do &= ~mask; + } + + mask = 0x5A; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= a_.u64[i] ^ c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x66; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= b_.u64[i] ^ c_.u64[i]; + to_do &= ~mask; + } + + mask = 0xA0; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= a_.u64[i] & c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x50; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~c_.u64[i] & a_.u64[i]; + to_do &= ~mask; + } + + mask = 0x0A; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~a_.u64[i] & c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x88; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= b_.u64[i] & c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x44; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~c_.u64[i] & b_.u64[i]; + to_do &= ~mask; + } + + mask = 0x22; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~b_.u64[i] & c_.u64[i]; + to_do &= ~mask; + } + + if (to_do & 0xc0) { + t = a_.u64[i] & b_.u64[i]; + if ((to_do & 0xc0) == 0xc0) r_.u64[i] |= t; + else if (to_do & 0x80) r_.u64[i] |= c_.u64[i] & t; + else r_.u64[i] |= ~c_.u64[i] & t; + } + + if (to_do & 0x30) { + t = ~b_.u64[i] & a_.u64[i]; + if ((to_do & 0x30) == 0x30) r_.u64[i] |= t; + else if (to_do & 0x20) r_.u64[i] |= c_.u64[i] & t; + else r_.u64[i] |= ~c_.u64[i] & t; + } + + if (to_do & 0x0c) { + t = ~a_.u64[i] & b_.u64[i]; + if ((to_do & 0x0c) == 0x0c) r_.u64[i] |= t; + else if (to_do & 0x08) r_.u64[i] |= c_.u64[i] & t; + else r_.u64[i] |= ~c_.u64[i] & t; + } + + if (to_do & 0x03) { + t = ~(a_.u64[i] | b_.u64[i]); + if ((to_do & 0x03) == 0x03) r_.u64[i] |= t; + else if (to_do & 0x02) r_.u64[i] |= c_.u64[i] & t; + else r_.u64[i] |= ~c_.u64[i] & t; + } + } + #endif + #else + SIMDE_X_TERNARYLOGIC_SWITCH(imm8 & 255) + #endif + + return simde__m256i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_ternarylogic_epi32 + #define _mm256_ternarylogic_epi32(a, b, c, imm8) simde_mm256_ternarylogic_epi32(a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm256_mask_ternarylogic_epi32(src, k, a, b, imm8) _mm256_mask_ternarylogic_epi32(src, k, a, b, imm8) +#else + #define simde_mm256_mask_ternarylogic_epi32(src, k, a, b, imm8) simde_mm256_mask_mov_epi32(src, k, simde_mm256_ternarylogic_epi32(src, a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_ternarylogic_epi32 + #define _mm256_mask_ternarylogic_epi32(src, k, a, b, imm8) simde_mm256_mask_ternarylogic_epi32(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm256_maskz_ternarylogic_epi32(k, a, b, c, imm8) _mm256_maskz_ternarylogic_epi32(k, a, b, c, imm8) +#else + #define simde_mm256_maskz_ternarylogic_epi32(k, a, b, c, imm8) simde_mm256_maskz_mov_epi32(k, simde_mm256_ternarylogic_epi32(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_ternarylogic_epi32 + #define _mm256_maskz_ternarylogic_epi32(k, a, b, c, imm8) simde_mm256_maskz_ternarylogic_epi32(k, a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_ternarylogic_epi32(a, b, c, imm8) _mm512_ternarylogic_epi32(a, b, c, imm8) +#else + SIMDE_HUGE_FUNCTION_ATTRIBUTES + simde__m512i + simde_mm512_ternarylogic_epi32(simde__m512i a, simde__m512i b, simde__m512i c, int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b), + c_ = simde__m512i_to_private(c); + + #if defined(SIMDE_TERNARYLOGIC_COMPRESSION) + int to_do, mask; + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + simde__m512i_private t_; + to_do = imm8; + + r_.u64 = a_.u64 ^ a_.u64; + + mask = 0xFF; + if ((to_do & mask) == mask) { + r_.u64 = ~r_.u64; + to_do &= ~mask; + } + + mask = 0xF0; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 = a_.u64; + to_do &= ~mask; + } + + mask = 0xCC; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= b_.u64; + to_do &= ~mask; + } + + mask = 0xAA; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= c_.u64; + to_do &= ~mask; + } + + mask = 0x0F; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~a_.u64; + to_do &= ~mask; + } + + mask = 0x33; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~b_.u64; + to_do &= ~mask; + } + + mask = 0x55; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~c_.u64; + to_do &= ~mask; + } + + mask = 0x3C; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= a_.u64 ^ b_.u64; + to_do &= ~mask; + } + + mask = 0x5A; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= a_.u64 ^ c_.u64; + to_do &= ~mask; + } + + mask = 0x66; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= b_.u64 ^ c_.u64; + to_do &= ~mask; + } + + mask = 0xA0; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= a_.u64 & c_.u64; + to_do &= ~mask; + } + + mask = 0x50; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~c_.u64 & a_.u64; + to_do &= ~mask; + } + + mask = 0x0A; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~a_.u64 & c_.u64; + to_do &= ~mask; + } + + mask = 0x88; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= b_.u64 & c_.u64; + to_do &= ~mask; + } + + mask = 0x44; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~c_.u64 & b_.u64; + to_do &= ~mask; + } + + mask = 0x22; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64 |= ~b_.u64 & c_.u64; + to_do &= ~mask; + } + + if (to_do & 0xc0) { + t_.u64 = a_.u64 & b_.u64; + if ((to_do & 0xc0) == 0xc0) r_.u64 |= t_.u64; + else if (to_do & 0x80) r_.u64 |= c_.u64 & t_.u64; + else r_.u64 |= ~c_.u64 & t_.u64; + } + + if (to_do & 0x30) { + t_.u64 = ~b_.u64 & a_.u64; + if ((to_do & 0x30) == 0x30) r_.u64 |= t_.u64; + else if (to_do & 0x20) r_.u64 |= c_.u64 & t_.u64; + else r_.u64 |= ~c_.u64 & t_.u64; + } + + if (to_do & 0x0c) { + t_.u64 = ~a_.u64 & b_.u64; + if ((to_do & 0x0c) == 0x0c) r_.u64 |= t_.u64; + else if (to_do & 0x08) r_.u64 |= c_.u64 & t_.u64; + else r_.u64 |= ~c_.u64 & t_.u64; + } + + if (to_do & 0x03) { + t_.u64 = ~(a_.u64 | b_.u64); + if ((to_do & 0x03) == 0x03) r_.u64 |= t_.u64; + else if (to_do & 0x02) r_.u64 |= c_.u64 & t_.u64; + else r_.u64 |= ~c_.u64 & t_.u64; + } + #else + uint64_t t; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + to_do = imm8; + + mask = 0xFF; + if ((to_do & mask) == mask) { + r_.u64[i] = UINT64_MAX; + to_do &= ~mask; + } + else r_.u64[i] = 0; + + mask = 0xF0; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] = a_.u64[i]; + to_do &= ~mask; + } + + mask = 0xCC; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= b_.u64[i]; + to_do &= ~mask; + } + + mask = 0xAA; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x0F; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~a_.u64[i]; + to_do &= ~mask; + } + + mask = 0x33; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~b_.u64[i]; + to_do &= ~mask; + } + + mask = 0x55; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x3C; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= a_.u64[i] ^ b_.u64[i]; + to_do &= ~mask; + } + + mask = 0x5A; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= a_.u64[i] ^ c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x66; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= b_.u64[i] ^ c_.u64[i]; + to_do &= ~mask; + } + + mask = 0xA0; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= a_.u64[i] & c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x50; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~c_.u64[i] & a_.u64[i]; + to_do &= ~mask; + } + + mask = 0x0A; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~a_.u64[i] & c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x88; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= b_.u64[i] & c_.u64[i]; + to_do &= ~mask; + } + + mask = 0x44; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~c_.u64[i] & b_.u64[i]; + to_do &= ~mask; + } + + mask = 0x22; + if ((to_do & mask) && ((imm8 & mask) == mask)) { + r_.u64[i] |= ~b_.u64[i] & c_.u64[i]; + to_do &= ~mask; + } + + if (to_do & 0xc0) { + t = a_.u64[i] & b_.u64[i]; + if ((to_do & 0xc0) == 0xc0) r_.u64[i] |= t; + else if (to_do & 0x80) r_.u64[i] |= c_.u64[i] & t; + else r_.u64[i] |= ~c_.u64[i] & t; + } + + if (to_do & 0x30) { + t = ~b_.u64[i] & a_.u64[i]; + if ((to_do & 0x30) == 0x30) r_.u64[i] |= t; + else if (to_do & 0x20) r_.u64[i] |= c_.u64[i] & t; + else r_.u64[i] |= ~c_.u64[i] & t; + } + + if (to_do & 0x0c) { + t = ~a_.u64[i] & b_.u64[i]; + if ((to_do & 0x0c) == 0x0c) r_.u64[i] |= t; + else if (to_do & 0x08) r_.u64[i] |= c_.u64[i] & t; + else r_.u64[i] |= ~c_.u64[i] & t; + } + + if (to_do & 0x03) { + t = ~(a_.u64[i] | b_.u64[i]); + if ((to_do & 0x03) == 0x03) r_.u64[i] |= t; + else if (to_do & 0x02) r_.u64[i] |= c_.u64[i] & t; + else r_.u64[i] |= ~c_.u64[i] & t; + } + } + #endif + #else + SIMDE_X_TERNARYLOGIC_SWITCH(imm8 & 255) + #endif + + return simde__m512i_from_private(r_); + } +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_ternarylogic_epi32 + #define _mm512_ternarylogic_epi32(a, b, c, imm8) simde_mm512_ternarylogic_epi32(a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_ternarylogic_epi32(src, k, a, b, imm8) _mm512_mask_ternarylogic_epi32(src, k, a, b, imm8) +#else + #define simde_mm512_mask_ternarylogic_epi32(src, k, a, b, imm8) simde_mm512_mask_mov_epi32(src, k, simde_mm512_ternarylogic_epi32(src, a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_ternarylogic_epi32 + #define _mm512_mask_ternarylogic_epi32(src, k, a, b, imm8) simde_mm512_mask_ternarylogic_epi32(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_maskz_ternarylogic_epi32(k, a, b, c, imm8) _mm512_maskz_ternarylogic_epi32(k, a, b, c, imm8) +#else + #define simde_mm512_maskz_ternarylogic_epi32(k, a, b, c, imm8) simde_mm512_maskz_mov_epi32(k, simde_mm512_ternarylogic_epi32(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_ternarylogic_epi32 + #define _mm512_maskz_ternarylogic_epi32(k, a, b, c, imm8) simde_mm512_maskz_ternarylogic_epi32(k, a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_ternarylogic_epi64(a, b, c, imm8) _mm_ternarylogic_epi64(a, b, c, imm8) +#else + #define simde_mm_ternarylogic_epi64(a, b, c, imm8) simde_mm_ternarylogic_epi32(a, b, c, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_ternarylogic_epi64 + #define _mm_ternarylogic_epi64(a, b, c, imm8) simde_mm_ternarylogic_epi64(a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_mask_ternarylogic_epi64(src, k, a, b, imm8) _mm_mask_ternarylogic_epi64(src, k, a, b, imm8) +#else + #define simde_mm_mask_ternarylogic_epi64(src, k, a, b, imm8) simde_mm_mask_mov_epi64(src, k, simde_mm_ternarylogic_epi64(src, a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_ternarylogic_epi64 + #define _mm_mask_ternarylogic_epi64(src, k, a, b, imm8) simde_mm_mask_ternarylogic_epi64(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_maskz_ternarylogic_epi64(k, a, b, c, imm8) _mm_maskz_ternarylogic_epi64(k, a, b, c, imm8) +#else + #define simde_mm_maskz_ternarylogic_epi64(k, a, b, c, imm8) simde_mm_maskz_mov_epi64(k, simde_mm_ternarylogic_epi64(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_ternarylogic_epi64 + #define _mm_maskz_ternarylogic_epi64(k, a, b, c, imm8) simde_mm_maskz_ternarylogic_epi64(k, a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_ternarylogic_epi64(a, b, c, imm8) _mm256_ternarylogic_epi64(a, b, c, imm8) +#else + #define simde_mm256_ternarylogic_epi64(a, b, c, imm8) simde_mm256_ternarylogic_epi32(a, b, c, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_ternarylogic_epi64 + #define _mm256_ternarylogic_epi64(a, b, c, imm8) simde_mm256_ternarylogic_epi64(a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_ternarylogic_epi64(src, k, a, b, imm8) _mm256_mask_ternarylogic_epi64(src, k, a, b, imm8) +#else + #define simde_mm256_mask_ternarylogic_epi64(src, k, a, b, imm8) simde_mm256_mask_mov_epi64(src, k, simde_mm256_ternarylogic_epi64(src, a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_ternarylogic_epi64 + #define _mm256_mask_ternarylogic_epi64(src, k, a, b, imm8) simde_mm256_mask_ternarylogic_epi64(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_maskz_ternarylogic_epi64(k, a, b, c, imm8) _mm256_maskz_ternarylogic_epi64(k, a, b, c, imm8) +#else + #define simde_mm256_maskz_ternarylogic_epi64(k, a, b, c, imm8) simde_mm256_maskz_mov_epi64(k, simde_mm256_ternarylogic_epi64(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_ternarylogic_epi64 + #define _mm256_maskz_ternarylogic_epi64(k, a, b, c, imm8) simde_mm256_maskz_ternarylogic_epi64(k, a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_ternarylogic_epi64(a, b, c, imm8) _mm512_ternarylogic_epi64(a, b, c, imm8) +#else + #define simde_mm512_ternarylogic_epi64(a, b, c, imm8) simde_mm512_ternarylogic_epi32(a, b, c, imm8) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_ternarylogic_epi64 + #define _mm512_ternarylogic_epi64(a, b, c, imm8) simde_mm512_ternarylogic_epi64(a, b, c, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_ternarylogic_epi64(src, k, a, b, imm8) _mm512_mask_ternarylogic_epi64(src, k, a, b, imm8) +#else + #define simde_mm512_mask_ternarylogic_epi64(src, k, a, b, imm8) simde_mm512_mask_mov_epi64(src, k, simde_mm512_ternarylogic_epi64(src, a, b, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_ternarylogic_epi64 + #define _mm512_mask_ternarylogic_epi64(src, k, a, b, imm8) simde_mm512_mask_ternarylogic_epi64(src, k, a, b, imm8) +#endif + +#if defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_maskz_ternarylogic_epi64(k, a, b, c, imm8) _mm512_maskz_ternarylogic_epi64(k, a, b, c, imm8) +#else + #define simde_mm512_maskz_ternarylogic_epi64(k, a, b, c, imm8) simde_mm512_maskz_mov_epi64(k, simde_mm512_ternarylogic_epi64(a, b, c, imm8)) +#endif +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_ternarylogic_epi64 + #define _mm512_maskz_ternarylogic_epi64(k, a, b, c, imm8) simde_mm512_maskz_ternarylogic_epi64(k, a, b, c, imm8) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_TERNARYLOGIC_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/test.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/test.h new file mode 100644 index 00000000..0d386341 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/test.h @@ -0,0 +1,232 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + * 2020 Christopher Moore + * 2021 Andrew Rodriguez + */ + +#if !defined(SIMDE_X86_AVX512_TEST_H) +#define SIMDE_X86_AVX512_TEST_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_test_epi32_mask (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_test_epi32_mask(a, b); + #else + simde__m256i_private + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + simde__mmask8 r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r |= HEDLEY_STATIC_CAST(simde__mmask16, !!(a_.i32[i] & b_.i32[i]) << i); + } + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_test_epi32_mask +#define _mm256_test_epi32_mask(a, b) simde_mm256_test_epi32_mask(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm256_mask_test_epi32_mask (simde__mmask8 k1, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_test_epi32_mask(k1, a, b); + #else + return simde_mm256_test_epi32_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_test_epi32_mask + #define _mm256_mask_test_epi32_mask(k1, a, b) simde_mm256_mask_test_epi32_mask(k1, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_test_epi16_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_test_epi16_mask(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + simde__mmask32 r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i16) / sizeof(a_.i16[0])) ; i++) { + r |= HEDLEY_STATIC_CAST(simde__mmask32, !!(a_.i16[i] & b_.i16[i]) << i); + } + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_test_epi16_mask + #define _mm512_test_epi16_mask(a, b) simde_mm512_test_epi16_mask(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_test_epi32_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_test_epi32_mask(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + simde__mmask16 r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i32) / sizeof(a_.i32[0])) ; i++) { + r |= HEDLEY_STATIC_CAST(simde__mmask16, !!(a_.i32[i] & b_.i32[i]) << i); + } + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_test_epi32_mask +#define _mm512_test_epi32_mask(a, b) simde_mm512_test_epi32_mask(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_test_epi64_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_test_epi64_mask(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + simde__mmask8 r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r |= HEDLEY_STATIC_CAST(simde__mmask8, !!(a_.i64[i] & b_.i64[i]) << i); + } + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_test_epi64_mask + #define _mm512_test_epi64_mask(a, b) simde_mm512_test_epi64_mask(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_test_epi8_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_test_epi8_mask(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + simde__mmask64 r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i8) / sizeof(a_.i8[0])) ; i++) { + r |= HEDLEY_STATIC_CAST(simde__mmask64, HEDLEY_STATIC_CAST(uint64_t, !!(a_.i8[i] & b_.i8[i])) << i); + } + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_test_epi8_mask + #define _mm512_test_epi8_mask(a, b) simde_mm512_test_epi8_mask(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask32 +simde_mm512_mask_test_epi16_mask (simde__mmask32 k1, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_test_epi16_mask(k1, a, b); + #else + return simde_mm512_test_epi16_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_test_epi16_mask + #define _mm512_mask_test_epi16_mask(k1, a, b) simde_mm512_mask_test_epi16_mask(k1, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask16 +simde_mm512_mask_test_epi32_mask (simde__mmask16 k1, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_test_epi32_mask(k1, a, b); + #else + return simde_mm512_test_epi32_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_test_epi32_mask + #define _mm512_mask_test_epi32_mask(k1, a, b) simde_mm512_mask_test_epi32_mask(k1, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_mask_test_epi64_mask (simde__mmask8 k1, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_test_epi64_mask(k1, a, b); + #else + return simde_mm512_test_epi64_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_test_epi64_mask + #define _mm512_mask_test_epi64_mask(k1, a, b) simde_mm512_mask_test_epi64_mask(k1, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask64 +simde_mm512_mask_test_epi8_mask (simde__mmask64 k1, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_test_epi8_mask(k1, a, b); + #else + return simde_mm512_test_epi8_mask(a, b) & k1; + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_test_epi8_mask + #define _mm512_mask_test_epi8_mask(k1, a, b) simde_mm512_mask_test_epi8_mask(k1, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_TEST_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/testn.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/testn.h new file mode 100644 index 00000000..4879235a --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/testn.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2021 Andrew Rodriguez + */ + +#if !defined(SIMDE_X86_AVX512_TESTN_H) +#define SIMDE_X86_AVX512_TESTN_H + +#include "types.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__mmask8 +simde_mm512_testn_epi64_mask (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_testn_epi64_mask(a, b); + #else + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + simde__mmask8 r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i64) / sizeof(a_.i64[0])) ; i++) { + r |= (!(a_.i64[i] & b_.i64[i])) << i; + } + + return r; + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_testn_epi64_mask + #define _mm512_testn_epi64_mask(a, b) simde_mm512_testn_epi64_mask(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_TESTN_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/types.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/types.h new file mode 100644 index 00000000..639df259 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/types.h @@ -0,0 +1,821 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_AVX512_TYPES_H) +#define SIMDE_X86_AVX512_TYPES_H +#include "../avx.h" +#include "../../simde-f16.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +/* The problem is that Microsoft doesn't support 64-byte aligned parameters, except for + * __m512/__m512i/__m512d. Since our private union has an __m512 member it will be 64-byte + * aligned even if we reduce the alignment requirements of other members. + * + * Even if we're on x86 and use the native AVX-512 types for arguments/return values, the + * to/from private functions will break, and I'm not willing to change their APIs to use + * pointers (which would also require more verbose code on the caller side) just to make + * MSVC happy. + * + * If you want to use AVX-512 in SIMDe, you'll need to either upgrade to MSVC 2017 or later, + * or upgrade to a different compiler (clang-cl, perhaps?). If you have an idea of how to + * fix this without requiring API changes (except transparently through macros), patches + * are welcome. + */ + +# if defined(HEDLEY_MSVC_VERSION) && !HEDLEY_MSVC_VERSION_CHECK(19,10,0) +# if defined(SIMDE_X86_AVX512F_NATIVE) +# undef SIMDE_X86_AVX512F_NATIVE +# pragma message("Native AVX-512 support requires MSVC 2017 or later. See comment above (in code) for details.") +# endif +# define SIMDE_AVX512_ALIGN SIMDE_ALIGN_TO_32 +# else +# define SIMDE_AVX512_ALIGN SIMDE_ALIGN_TO_64 +# endif + +typedef union { + #if defined(SIMDE_VECTOR_SUBSCRIPT) + SIMDE_ALIGN_TO_16 int8_t i8 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int16_t i16 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int32_t i32 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int64_t i64 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint8_t u8 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint16_t u16 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint32_t u32 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint64_t u64 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_ALIGN_TO_16 simde_int128 i128 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 simde_uint128 u128 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + #endif + SIMDE_ALIGN_TO_16 simde_float32 f32 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int_fast32_t i32f SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint_fast32_t u32f SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + #else + SIMDE_ALIGN_TO_16 int8_t i8[16]; + SIMDE_ALIGN_TO_16 int16_t i16[8]; + SIMDE_ALIGN_TO_16 int32_t i32[4]; + SIMDE_ALIGN_TO_16 int64_t i64[2]; + SIMDE_ALIGN_TO_16 uint8_t u8[16]; + SIMDE_ALIGN_TO_16 uint16_t u16[8]; + SIMDE_ALIGN_TO_16 uint32_t u32[4]; + SIMDE_ALIGN_TO_16 uint64_t u64[2]; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_ALIGN_TO_16 simde_int128 i128[1]; + SIMDE_ALIGN_TO_16 simde_uint128 u128[1]; + #endif + SIMDE_ALIGN_TO_16 simde_float32 f32[4]; + SIMDE_ALIGN_TO_16 int_fast32_t i32f[16 / sizeof(int_fast32_t)]; + SIMDE_ALIGN_TO_16 uint_fast32_t u32f[16 / sizeof(uint_fast32_t)]; + #endif + + SIMDE_ALIGN_TO_16 simde__m64_private m64_private[2]; + SIMDE_ALIGN_TO_16 simde__m64 m64[2]; + + #if defined(SIMDE_X86_AVX512BF16_NATIVE) + SIMDE_ALIGN_TO_16 __m128bh n; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_ALIGN_TO_16 int8x16_t neon_i8; + SIMDE_ALIGN_TO_16 int16x8_t neon_i16; + SIMDE_ALIGN_TO_16 int32x4_t neon_i32; + SIMDE_ALIGN_TO_16 int64x2_t neon_i64; + SIMDE_ALIGN_TO_16 uint8x16_t neon_u8; + SIMDE_ALIGN_TO_16 uint16x8_t neon_u16; + SIMDE_ALIGN_TO_16 uint32x4_t neon_u32; + SIMDE_ALIGN_TO_16 uint64x2_t neon_u64; + SIMDE_ALIGN_TO_16 float32x4_t neon_f32; + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + SIMDE_ALIGN_TO_16 float64x2_t neon_f64; + #endif + #elif defined(SIMDE_MIPS_MSA_NATIVE) + v16i8 msa_i8; + v8i16 msa_i16; + v4i32 msa_i32; + v2i64 msa_i64; + v16u8 msa_u8; + v8u16 msa_u16; + v4u32 msa_u32; + v2u64 msa_u64; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + SIMDE_ALIGN_TO_16 v128_t wasm_v128; + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) altivec_u8; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) altivec_u16; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed char) altivec_i8; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed short) altivec_i16; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed int) altivec_i32; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(float) altivec_f32; + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long) altivec_u64; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed long long) altivec_i64; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(double) altivec_f64; + #endif + #endif +} simde__m128bh_private; + +typedef union { + #if defined(SIMDE_VECTOR_SUBSCRIPT) + SIMDE_ALIGN_TO_32 int8_t i8 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int16_t i16 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int32_t i32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int64_t i64 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint8_t u8 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint16_t u16 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint32_t u32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint64_t u64 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_ALIGN_TO_32 simde_int128 i128 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 simde_uint128 u128 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + #endif + SIMDE_ALIGN_TO_32 simde_float32 f32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 simde_float64 f64 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 int_fast32_t i32f SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_32 uint_fast32_t u32f SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + #else + SIMDE_ALIGN_TO_32 int8_t i8[32]; + SIMDE_ALIGN_TO_32 int16_t i16[16]; + SIMDE_ALIGN_TO_32 int32_t i32[8]; + SIMDE_ALIGN_TO_32 int64_t i64[4]; + SIMDE_ALIGN_TO_32 uint8_t u8[32]; + SIMDE_ALIGN_TO_32 uint16_t u16[16]; + SIMDE_ALIGN_TO_32 uint32_t u32[8]; + SIMDE_ALIGN_TO_32 uint64_t u64[4]; + SIMDE_ALIGN_TO_32 int_fast32_t i32f[32 / sizeof(int_fast32_t)]; + SIMDE_ALIGN_TO_32 uint_fast32_t u32f[32 / sizeof(uint_fast32_t)]; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_ALIGN_TO_32 simde_int128 i128[2]; + SIMDE_ALIGN_TO_32 simde_uint128 u128[2]; + #endif + SIMDE_ALIGN_TO_32 simde_float32 f32[8]; + SIMDE_ALIGN_TO_32 simde_float64 f64[4]; + #endif + + SIMDE_ALIGN_TO_32 simde__m128_private m128_private[2]; + SIMDE_ALIGN_TO_32 simde__m128 m128[2]; + + #if defined(SIMDE_X86_BF16_NATIVE) + SIMDE_ALIGN_TO_32 __m256bh n; + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) altivec_u8[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) altivec_u16[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed char) altivec_i8[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed short) altivec_i16[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(int) altivec_i32[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(float) altivec_f32[2]; + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long) altivec_u64[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(long long) altivec_i64[2]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(double) altivec_f64[2]; + #endif + #endif +} simde__m256bh_private; + +typedef union { + #if defined(SIMDE_VECTOR_SUBSCRIPT) + SIMDE_AVX512_ALIGN int8_t i8 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int16_t i16 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int32_t i32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int64_t i64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint8_t u8 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint16_t u16 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint32_t u32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint64_t u64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_AVX512_ALIGN simde_int128 i128 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN simde_uint128 u128 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #endif + SIMDE_AVX512_ALIGN simde_float32 f32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN simde_float64 f64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int_fast32_t i32f SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint_fast32_t u32f SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #else + SIMDE_AVX512_ALIGN int8_t i8[64]; + SIMDE_AVX512_ALIGN int16_t i16[32]; + SIMDE_AVX512_ALIGN int32_t i32[16]; + SIMDE_AVX512_ALIGN int64_t i64[8]; + SIMDE_AVX512_ALIGN uint8_t u8[64]; + SIMDE_AVX512_ALIGN uint16_t u16[32]; + SIMDE_AVX512_ALIGN uint32_t u32[16]; + SIMDE_AVX512_ALIGN uint64_t u64[8]; + SIMDE_AVX512_ALIGN int_fast32_t i32f[64 / sizeof(int_fast32_t)]; + SIMDE_AVX512_ALIGN uint_fast32_t u32f[64 / sizeof(uint_fast32_t)]; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_AVX512_ALIGN simde_int128 i128[4]; + SIMDE_AVX512_ALIGN simde_uint128 u128[4]; + #endif + SIMDE_AVX512_ALIGN simde_float32 f32[16]; + SIMDE_AVX512_ALIGN simde_float64 f64[8]; + #endif + + SIMDE_AVX512_ALIGN simde__m128_private m128_private[4]; + SIMDE_AVX512_ALIGN simde__m128 m128[4]; + SIMDE_AVX512_ALIGN simde__m256_private m256_private[2]; + SIMDE_AVX512_ALIGN simde__m256 m256[2]; + + #if defined(SIMDE_X86_AVX512BF16_NATIVE) + SIMDE_AVX512_ALIGN __m512bh n; + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) altivec_u8[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) altivec_u16[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed char) altivec_i8[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed short) altivec_i16[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed int) altivec_i32[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(float) altivec_f32[4]; + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long) altivec_u64[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed long long) altivec_i64[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(double) altivec_f64[4]; + #endif + #endif +} simde__m512bh_private; + +typedef union { + #if defined(SIMDE_VECTOR_SUBSCRIPT) + SIMDE_AVX512_ALIGN int8_t i8 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int16_t i16 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int32_t i32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int64_t i64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint8_t u8 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint16_t u16 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint32_t u32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint64_t u64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_AVX512_ALIGN simde_int128 i128 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN simde_uint128 u128 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #endif + SIMDE_AVX512_ALIGN simde_float32 f32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN simde_float64 f64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int_fast32_t i32f SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint_fast32_t u32f SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #else + SIMDE_AVX512_ALIGN int8_t i8[64]; + SIMDE_AVX512_ALIGN int16_t i16[32]; + SIMDE_AVX512_ALIGN int32_t i32[16]; + SIMDE_AVX512_ALIGN int64_t i64[8]; + SIMDE_AVX512_ALIGN uint8_t u8[64]; + SIMDE_AVX512_ALIGN uint16_t u16[32]; + SIMDE_AVX512_ALIGN uint32_t u32[16]; + SIMDE_AVX512_ALIGN uint64_t u64[8]; + SIMDE_AVX512_ALIGN int_fast32_t i32f[64 / sizeof(int_fast32_t)]; + SIMDE_AVX512_ALIGN uint_fast32_t u32f[64 / sizeof(uint_fast32_t)]; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_AVX512_ALIGN simde_int128 i128[4]; + SIMDE_AVX512_ALIGN simde_uint128 u128[4]; + #endif + SIMDE_AVX512_ALIGN simde_float32 f32[16]; + SIMDE_AVX512_ALIGN simde_float64 f64[8]; + #endif + + SIMDE_AVX512_ALIGN simde__m128_private m128_private[4]; + SIMDE_AVX512_ALIGN simde__m128 m128[4]; + SIMDE_AVX512_ALIGN simde__m256_private m256_private[2]; + SIMDE_AVX512_ALIGN simde__m256 m256[2]; + + #if defined(SIMDE_X86_AVX512F_NATIVE) + SIMDE_AVX512_ALIGN __m512 n; + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) altivec_u8[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) altivec_u16[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed char) altivec_i8[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed short) altivec_i16[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed int) altivec_i32[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(float) altivec_f32[4]; + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long) altivec_u64[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed long long) altivec_i64[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(double) altivec_f64[4]; + #endif + #endif +} simde__m512_private; + +typedef union { + #if defined(SIMDE_VECTOR_SUBSCRIPT) + SIMDE_AVX512_ALIGN int8_t i8 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int16_t i16 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int32_t i32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int64_t i64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint8_t u8 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint16_t u16 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint32_t u32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint64_t u64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_AVX512_ALIGN simde_int128 i128 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN simde_uint128 u128 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #endif + SIMDE_AVX512_ALIGN simde_float32 f32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN simde_float64 f64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int_fast32_t i32f SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint_fast32_t u32f SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #else + SIMDE_AVX512_ALIGN int8_t i8[64]; + SIMDE_AVX512_ALIGN int16_t i16[32]; + SIMDE_AVX512_ALIGN int32_t i32[16]; + SIMDE_AVX512_ALIGN int64_t i64[8]; + SIMDE_AVX512_ALIGN uint8_t u8[64]; + SIMDE_AVX512_ALIGN uint16_t u16[32]; + SIMDE_AVX512_ALIGN uint32_t u32[16]; + SIMDE_AVX512_ALIGN uint64_t u64[8]; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_AVX512_ALIGN simde_int128 i128[4]; + SIMDE_AVX512_ALIGN simde_uint128 u128[4]; + #endif + SIMDE_AVX512_ALIGN simde_float32 f32[16]; + SIMDE_AVX512_ALIGN simde_float64 f64[8]; + SIMDE_AVX512_ALIGN int_fast32_t i32f[64 / sizeof(int_fast32_t)]; + SIMDE_AVX512_ALIGN uint_fast32_t u32f[64 / sizeof(uint_fast32_t)]; + #endif + + SIMDE_AVX512_ALIGN simde__m128d_private m128d_private[4]; + SIMDE_AVX512_ALIGN simde__m128d m128d[4]; + SIMDE_AVX512_ALIGN simde__m256d_private m256d_private[2]; + SIMDE_AVX512_ALIGN simde__m256d m256d[2]; + + #if defined(SIMDE_X86_AVX512F_NATIVE) + SIMDE_AVX512_ALIGN __m512d n; + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) altivec_u8[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) altivec_u16[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed char) altivec_i8[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed short) altivec_i16[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed int) altivec_i32[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(float) altivec_f32[4]; + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long) altivec_u64[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed long long) altivec_i64[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(double) altivec_f64[4]; + #endif + #endif +} simde__m512d_private; + +typedef union { + #if defined(SIMDE_VECTOR_SUBSCRIPT) + SIMDE_AVX512_ALIGN int8_t i8 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int16_t i16 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int32_t i32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int64_t i64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint8_t u8 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint16_t u16 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint32_t u32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint64_t u64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_AVX512_ALIGN simde_int128 i128 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN simde_uint128 u128 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #endif + #if defined(SIMDE_FLOAT16_VECTOR) + SIMDE_ALIGN_TO_16 simde_float16 f16 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #else + SIMDE_AVX512_ALIGN simde_float16 f16[32]; + #endif + SIMDE_AVX512_ALIGN simde_float32 f32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN simde_float64 f64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int_fast32_t i32f SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint_fast32_t u32f SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #else + SIMDE_AVX512_ALIGN int8_t i8[64]; + SIMDE_AVX512_ALIGN int16_t i16[32]; + SIMDE_AVX512_ALIGN int32_t i32[16]; + SIMDE_AVX512_ALIGN int64_t i64[8]; + SIMDE_AVX512_ALIGN uint8_t u8[64]; + SIMDE_AVX512_ALIGN uint16_t u16[32]; + SIMDE_AVX512_ALIGN uint32_t u32[16]; + SIMDE_AVX512_ALIGN uint64_t u64[8]; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_AVX512_ALIGN simde_int128 i128[4]; + SIMDE_AVX512_ALIGN simde_uint128 u128[4]; + #endif + SIMDE_AVX512_ALIGN simde_float16 f16[32]; + SIMDE_AVX512_ALIGN simde_float32 f32[16]; + SIMDE_AVX512_ALIGN simde_float64 f64[8]; + SIMDE_AVX512_ALIGN int_fast32_t i32f[64 / sizeof(int_fast32_t)]; + SIMDE_AVX512_ALIGN uint_fast32_t u32f[64 / sizeof(uint_fast32_t)]; + #endif + + SIMDE_AVX512_ALIGN simde__m128d_private m128d_private[4]; + SIMDE_AVX512_ALIGN simde__m128d m128d[4]; + SIMDE_AVX512_ALIGN simde__m256d_private m256d_private[2]; + SIMDE_AVX512_ALIGN simde__m256d m256d[2]; + + #if defined(SIMDE_X86_AVX512FP16_NATIVE) + SIMDE_AVX512_ALIGN __m512h n; + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) altivec_u8[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) altivec_u16[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed char) altivec_i8[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed short) altivec_i16[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed int) altivec_i32[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(float) altivec_f32[4]; + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long) altivec_u64[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed long long) altivec_i64[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(double) altivec_f64[4]; + #endif + #endif +} simde__m512h_private; + + +typedef union { + #if defined(SIMDE_VECTOR_SUBSCRIPT) + SIMDE_AVX512_ALIGN int8_t i8 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int16_t i16 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int32_t i32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int64_t i64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint8_t u8 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint16_t u16 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint32_t u32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint64_t u64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_AVX512_ALIGN simde_int128 i128 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN simde_uint128 u128 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #endif + SIMDE_AVX512_ALIGN simde_float32 f32 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN simde_float64 f64 SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN int_fast32_t i32f SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + SIMDE_AVX512_ALIGN uint_fast32_t u32f SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #else + SIMDE_AVX512_ALIGN int8_t i8[64]; + SIMDE_AVX512_ALIGN int16_t i16[32]; + SIMDE_AVX512_ALIGN int32_t i32[16]; + SIMDE_AVX512_ALIGN int64_t i64[8]; + SIMDE_AVX512_ALIGN uint8_t u8[64]; + SIMDE_AVX512_ALIGN uint16_t u16[32]; + SIMDE_AVX512_ALIGN uint32_t u32[16]; + SIMDE_AVX512_ALIGN uint64_t u64[8]; + SIMDE_AVX512_ALIGN int_fast32_t i32f[64 / sizeof(int_fast32_t)]; + SIMDE_AVX512_ALIGN uint_fast32_t u32f[64 / sizeof(uint_fast32_t)]; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_AVX512_ALIGN simde_int128 i128[4]; + SIMDE_AVX512_ALIGN simde_uint128 u128[4]; + #endif + SIMDE_AVX512_ALIGN simde_float32 f32[16]; + SIMDE_AVX512_ALIGN simde_float64 f64[8]; + #endif + + SIMDE_AVX512_ALIGN simde__m128i_private m128i_private[4]; + SIMDE_AVX512_ALIGN simde__m128i m128i[4]; + SIMDE_AVX512_ALIGN simde__m256i_private m256i_private[2]; + SIMDE_AVX512_ALIGN simde__m256i m256i[2]; + + #if defined(SIMDE_X86_AVX512F_NATIVE) + SIMDE_AVX512_ALIGN __m512i n; + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) altivec_u8[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) altivec_u16[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed char) altivec_i8[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed short) altivec_i16[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed int) altivec_i32[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(float) altivec_f32[4]; + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long) altivec_u64[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed long long) altivec_i64[4]; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(double) altivec_f64[4]; + #endif + #endif +} simde__m512i_private; + +/* Intel uses the same header (immintrin.h) for everything AVX and + * later. If native aliases are enabled, and the machine has native + * support for AVX imintrin.h will already have been included, which + * means simde__m512* will already have been defined. So, even + * if the machine doesn't support AVX512F we need to use the native + * type; it has already been defined. + * + * However, we also can't just assume that including immintrin.h does + * actually define these. It could be a compiler which supports AVX + * but not AVX512F, such as GCC < 4.9 or VS < 2017. That's why we + * check to see if _MM_CMPINT_GE is defined; it's part of AVX512F, + * so we assume that if it's present AVX-512F has already been + * declared. + * + * Note that the choice of _MM_CMPINT_GE is deliberate; while GCC + * uses the preprocessor to define all the _MM_CMPINT_* members, + * in most compilers they are simply normal enum members. However, + * all compilers I've looked at use an object-like macro for + * _MM_CMPINT_GE, which is defined to _MM_CMPINT_NLT. _MM_CMPINT_NLT + * is included in case a compiler does the reverse, though I haven't + * run into one which does. + * + * As for the ICC check, unlike other compilers, merely using the + * AVX-512 types causes ICC to generate AVX-512 instructions. */ +#if (defined(_MM_CMPINT_GE) || defined(_MM_CMPINT_NLT)) && \ + (defined(SIMDE_X86_AVX512F_NATIVE) || \ + !(defined(HEDLEY_INTEL_VERSION) || (defined(HEDLEY_MSVC_VERSION) && !defined(__clang__)))) + typedef __m512 simde__m512; + typedef __m512i simde__m512i; + typedef __m512d simde__m512d; + + typedef __mmask8 simde__mmask8; + typedef __mmask16 simde__mmask16; +#else + #if defined(SIMDE_VECTOR_SUBSCRIPT) + typedef simde_float32 simde__m512 SIMDE_AVX512_ALIGN SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + typedef int_fast32_t simde__m512i SIMDE_AVX512_ALIGN SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + typedef simde_float64 simde__m512d SIMDE_AVX512_ALIGN SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #else + typedef simde__m512_private simde__m512; + typedef simde__m512i_private simde__m512i; + typedef simde__m512d_private simde__m512d; + #endif + + typedef uint8_t simde__mmask8; + typedef uint16_t simde__mmask16; +#endif + +#if (defined(_AVX512BF16INTRIN_H_INCLUDED) || defined(__AVX512BF16INTRIN_H)) && (defined(SIMDE_X86_AVX512BF16_NATIVE) || !defined(HEDLEY_INTEL_VERSION)) + typedef __m128bh simde__m128bh; + typedef __m256bh simde__m256bh; + typedef __m512bh simde__m512bh; +#else + #if defined(SIMDE_VECTOR_SUBSCRIPT) + typedef simde_float32 simde__m128bh SIMDE_ALIGN_TO_16 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + typedef simde_float32 simde__m256bh SIMDE_ALIGN_TO_32 SIMDE_VECTOR(32) SIMDE_MAY_ALIAS; + typedef simde_float32 simde__m512bh SIMDE_AVX512_ALIGN SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #else + typedef simde__m128bh_private simde__m128bh; + typedef simde__m256bh_private simde__m256bh; + typedef simde__m512bh_private simde__m512bh; + #endif +#endif + +#if defined(SIMDE_X86_AVX512FP16_NATIVE) + typedef __m512h simde__m512h; +#else + #if defined(SIMDE_VECTOR_SUBSCRIPT) && defined(SIMDE_FLOAT16_VECTOR) + typedef simde_float16 simde__m512h SIMDE_AVX512_ALIGN SIMDE_VECTOR(64) SIMDE_MAY_ALIAS; + #else + typedef simde__m512h_private simde__m512h; + #endif +#endif + +/* These are really part of AVX-512VL / AVX-512BW (in GCC __mmask32 is + * in avx512vlintrin.h and __mmask64 is in avx512bwintrin.h, in clang + * both are in avx512bwintrin.h), not AVX-512F. However, we don't have + * a good (not-compiler-specific) way to detect if these headers have + * been included. In compilers which support AVX-512F but not + * AVX-512BW/VL (e.g., GCC 4.9) we need typedefs since __mmask{32,64) + * won't exist. + * + * AFAICT __mmask{32,64} are always just typedefs to uint{32,64}_t + * in all compilers, so it's safe to use these instead of typedefs to + * __mmask{16,32}. If you run into a problem with this please file an + * issue and we'll try to figure out a work-around. */ +typedef uint32_t simde__mmask32; +typedef uint64_t simde__mmask64; +#if !defined(__mmask16) && defined(SIMDE_ENABLE_NATIVE_ALIASES) + #if !defined(HEDLEY_INTEL_VERSION) + typedef uint16_t __mmask16; + #else + #define __mmask16 uint16_t; + #endif +#endif +#if !defined(__mmask32) && defined(SIMDE_ENABLE_NATIVE_ALIASES) + #if !defined(HEDLEY_INTEL_VERSION) + typedef uint32_t __mmask32; + #else + #define __mmask32 uint32_t; + #endif +#endif +#if !defined(__mmask64) && defined(SIMDE_ENABLE_NATIVE_ALIASES) + #if !defined(HEDLEY_INTEL_VERSION) + #if defined(HEDLEY_GCC_VERSION) + typedef unsigned long long __mmask64; + #else + typedef uint64_t __mmask64; + #endif + #else + #define __mmask64 uint64_t; + #endif +#endif + +#if !defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_ENABLE_NATIVE_ALIASES) + #if !defined(HEDLEY_INTEL_VERSION) + typedef simde__m512 __m512; + typedef simde__m512i __m512i; + typedef simde__m512d __m512d; + #else + #define __m512 simde__m512 + #define __m512i simde__m512i + #define __m512d simde__m512d + #endif +#endif + +#if !defined(SIMDE_X86_AVX512BF16_NATIVE) && defined(SIMDE_ENABLE_NATIVE_ALIASES) + #if !defined(HEDLEY_INTEL_VERSION) + typedef simde__m128bh __m128bh; + typedef simde__m256bh __m256bh; + typedef simde__m512bh __m512bh; + #else + #define __m128bh simde__m128bh + #define __m256bh simde__m256bh + #define __m512bh simde__m512bh + #endif +#endif + +#if !defined(SIMDE_X86_AVX512FP16_NATIVE) && defined(SIMDE_ENABLE_NATIVE_ALIASES) + #if !defined(HEDLEY_INTEL_VERSION) + //typedef simde__m128h __m128h; + //typedef simde__m256h __m256h; + typedef simde__m512h __m512h; + #else + //#define __m128h simde__m128h + //#define __m256h simde__m256h + #define __m512h simde__m512h + #endif +#endif + +HEDLEY_STATIC_ASSERT(16 == sizeof(simde__m128bh), "simde__m128bh size incorrect"); +HEDLEY_STATIC_ASSERT(16 == sizeof(simde__m128bh_private), "simde__m128bh_private size incorrect"); +HEDLEY_STATIC_ASSERT(32 == sizeof(simde__m256bh), "simde__m256bh size incorrect"); +HEDLEY_STATIC_ASSERT(32 == sizeof(simde__m256bh_private), "simde__m256bh_private size incorrect"); +HEDLEY_STATIC_ASSERT(64 == sizeof(simde__m512bh), "simde__m512bh size incorrect"); +HEDLEY_STATIC_ASSERT(64 == sizeof(simde__m512bh_private), "simde__m512bh_private size incorrect"); +HEDLEY_STATIC_ASSERT(64 == sizeof(simde__m512), "simde__m512 size incorrect"); +HEDLEY_STATIC_ASSERT(64 == sizeof(simde__m512_private), "simde__m512_private size incorrect"); +HEDLEY_STATIC_ASSERT(64 == sizeof(simde__m512i), "simde__m512i size incorrect"); +HEDLEY_STATIC_ASSERT(64 == sizeof(simde__m512i_private), "simde__m512i_private size incorrect"); +HEDLEY_STATIC_ASSERT(64 == sizeof(simde__m512d), "simde__m512d size incorrect"); +HEDLEY_STATIC_ASSERT(64 == sizeof(simde__m512d_private), "simde__m512d_private size incorrect"); +HEDLEY_STATIC_ASSERT(64 == sizeof(simde__m512h), "simde__m512h size incorrect"); +HEDLEY_STATIC_ASSERT(64 == sizeof(simde__m512h_private), "simde__m512h_private size incorrect"); +#if defined(SIMDE_CHECK_ALIGNMENT) && defined(SIMDE_ALIGN_OF) +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m128bh) == 16, "simde__m128bh is not 16-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m128bh_private) == 16, "simde__m128bh_private is not 16-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m256bh) == 32, "simde__m256bh is not 16-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m256bh_private) == 32, "simde__m256bh_private is not 16-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m512bh) == 32, "simde__m512bh is not 32-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m512bh_private) == 32, "simde__m512bh_private is not 32-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m512) == 32, "simde__m512 is not 32-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m512_private) == 32, "simde__m512_private is not 32-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m512i) == 32, "simde__m512i is not 32-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m512i_private) == 32, "simde__m512i_private is not 32-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m512d) == 32, "simde__m512d is not 32-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m512d_private) == 32, "simde__m512d_private is not 32-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m512h) == 32, "simde__m512h is not 32-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m512h_private) == 32, "simde__m512h_private is not 32-byte aligned"); +#endif + +#define SIMDE_MM_CMPINT_EQ 0 +#define SIMDE_MM_CMPINT_LT 1 +#define SIMDE_MM_CMPINT_LE 2 +#define SIMDE_MM_CMPINT_FALSE 3 +#define SIMDE_MM_CMPINT_NE 4 +#define SIMDE_MM_CMPINT_NLT 5 +#define SIMDE_MM_CMPINT_NLE 6 +#define SIMDE_MM_CMPINT_TRUE 7 +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && !defined(_MM_CMPINT_EQ) +#define _MM_CMPINT_EQ SIMDE_MM_CMPINT_EQ +#define _MM_CMPINT_LT SIMDE_MM_CMPINT_LT +#define _MM_CMPINT_LE SIMDE_MM_CMPINT_LE +#define _MM_CMPINT_FALSE SIMDE_MM_CMPINT_FALSE +#define _MM_CMPINT_NE SIMDE_MM_CMPINT_NE +#define _MM_CMPINT_NLT SIMDE_MM_CMPINT_NLT +#define _MM_CMPINT_NLE SIMDE_MM_CMPINT_NLE +#define _MM_CMPINT_TRUE SIMDE_CMPINT_TRUE +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128bh +simde__m128bh_from_private(simde__m128bh_private v) { + simde__m128bh r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128bh_private +simde__m128bh_to_private(simde__m128bh v) { + simde__m128bh_private r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256bh +simde__m256bh_from_private(simde__m256bh_private v) { + simde__m256bh r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256bh_private +simde__m256bh_to_private(simde__m256bh v) { + simde__m256bh_private r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512bh +simde__m512bh_from_private(simde__m512bh_private v) { + simde__m512bh r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512bh_private +simde__m512bh_to_private(simde__m512bh v) { + simde__m512bh_private r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde__m512_from_private(simde__m512_private v) { + simde__m512 r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512_private +simde__m512_to_private(simde__m512 v) { + simde__m512_private r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde__m512i_from_private(simde__m512i_private v) { + simde__m512i r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i_private +simde__m512i_to_private(simde__m512i v) { + simde__m512i_private r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde__m512d_from_private(simde__m512d_private v) { + simde__m512d r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d_private +simde__m512d_to_private(simde__m512d v) { + simde__m512d_private r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512h +simde__m512h_from_private(simde__m512h_private v) { + simde__m512h r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512h_private +simde__m512h_to_private(simde__m512h v) { + simde__m512h_private r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_TYPES_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/unpackhi.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/unpackhi.h new file mode 100644 index 00000000..a67a1531 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/unpackhi.h @@ -0,0 +1,753 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_UNPACKHI_H) +#define SIMDE_X86_AVX512_UNPACKHI_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_unpackhi_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_unpackhi_epi8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.i8 = SIMDE_SHUFFLE_VECTOR_(8, 64, a_.i8, b_.i8, + 8, 72, 9, 73, 10, 74, 11, 75, + 12, 76, 13, 77, 14, 78, 15, 79, + 24, 88, 25, 89, 26, 90, 27, 91, + 28, 92, 29, 93, 30, 94, 31, 95, + 40, 104, 41, 105, 42, 106, 43, 107, + 44, 108, 45, 109, 46, 110, 47, 111, + 56, 120, 57, 121, 58, 122, 59, 123, + 60, 124, 61, 125, 62, 126, 63, 127); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_unpackhi_epi8(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_unpackhi_epi8(a_.m256i[1], b_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0]) / 2) ; i++) { + r_.i8[2 * i] = a_.i8[i + 8 + ~(~i | 7)]; + r_.i8[2 * i + 1] = b_.i8[i + 8 + ~(~i | 7)]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_unpackhi_epi8 + #define _mm512_unpackhi_epi8(a, b) simde_mm512_unpackhi_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_unpackhi_epi8(simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_unpackhi_epi8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_unpackhi_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_unpackhi_epi8 + #define _mm512_mask_unpackhi_epi8(src, k, a, b) simde_mm512_mask_unpackhi_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_unpackhi_epi8(simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_unpackhi_epi8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_unpackhi_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_unpackhi_epi8 + #define _mm512_maskz_unpackhi_epi8(k, a, b) simde_mm512_maskz_unpackhi_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_unpackhi_epi8(simde__m256i src, simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_unpackhi_epi8(src, k, a, b); + #else + return simde_mm256_mask_mov_epi8(src, k, simde_mm256_unpackhi_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_unpackhi_epi8 + #define _mm256_mask_unpackhi_epi8(src, k, a, b) simde_mm256_mask_unpackhi_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_unpackhi_epi8(simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_unpackhi_epi8(k, a, b); + #else + return simde_mm256_maskz_mov_epi8(k, simde_mm256_unpackhi_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_unpackhi_epi8 + #define _mm256_maskz_unpackhi_epi8(k, a, b) simde_mm256_maskz_unpackhi_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_unpackhi_epi8(simde__m128i src, simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_unpackhi_epi8(src, k, a, b); + #else + return simde_mm_mask_mov_epi8(src, k, simde_mm_unpackhi_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_unpackhi_epi8 + #define _mm_mask_unpackhi_epi8(src, k, a, b) simde_mm_mask_unpackhi_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_unpackhi_epi8(simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_unpackhi_epi8(k, a, b); + #else + return simde_mm_maskz_mov_epi8(k, simde_mm_unpackhi_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_unpackhi_epi8 + #define _mm_maskz_unpackhi_epi8(k, a, b) simde_mm_maskz_unpackhi_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_unpackhi_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_unpackhi_epi16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.i16 =SIMDE_SHUFFLE_VECTOR_(16, 64, a_.i16, b_.i16, + 4, 36, 5, 37, 6, 38, 7, 39, 12, 44, 13, 45, 14, 46, 15, 47, + 20, 52, 21, 53, 22, 54, 23, 55, 28, 60, 29, 61, 30, 62, 31, 63); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_unpackhi_epi16(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_unpackhi_epi16(a_.m256i[1], b_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0]) / 2) ; i++) { + r_.i16[2 * i] = a_.i16[i + 4 + ~(~i | 3)]; + r_.i16[2 * i + 1] = b_.i16[i + 4 + ~(~i | 3)]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_unpackhi_epi16 + #define _mm512_unpackhi_epi16(a, b) simde_mm512_unpackhi_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_unpackhi_epi16(simde__m512i src, simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_unpackhi_epi16(src, k, a, b); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_unpackhi_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_unpackhi_epi16 + #define _mm512_mask_unpackhi_epi16(src, k, a, b) simde_mm512_mask_unpackhi_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_unpackhi_epi16(simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_unpackhi_epi16(k, a, b); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_unpackhi_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_unpackhi_epi16 + #define _mm512_maskz_unpackhi_epi16(k, a, b) simde_mm512_maskz_unpackhi_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_unpackhi_epi16(simde__m256i src, simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_unpackhi_epi16(src, k, a, b); + #else + return simde_mm256_mask_mov_epi16(src, k, simde_mm256_unpackhi_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_unpackhi_epi16 + #define _mm256_mask_unpackhi_epi16(src, k, a, b) simde_mm256_mask_unpackhi_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_unpackhi_epi16(simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_unpackhi_epi16(k, a, b); + #else + return simde_mm256_maskz_mov_epi16(k, simde_mm256_unpackhi_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_unpackhi_epi16 + #define _mm256_maskz_unpackhi_epi16(k, a, b) simde_mm256_maskz_unpackhi_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_unpackhi_epi16(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_unpackhi_epi16(src, k, a, b); + #else + return simde_mm_mask_mov_epi16(src, k, simde_mm_unpackhi_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_unpackhi_epi16 + #define _mm_mask_unpackhi_epi16(src, k, a, b) simde_mm_mask_unpackhi_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_unpackhi_epi16(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_unpackhi_epi16(k, a, b); + #else + return simde_mm_maskz_mov_epi16(k, simde_mm_unpackhi_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_unpackhi_epi16 + #define _mm_maskz_unpackhi_epi16(k, a, b) simde_mm_maskz_unpackhi_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_unpackhi_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_unpackhi_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.i32 = SIMDE_SHUFFLE_VECTOR_(32, 64, a_.i32, b_.i32, + 2, 18, 3 , 19, 6, 22, 7, 23, + 10, 26, 11, 27, 14, 30, 15, 31); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_unpackhi_epi32(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_unpackhi_epi32(a_.m256i[1], b_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0]) / 2) ; i++) { + r_.i32[2 * i] = a_.i32[i + 2 + ~(~i | 1)]; + r_.i32[2 * i + 1] = b_.i32[i + 2 + ~(~i | 1)]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_unpackhi_epi32 + #define _mm512_unpackhi_epi32(a, b) simde_mm512_unpackhi_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_unpackhi_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_unpackhi_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_unpackhi_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_unpackhi_epi32 + #define _mm512_mask_unpackhi_epi32(src, k, a, b) simde_mm512_mask_unpackhi_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_unpackhi_epi32(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_unpackhi_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_unpackhi_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_unpackhi_epi32 + #define _mm512_maskz_unpackhi_epi32(k, a, b) simde_mm512_maskz_unpackhi_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_unpackhi_epi32(simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_unpackhi_epi32(src, k, a, b); + #else + return simde_mm256_mask_mov_epi32(src, k, simde_mm256_unpackhi_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_unpackhi_epi32 + #define _mm256_mask_unpackhi_epi32(src, k, a, b) simde_mm256_mask_unpackhi_epi32(src, k, a, b) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_unpackhi_epi32(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_unpackhi_epi32(k, a, b); + #else + return simde_mm256_maskz_mov_epi32(k, simde_mm256_unpackhi_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_unpackhi_epi32 + #define _mm256_maskz_unpackhi_epi32(k, a, b) simde_mm256_maskz_unpackhi_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_unpackhi_epi32(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_unpackhi_epi32(src, k, a, b); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_unpackhi_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_unpackhi_epi32 + #define _mm_mask_unpackhi_epi32(src, k, a, b) simde_mm_mask_unpackhi_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_unpackhi_epi32(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_unpackhi_epi32(k, a, b); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_unpackhi_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_unpackhi_epi32 + #define _mm_maskz_unpackhi_epi32(k, a, b) simde_mm_maskz_unpackhi_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_unpackhi_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_unpackhi_epi64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.i64 = SIMDE_SHUFFLE_VECTOR_(64, 64, a_.i64, b_.i64, 1, 9, 3, 11, 5, 13, 7, 15); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_unpackhi_epi64(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_unpackhi_epi64(a_.m256i[1], b_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0]) / 2) ; i++) { + r_.i64[2 * i] = a_.i64[2 * i + 1]; + r_.i64[2 * i + 1] = b_.i64[2 * i + 1]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_unpackhi_epi64 + #define _mm512_unpackhi_epi64(a, b) simde_mm512_unpackhi_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_unpackhi_epi64(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_unpackhi_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_unpackhi_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_unpackhi_epi64 + #define _mm512_mask_unpackhi_epi64(src, k, a, b) simde_mm512_mask_unpackhi_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_unpackhi_epi64(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_unpackhi_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_unpackhi_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_unpackhi_epi64 + #define _mm512_maskz_unpackhi_epi64(k, a, b) simde_mm512_maskz_unpackhi_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_unpackhi_epi64(simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_unpackhi_epi64(src, k, a, b); + #else + return simde_mm256_mask_mov_epi64(src, k, simde_mm256_unpackhi_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_unpackhi_epi64 + #define _mm256_mask_unpackhi_epi64(src, k, a, b) simde_mm256_mask_unpackhi_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_unpackhi_epi64(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_unpackhi_epi64(k, a, b); + #else + return simde_mm256_maskz_mov_epi64(k, simde_mm256_unpackhi_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_unpackhi_epi64 + #define _mm256_maskz_unpackhi_epi64(k, a, b) simde_mm256_maskz_unpackhi_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_unpackhi_epi64(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_unpackhi_epi64(src, k, a, b); + #else + return simde_mm_mask_mov_epi64(src, k, simde_mm_unpackhi_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_unpackhi_epi64 + #define _mm_mask_unpackhi_epi64(src, k, a, b) simde_mm_mask_unpackhi_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_unpackhi_epi64(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_unpackhi_epi64(k, a, b); + #else + return simde_mm_maskz_mov_epi64(k, simde_mm_unpackhi_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_unpackhi_epi64 + #define _mm_maskz_unpackhi_epi64(k, a, b) simde_mm_maskz_unpackhi_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_unpackhi_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_unpackhi_ps(a, b); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 64, a_.f32, b_.f32, + 2, 18, 3 , 19, 6, 22, 7, 23, + 10, 26, 11, 27, 14, 30, 15, 31); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256[0] = simde_mm256_unpackhi_ps(a_.m256[0], b_.m256[0]); + r_.m256[1] = simde_mm256_unpackhi_ps(a_.m256[1], b_.m256[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0]) / 2) ; i++) { + r_.f32[2 * i] = a_.f32[i + 2 + ~(~i | 1)]; + r_.f32[2 * i + 1] = b_.f32[i + 2 + ~(~i | 1)]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_unpackhi_ps + #define _mm512_unpackhi_ps(a, b) simde_mm512_unpackhi_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_unpackhi_ps(simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_unpackhi_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_unpackhi_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_unpackhi_ps + #define _mm512_mask_unpackhi_ps(src, k, a, b) simde_mm512_mask_unpackhi_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_unpackhi_ps(simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_unpackhi_ps(k, a, b); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_unpackhi_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_unpackhi_ps + #define _mm512_maskz_unpackhi_ps(k, a, b) simde_mm512_maskz_unpackhi_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask_unpackhi_ps(simde__m256 src, simde__mmask8 k, simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_unpackhi_ps(src, k, a, b); + #else + return simde_mm256_mask_mov_ps(src, k, simde_mm256_unpackhi_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_unpackhi_ps + #define _mm256_mask_unpackhi_ps(src, k, a, b) simde_mm256_mask_unpackhi_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_maskz_unpackhi_ps(simde__mmask8 k, simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_unpackhi_ps(k, a, b); + #else + return simde_mm256_maskz_mov_ps(k, simde_mm256_unpackhi_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_unpackhi_ps + #define _mm256_maskz_unpackhi_ps(k, a, b) simde_mm256_maskz_unpackhi_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mask_unpackhi_ps(simde__m128 src, simde__mmask8 k, simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_unpackhi_ps(src, k, a, b); + #else + return simde_mm_mask_mov_ps(src, k, simde_mm_unpackhi_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_unpackhi_ps + #define _mm_mask_unpackhi_ps(src, k, a, b) simde_mm_mask_unpackhi_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_maskz_unpackhi_ps(simde__mmask8 k, simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_unpackhi_ps(k, a, b); + #else + return simde_mm_maskz_mov_ps(k, simde_mm_unpackhi_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_unpackhi_ps + #define _mm_maskz_unpackhi_ps(k, a, b) simde_mm_maskz_unpackhi_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_unpackhi_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_unpackhi_pd(a, b); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 64, a_.f64, b_.f64, 1, 9, 3, 11, 5, 13, 7, 15); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256d[0] = simde_mm256_unpackhi_pd(a_.m256d[0], b_.m256d[0]); + r_.m256d[1] = simde_mm256_unpackhi_pd(a_.m256d[1], b_.m256d[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0]) / 2) ; i++) { + r_.f64[2 * i] = a_.f64[2 * i + 1]; + r_.f64[2 * i + 1] = b_.f64[2 * i + 1]; + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_unpackhi_pd + #define _mm512_unpackhi_pd(a, b) simde_mm512_unpackhi_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_unpackhi_pd(simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_unpackhi_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_unpackhi_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_unpackhi_pd + #define _mm512_mask_unpackhi_pd(src, k, a, b) simde_mm512_mask_unpackhi_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_unpackhi_pd(simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_unpackhi_pd(k, a, b); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_unpackhi_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_unpackhi_pd + #define _mm512_maskz_unpackhi_pd(k, a, b) simde_mm512_maskz_unpackhi_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mask_unpackhi_pd(simde__m256d src, simde__mmask8 k, simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_unpackhi_pd(src, k, a, b); + #else + return simde_mm256_mask_mov_pd(src, k, simde_mm256_unpackhi_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_unpackhi_pd + #define _mm256_mask_unpackhi_pd(src, k, a, b) simde_mm256_mask_unpackhi_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_maskz_unpackhi_pd(simde__mmask8 k, simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_unpackhi_pd(k, a, b); + #else + return simde_mm256_maskz_mov_pd(k, simde_mm256_unpackhi_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_unpackhi_pd + #define _mm256_maskz_unpackhi_pd(k, a, b) simde_mm256_maskz_unpackhi_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mask_unpackhi_pd(simde__m128d src, simde__mmask8 k, simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_unpackhi_pd(src, k, a, b); + #else + return simde_mm_mask_mov_pd(src, k, simde_mm_unpackhi_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_unpackhi_pd + #define _mm_mask_unpackhi_pd(src, k, a, b) simde_mm_mask_unpackhi_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_maskz_unpackhi_pd(simde__mmask8 k, simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_unpackhi_pd(k, a, b); + #else + return simde_mm_maskz_mov_pd(k, simde_mm_unpackhi_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_unpackhi_pd + #define _mm_maskz_unpackhi_pd(k, a, b) simde_mm_maskz_unpackhi_pd(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_UNPACKHI_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/unpacklo.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/unpacklo.h new file mode 100644 index 00000000..efaa61e9 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/unpacklo.h @@ -0,0 +1,752 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_UNPACKLO_H) +#define SIMDE_X86_AVX512_UNPACKLO_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_unpacklo_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_unpacklo_epi8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.i8 = SIMDE_SHUFFLE_VECTOR_(8, 64, a_.i8, b_.i8, + 0, 64, 1, 65, 2, 66, 3, 67, + 4, 68, 5, 69, 6, 70, 7, 71, + 16, 80, 17, 81, 18, 82, 19, 83, + 20, 84, 21, 85, 22, 86, 23, 87, + 32, 96, 33, 97, 34, 98, 35, 99, + 36, 100, 37, 101, 38, 102, 39, 103, + 48, 112, 49, 113, 50, 114, 51, 115, + 52, 116, 53, 117, 54, 118, 55, 119); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_unpacklo_epi8(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_unpacklo_epi8(a_.m256i[1], b_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0]) / 2) ; i++) { + r_.i8[2 * i] = a_.i8[i + ~(~i | 7)]; + r_.i8[2 * i + 1] = b_.i8[i + ~(~i | 7)]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_unpacklo_epi8 + #define _mm512_unpacklo_epi8(a, b) simde_mm512_unpacklo_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_unpacklo_epi8(simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_unpacklo_epi8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_unpacklo_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_unpacklo_epi8 + #define _mm512_mask_unpacklo_epi8(src, k, a, b) simde_mm512_mask_unpacklo_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_unpacklo_epi8(simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_unpacklo_epi8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_unpacklo_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_unpacklo_epi8 + #define _mm512_maskz_unpacklo_epi8(k, a, b) simde_mm512_maskz_unpacklo_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_unpacklo_epi8(simde__m256i src, simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_unpacklo_epi8(src, k, a, b); + #else + return simde_mm256_mask_mov_epi8(src, k, simde_mm256_unpacklo_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_unpacklo_epi8 + #define _mm256_mask_unpacklo_epi8(src, k, a, b) simde_mm256_mask_unpacklo_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_unpacklo_epi8(simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_unpacklo_epi8(k, a, b); + #else + return simde_mm256_maskz_mov_epi8(k, simde_mm256_unpacklo_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_unpacklo_epi8 + #define _mm256_maskz_unpacklo_epi8(k, a, b) simde_mm256_maskz_unpacklo_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_unpacklo_epi8(simde__m128i src, simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_unpacklo_epi8(src, k, a, b); + #else + return simde_mm_mask_mov_epi8(src, k, simde_mm_unpacklo_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_unpacklo_epi8 + #define _mm_mask_unpacklo_epi8(src, k, a, b) simde_mm_mask_unpacklo_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_unpacklo_epi8(simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_unpacklo_epi8(k, a, b); + #else + return simde_mm_maskz_mov_epi8(k, simde_mm_unpacklo_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_unpacklo_epi8 + #define _mm_maskz_unpacklo_epi8(k, a, b) simde_mm_maskz_unpacklo_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_unpacklo_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_unpacklo_epi16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.i16 = SIMDE_SHUFFLE_VECTOR_(16, 64, a_.i16, b_.i16, + 0, 32, 1, 33, 2, 34, 3, 35, 8, 40, 9, 41, 10, 42, 11, 43, + 16, 48, 17, 49, 18, 50, 19, 51, 24, 56, 25, 57, 26, 58, 27, 59); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_unpacklo_epi16(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_unpacklo_epi16(a_.m256i[1], b_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0]) / 2) ; i++) { + r_.i16[2 * i] = a_.i16[i + ~(~i | 3)]; + r_.i16[2 * i + 1] = b_.i16[i + ~(~i | 3)]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_unpacklo_epi16 + #define _mm512_unpacklo_epi16(a, b) simde_mm512_unpacklo_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_unpacklo_epi16(simde__m512i src, simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_mask_unpacklo_epi16(src, k, a, b); + #else + return simde_mm512_mask_mov_epi16(src, k, simde_mm512_unpacklo_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_unpacklo_epi16 + #define _mm512_mask_unpacklo_epi16(src, k, a, b) simde_mm512_mask_unpacklo_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_unpacklo_epi16(simde__mmask32 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + return _mm512_maskz_unpacklo_epi16(k, a, b); + #else + return simde_mm512_maskz_mov_epi16(k, simde_mm512_unpacklo_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_unpacklo_epi16 + #define _mm512_maskz_unpacklo_epi16(k, a, b) simde_mm512_maskz_unpacklo_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_unpacklo_epi16(simde__m256i src, simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_unpacklo_epi16(src, k, a, b); + #else + return simde_mm256_mask_mov_epi16(src, k, simde_mm256_unpacklo_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_unpacklo_epi16 + #define _mm256_mask_unpacklo_epi16(src, k, a, b) simde_mm256_mask_unpacklo_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_unpacklo_epi16(simde__mmask16 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_unpacklo_epi16(k, a, b); + #else + return simde_mm256_maskz_mov_epi16(k, simde_mm256_unpacklo_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_unpacklo_epi16 + #define _mm256_maskz_unpacklo_epi16(k, a, b) simde_mm256_maskz_unpacklo_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_unpacklo_epi16(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_unpacklo_epi16(src, k, a, b); + #else + return simde_mm_mask_mov_epi16(src, k, simde_mm_unpacklo_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_unpacklo_epi16 + #define _mm_mask_unpacklo_epi16(src, k, a, b) simde_mm_mask_unpacklo_epi16(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_unpacklo_epi16(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_unpacklo_epi16(k, a, b); + #else + return simde_mm_maskz_mov_epi16(k, simde_mm_unpacklo_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_unpacklo_epi16 + #define _mm_maskz_unpacklo_epi16(k, a, b) simde_mm_maskz_unpacklo_epi16(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_unpacklo_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_unpacklo_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.i32 = SIMDE_SHUFFLE_VECTOR_(32, 64, a_.i32, b_.i32, + 0, 16, 1, 17, 4, 20, 5, 21, + 8, 24, 9, 25, 12, 28, 13, 29); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_unpacklo_epi32(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_unpacklo_epi32(a_.m256i[1], b_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0]) / 2) ; i++) { + r_.i32[2 * i] = a_.i32[i + ~(~i | 1)]; + r_.i32[2 * i + 1] = b_.i32[i + ~(~i | 1)]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_unpacklo_epi32 + #define _mm512_unpacklo_epi32(a, b) simde_mm512_unpacklo_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_unpacklo_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_unpacklo_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_unpacklo_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_unpacklo_epi32 + #define _mm512_mask_unpacklo_epi32(src, k, a, b) simde_mm512_mask_unpacklo_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_unpacklo_epi32(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_unpacklo_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_unpacklo_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_unpacklo_epi32 + #define _mm512_maskz_unpacklo_epi32(k, a, b) simde_mm512_maskz_unpacklo_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_unpacklo_epi32(simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_unpacklo_epi32(src, k, a, b); + #else + return simde_mm256_mask_mov_epi32(src, k, simde_mm256_unpacklo_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_unpacklo_epi32 + #define _mm256_mask_unpacklo_epi32(src, k, a, b) simde_mm256_mask_unpacklo_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_unpacklo_epi32(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_unpacklo_epi32(k, a, b); + #else + return simde_mm256_maskz_mov_epi32(k, simde_mm256_unpacklo_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_unpacklo_epi32 + #define _mm256_maskz_unpacklo_epi32(k, a, b) simde_mm256_maskz_unpacklo_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_unpacklo_epi32(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_unpacklo_epi32(src, k, a, b); + #else + return simde_mm_mask_mov_epi32(src, k, simde_mm_unpacklo_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_unpacklo_epi32 + #define _mm_mask_unpacklo_epi32(src, k, a, b) simde_mm_mask_unpacklo_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_unpacklo_epi32(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_unpacklo_epi32(k, a, b); + #else + return simde_mm_maskz_mov_epi32(k, simde_mm_unpacklo_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_unpacklo_epi32 + #define _mm_maskz_unpacklo_epi32(k, a, b) simde_mm_maskz_unpacklo_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_unpacklo_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_unpacklo_epi64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.i64 = SIMDE_SHUFFLE_VECTOR_(64, 64, a_.i64, b_.i64, 0, 8, 2, 10, 4, 12, 6, 14); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256i[0] = simde_mm256_unpacklo_epi64(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_unpacklo_epi64(a_.m256i[1], b_.m256i[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0]) / 2) ; i++) { + r_.i64[2 * i] = a_.i64[2 * i]; + r_.i64[2 * i + 1] = b_.i64[2 * i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_unpacklo_epi64 + #define _mm512_unpacklo_epi64(a, b) simde_mm512_unpacklo_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_unpacklo_epi64(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_unpacklo_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_unpacklo_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_unpacklo_epi64 + #define _mm512_mask_unpacklo_epi64(src, k, a, b) simde_mm512_mask_unpacklo_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_unpacklo_epi64(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_unpacklo_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_unpacklo_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_unpacklo_epi64 + #define _mm512_maskz_unpacklo_epi64(k, a, b) simde_mm512_maskz_unpacklo_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_unpacklo_epi64(simde__m256i src, simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_unpacklo_epi64(src, k, a, b); + #else + return simde_mm256_mask_mov_epi64(src, k, simde_mm256_unpacklo_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_unpacklo_epi64 + #define _mm256_mask_unpacklo_epi64(src, k, a, b) simde_mm256_mask_unpacklo_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_unpacklo_epi64(simde__mmask8 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_unpacklo_epi64(k, a, b); + #else + return simde_mm256_maskz_mov_epi64(k, simde_mm256_unpacklo_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_unpacklo_epi64 + #define _mm256_maskz_unpacklo_epi64(k, a, b) simde_mm256_maskz_unpacklo_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_unpacklo_epi64(simde__m128i src, simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_unpacklo_epi64(src, k, a, b); + #else + return simde_mm_mask_mov_epi64(src, k, simde_mm_unpacklo_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_unpacklo_epi64 + #define _mm_mask_unpacklo_epi64(src, k, a, b) simde_mm_mask_unpacklo_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_unpacklo_epi64(simde__mmask8 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_unpacklo_epi64(k, a, b); + #else + return simde_mm_maskz_mov_epi64(k, simde_mm_unpacklo_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_unpacklo_epi64 + #define _mm_maskz_unpacklo_epi64(k, a, b) simde_mm_maskz_unpacklo_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_unpacklo_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_unpacklo_ps(a, b); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 64, a_.f32, b_.f32, + 0, 16, 1, 17, 4, 20, 5, 21, + 8, 24, 9, 25, 12, 28, 13, 29); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256[0] = simde_mm256_unpacklo_ps(a_.m256[0], b_.m256[0]); + r_.m256[1] = simde_mm256_unpacklo_ps(a_.m256[1], b_.m256[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0]) / 2) ; i++) { + r_.f32[2 * i] = a_.f32[i + ~(~i | 1)]; + r_.f32[2 * i + 1] = b_.f32[i + ~(~i | 1)]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_unpacklo_ps + #define _mm512_unpacklo_ps(a, b) simde_mm512_unpacklo_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_unpacklo_ps(simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_unpacklo_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_unpacklo_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_unpacklo_ps + #define _mm512_mask_unpacklo_ps(src, k, a, b) simde_mm512_mask_unpacklo_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_unpacklo_ps(simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_unpacklo_ps(k, a, b); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_unpacklo_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_unpacklo_ps + #define _mm512_maskz_unpacklo_ps(k, a, b) simde_mm512_maskz_unpacklo_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_mask_unpacklo_ps(simde__m256 src, simde__mmask8 k, simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_unpacklo_ps(src, k, a, b); + #else + return simde_mm256_mask_mov_ps(src, k, simde_mm256_unpacklo_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_unpacklo_ps + #define _mm256_mask_unpacklo_ps(src, k, a, b) simde_mm256_mask_unpacklo_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_maskz_unpacklo_ps(simde__mmask8 k, simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_unpacklo_ps(k, a, b); + #else + return simde_mm256_maskz_mov_ps(k, simde_mm256_unpacklo_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_unpacklo_ps + #define _mm256_maskz_unpacklo_ps(k, a, b) simde_mm256_maskz_unpacklo_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mask_unpacklo_ps(simde__m128 src, simde__mmask8 k, simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_unpacklo_ps(src, k, a, b); + #else + return simde_mm_mask_mov_ps(src, k, simde_mm_unpacklo_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_unpacklo_ps + #define _mm_mask_unpacklo_ps(src, k, a, b) simde_mm_mask_unpacklo_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_maskz_unpacklo_ps(simde__mmask8 k, simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_unpacklo_ps(k, a, b); + #else + return simde_mm_maskz_mov_ps(k, simde_mm_unpacklo_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_unpacklo_ps + #define _mm_maskz_unpacklo_ps(k, a, b) simde_mm_maskz_unpacklo_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_unpacklo_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_unpacklo_pd(a, b); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + #if defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 64, a_.f64, b_.f64, 0, 8, 2, 10, 4, 12, 6, 14); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256d[0] = simde_mm256_unpacklo_pd(a_.m256d[0], b_.m256d[0]); + r_.m256d[1] = simde_mm256_unpacklo_pd(a_.m256d[1], b_.m256d[1]); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0]) / 2) ; i++) { + r_.f64[2 * i] = a_.f64[2 * i]; + r_.f64[2 * i + 1] = b_.f64[2 * i]; + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_unpacklo_pd + #define _mm512_unpacklo_pd(a, b) simde_mm512_unpacklo_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_unpacklo_pd(simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_unpacklo_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_unpacklo_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_unpacklo_pd + #define _mm512_mask_unpacklo_pd(src, k, a, b) simde_mm512_mask_unpacklo_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_unpacklo_pd(simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_unpacklo_pd(k, a, b); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_unpacklo_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_unpacklo_pd + #define _mm512_maskz_unpacklo_pd(k, a, b) simde_mm512_maskz_unpacklo_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_mask_unpacklo_pd(simde__m256d src, simde__mmask8 k, simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_unpacklo_pd(src, k, a, b); + #else + return simde_mm256_mask_mov_pd(src, k, simde_mm256_unpacklo_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_unpacklo_pd + #define _mm256_mask_unpacklo_pd(src, k, a, b) simde_mm256_mask_unpacklo_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_maskz_unpacklo_pd(simde__mmask8 k, simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_unpacklo_pd(k, a, b); + #else + return simde_mm256_maskz_mov_pd(k, simde_mm256_unpacklo_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_unpacklo_pd + #define _mm256_maskz_unpacklo_pd(k, a, b) simde_mm256_maskz_unpacklo_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mask_unpacklo_pd(simde__m128d src, simde__mmask8 k, simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_unpacklo_pd(src, k, a, b); + #else + return simde_mm_mask_mov_pd(src, k, simde_mm_unpacklo_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_unpacklo_pd + #define _mm_mask_unpacklo_pd(src, k, a, b) simde_mm_mask_unpacklo_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_maskz_unpacklo_pd(simde__mmask8 k, simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_unpacklo_pd(k, a, b); + #else + return simde_mm_maskz_mov_pd(k, simde_mm_unpacklo_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_unpacklo_pd + #define _mm_maskz_unpacklo_pd(k, a, b) simde_mm_maskz_unpacklo_pd(k, a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_UNPACKLO_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/xor.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/xor.h new file mode 100644 index 00000000..359ab1b0 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/xor.h @@ -0,0 +1,319 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_AVX512_XOR_H) +#define SIMDE_X86_AVX512_XOR_H + +#include "types.h" +#include "../avx2.h" +#include "mov.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_xor_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_xor_ps(a, b); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + /* TODO: generate reduced case to give to Intel */ + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) && !defined(HEDLEY_INTEL_VERSION) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_xor_ps(a_.m256[i], b_.m256[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_CLANG_BAD_VI64_OPS) + r_.i32f = a_.i32f ^ b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] ^ b_.i32f[i]; + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_xor_ps + #define _mm512_xor_ps(a, b) simde_mm512_xor_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_xor_ps(simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_mask_xor_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_xor_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_xor_ps + #define _mm512_mask_xor_ps(src, k, a, b) simde_mm512_mask_xor_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_maskz_xor_ps(simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_maskz_xor_ps(k, a, b); + #else + return simde_mm512_maskz_mov_ps(k, simde_mm512_xor_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_xor_ps + #define _mm512_maskz_xor_ps(k, a, b) simde_mm512_maskz_xor_ps(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_xor_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_xor_pd(a, b); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + r_.m256d[0] = simde_mm256_xor_pd(a_.m256d[0], b_.m256d[0]); + r_.m256d[1] = simde_mm256_xor_pd(a_.m256d[1], b_.m256d[1]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_CLANG_BAD_VI64_OPS) + r_.i32f = a_.i32f ^ b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] ^ b_.i32f[i]; + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_xor_pd + #define _mm512_xor_pd(a, b) simde_mm512_xor_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_xor_pd(simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_mask_xor_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_xor_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_xor_pd + #define _mm512_mask_xor_pd(src, k, a, b) simde_mm512_mask_xor_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_maskz_xor_pd(simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_AVX512DQ_NATIVE) + return _mm512_maskz_xor_pd(k, a, b); + #else + return simde_mm512_maskz_mov_pd(k, simde_mm512_xor_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512DQ_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_xor_pd + #define _mm512_maskz_xor_pd(k, a, b) simde_mm512_maskz_xor_pd(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_xor_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_xor_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_xor_si256(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 ^ b_.i32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] ^ b_.i32[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_xor_epi32 + #define _mm512_xor_epi32(a, b) simde_mm512_xor_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_xor_epi32(simde__m512i src, simde__mmask16 k, simde__m512i v2, simde__m512i v3) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_xor_epi32(src, k, v2, v3); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_xor_epi32(v2, v3)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_xor_epi32 + #define _mm512_mask_xor_epi32(src, k, v2, v3) simde_mm512_mask_xor_epi32(src, k, v2, v3) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_xor_epi32(simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_xor_epi32(k, a, b); + #else + return simde_mm512_maskz_mov_epi32(k, simde_mm512_xor_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_xor_epi32 + #define _mm512_maskz_xor_epi32(k, a, b) simde_mm512_maskz_xor_epi32(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_xor_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_xor_epi64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_xor_si256(a_.m256i[i], b_.m256i[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_CLANG_BAD_VI64_OPS) + r_.i32f = a_.i32f ^ b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] ^ b_.i32f[i]; + } + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_xor_epi64 + #define _mm512_xor_epi64(a, b) simde_mm512_xor_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_xor_epi64(simde__m512i src, simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_xor_epi64(src, k, a, b); + #else + return simde_mm512_mask_mov_epi64(src, k, simde_mm512_xor_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_xor_epi64 + #define _mm512_mask_xor_epi64(src, k, a, b) simde_mm512_mask_xor_epi64(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_xor_epi64(simde__mmask8 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_xor_epi64(k, a, b); + #else + return simde_mm512_maskz_mov_epi64(k, simde_mm512_xor_epi64(a, b)); + #endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_xor_epi64 + #define _mm512_maskz_xor_epi64(k, a, b) simde_mm512_maskz_xor_epi64(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_xor_si512 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_xor_si512(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_X86_AVX2_NATIVE) + r_.m256i[0] = simde_mm256_xor_si256(a_.m256i[0], b_.m256i[0]); + r_.m256i[1] = simde_mm256_xor_si256(a_.m256i[1], b_.m256i[1]); + #elif defined(SIMDE_X86_SSE2_NATIVE) + r_.m128i[0] = simde_mm_xor_si128(a_.m128i[0], b_.m128i[0]); + r_.m128i[1] = simde_mm_xor_si128(a_.m128i[1], b_.m128i[1]); + r_.m128i[2] = simde_mm_xor_si128(a_.m128i[2], b_.m128i[2]); + r_.m128i[3] = simde_mm_xor_si128(a_.m128i[3], b_.m128i[3]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f ^ b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] ^ b_.i32f[i]; + } + #endif + + return simde__m512i_from_private(r_); +#endif +} +#if defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_xor_si512 + #define _mm512_xor_si512(a, b) simde_mm512_xor_si512(a, b) +#endif + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_XOR_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/xorsign.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/xorsign.h new file mode 100644 index 00000000..f7fdc8c9 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/avx512/xorsign.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + */ + +/* This is a SIMDe extension which is not part of AVX-512. It exists + * because a lot of numerical methods in SIMDe have algorithms which do + * something like: + * + * float sgn = input < 0 ? -1 : 1; + * ... + * return res * sgn; + * + * Which can be replaced with a much more efficient call to xorsign: + * + * return simde_x_mm512_xorsign_ps(res, input); + * + * While this was originally intended for use in SIMDe, please feel + * free to use it in your code. + */ + +#if !defined(SIMDE_X86_AVX512_XORSIGN_H) +#define SIMDE_X86_AVX512_XORSIGN_H + +#include "types.h" +#include "mov.h" +#include "and.h" +#include "xor.h" +#include "set1.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_x_mm512_xorsign_ps(simde__m512 dest, simde__m512 src) { + return simde_mm512_xor_ps(simde_mm512_and_ps(simde_mm512_set1_ps(-0.0f), src), dest); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_x_mm512_xorsign_pd(simde__m512d dest, simde__m512d src) { + return simde_mm512_xor_pd(simde_mm512_and_pd(simde_mm512_set1_pd(-0.0), src), dest); +} + +SIMDE_END_DECLS_ +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_AVX512_XORSIGN_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/clmul.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/clmul.h new file mode 100644 index 00000000..e2d8b4cf --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/clmul.h @@ -0,0 +1,387 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2016 Thomas Pornin + */ + +/* The portable version is based on the implementation in BearSSL, + * which is MIT licensed, constant-time / branch-free, and documented + * at https://www.bearssl.org/constanttime.html (specifically, we use + * the implementation from ghash_ctmul64.c). */ + +#if !defined(SIMDE_X86_CLMUL_H) +#define SIMDE_X86_CLMUL_H + +#include "avx512/set.h" +#include "avx512/setzero.h" + +#if !defined(SIMDE_X86_PCLMUL_NATIVE) && defined(SIMDE_ENABLE_NATIVE_ALIASES) +# define SIMDE_X86_PCLMUL_ENABLE_NATIVE_ALIASES +#endif + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +uint64_t +simde_x_clmul_u64(uint64_t x, uint64_t y) { + uint64_t x0, x1, x2, x3; + uint64_t y0, y1, y2, y3; + uint64_t z0, z1, z2, z3; + + x0 = x & UINT64_C(0x1111111111111111); + x1 = x & UINT64_C(0x2222222222222222); + x2 = x & UINT64_C(0x4444444444444444); + x3 = x & UINT64_C(0x8888888888888888); + y0 = y & UINT64_C(0x1111111111111111); + y1 = y & UINT64_C(0x2222222222222222); + y2 = y & UINT64_C(0x4444444444444444); + y3 = y & UINT64_C(0x8888888888888888); + + z0 = (x0 * y0) ^ (x1 * y3) ^ (x2 * y2) ^ (x3 * y1); + z1 = (x0 * y1) ^ (x1 * y0) ^ (x2 * y3) ^ (x3 * y2); + z2 = (x0 * y2) ^ (x1 * y1) ^ (x2 * y0) ^ (x3 * y3); + z3 = (x0 * y3) ^ (x1 * y2) ^ (x2 * y1) ^ (x3 * y0); + + z0 &= UINT64_C(0x1111111111111111); + z1 &= UINT64_C(0x2222222222222222); + z2 &= UINT64_C(0x4444444444444444); + z3 &= UINT64_C(0x8888888888888888); + + return z0 | z1 | z2 | z3; +} + +static uint64_t +simde_x_bitreverse_u64(uint64_t v) { + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint8x8_t bytes = vreinterpret_u8_u64(vmov_n_u64(v)); + bytes = vrbit_u8(bytes); + bytes = vrev64_u8(bytes); + return vget_lane_u64(vreinterpret_u64_u8(bytes), 0); + #elif defined(SIMDE_X86_GFNI_NATIVE) + /* I don't think there is (or likely will ever be) a CPU with GFNI + * but not pclmulq, but this may be useful for things other than + * _mm_clmulepi64_si128. */ + __m128i vec = _mm_cvtsi64_si128(HEDLEY_STATIC_CAST(int64_t, v)); + + /* Reverse bits within each byte */ + vec = _mm_gf2p8affine_epi64_epi8(vec, _mm_cvtsi64_si128(HEDLEY_STATIC_CAST(int64_t, UINT64_C(0x8040201008040201))), 0); + + /* Reverse bytes */ + #if defined(SIMDE_X86_SSSE3_NATIVE) + vec = _mm_shuffle_epi8(vec, _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7)); + #else + vec = _mm_or_si128(_mm_slli_epi16(vec, 8), _mm_srli_epi16(vec, 8)); + vec = _mm_shufflelo_epi16(vec, _MM_SHUFFLE(0, 1, 2, 3)); + vec = _mm_shufflehi_epi16(vec, _MM_SHUFFLE(0, 1, 2, 3)); + #endif + + return HEDLEY_STATIC_CAST(uint64_t, _mm_cvtsi128_si64(vec)); + #elif HEDLEY_HAS_BUILTIN(__builtin_bitreverse64) + return __builtin_bitreverse64(v); + #else + v = ((v >> 1) & UINT64_C(0x5555555555555555)) | ((v & UINT64_C(0x5555555555555555)) << 1); + v = ((v >> 2) & UINT64_C(0x3333333333333333)) | ((v & UINT64_C(0x3333333333333333)) << 2); + v = ((v >> 4) & UINT64_C(0x0F0F0F0F0F0F0F0F)) | ((v & UINT64_C(0x0F0F0F0F0F0F0F0F)) << 4); + v = ((v >> 8) & UINT64_C(0x00FF00FF00FF00FF)) | ((v & UINT64_C(0x00FF00FF00FF00FF)) << 8); + v = ((v >> 16) & UINT64_C(0x0000FFFF0000FFFF)) | ((v & UINT64_C(0x0000FFFF0000FFFF)) << 16); + return (v >> 32) | (v << 32); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_clmulepi64_si128 (simde__m128i a, simde__m128i b, const int imm8) + SIMDE_REQUIRE_CONSTANT(imm8) { + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + r_; + + #if SIMDE_NATURAL_VECTOR_SIZE_GE(128) + #if defined(SIMDE_SHUFFLE_VECTOR_) + switch (imm8 & 0x11) { + case 0x00: + b_.u64 = SIMDE_SHUFFLE_VECTOR_(64, 16, b_.u64, b_.u64, 0, 0); + a_.u64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.u64, a_.u64, 0, 0); + break; + case 0x01: + b_.u64 = SIMDE_SHUFFLE_VECTOR_(64, 16, b_.u64, b_.u64, 0, 0); + a_.u64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.u64, a_.u64, 1, 1); + break; + case 0x10: + b_.u64 = SIMDE_SHUFFLE_VECTOR_(64, 16, b_.u64, b_.u64, 1, 1); + a_.u64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.u64, a_.u64, 0, 0); + break; + case 0x11: + b_.u64 = SIMDE_SHUFFLE_VECTOR_(64, 16, b_.u64, b_.u64, 1, 1); + a_.u64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.u64, a_.u64, 1, 1); + break; + } + #else + { + const uint64_t A = a_.u64[(imm8 ) & 1]; + const uint64_t B = b_.u64[(imm8 >> 4) & 1]; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + a_.u64[i] = A; + b_.u64[i] = B; + } + } + #endif + + simde__m128i_private reversed_; + { + #if defined(SIMDE_SHUFFLE_VECTOR_) + reversed_.u64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.u64, b_.u64, 1, 3); + #else + reversed_.u64[0] = a_.u64[1]; + reversed_.u64[1] = b_.u64[1]; + #endif + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(reversed_.u64) / sizeof(reversed_.u64[0])) ; i++) { + reversed_.u64[i] = simde_x_bitreverse_u64(reversed_.u64[i]); + } + } + + #if defined(SIMDE_SHUFFLE_VECTOR_) + a_.u64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.u64, reversed_.u64, 0, 2); + b_.u64 = SIMDE_SHUFFLE_VECTOR_(64, 16, b_.u64, reversed_.u64, 1, 3); + #else + a_.u64[1] = reversed_.u64[0]; + b_.u64[1] = reversed_.u64[1]; + #endif + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(reversed_.u64) / sizeof(reversed_.u64[0])) ; i++) { + r_.u64[i] = simde_x_clmul_u64(a_.u64[i], b_.u64[i]); + } + + r_.u64[1] = simde_x_bitreverse_u64(r_.u64[1]) >> 1; + #else + r_.u64[0] = simde_x_clmul_u64( a_.u64[imm8 & 1], b_.u64[(imm8 >> 4) & 1]); + r_.u64[1] = simde_x_bitreverse_u64(simde_x_clmul_u64(simde_x_bitreverse_u64(a_.u64[imm8 & 1]), simde_x_bitreverse_u64(b_.u64[(imm8 >> 4) & 1]))) >> 1; + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_PCLMUL_NATIVE) + #if defined(HEDLEY_MCST_LCC_VERSION) + #define simde_mm_clmulepi64_si128(a, b, imm8) (__extension__ ({ \ + SIMDE_LCC_DISABLE_DEPRECATED_WARNINGS \ + _mm_clmulepi64_si128((a), (b), (imm8)); \ + SIMDE_LCC_REVERT_DEPRECATED_WARNINGS \ + })) + #else + #define simde_mm_clmulepi64_si128(a, b, imm8) _mm_clmulepi64_si128(a, b, imm8) + #endif +#elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) && defined(SIMDE_ARCH_ARM_AES) && !defined(__clang__) + #define simde_mm_clmulepi64_si128(a, b, imm8) \ + simde__m128i_from_neon_u64( \ + vreinterpretq_u64_p128( \ + vmull_p64( \ + vgetq_lane_p64(vreinterpretq_p64_u64(simde__m128i_to_neon_u64(a)), (imm8 ) & 1), \ + vgetq_lane_p64(vreinterpretq_p64_u64(simde__m128i_to_neon_u64(b)), (imm8 >> 4) & 1) \ + ) \ + ) \ + ) +#endif +#if defined(SIMDE_X86_PCLMUL_ENABLE_NATIVE_ALIASES) + #undef _mm_clmulepi64_si128 + #define _mm_clmulepi64_si128(a, b, imm8) simde_mm_clmulepi64_si128(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_clmulepi64_epi128 (simde__m256i a, simde__m256i b, const int imm8) + SIMDE_REQUIRE_CONSTANT(imm8) { + simde__m256i_private + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b), + r_; + + simde__m128i_private a_lo_, b_lo_, r_lo_, a_hi_, b_hi_, r_hi_; + + #if HEDLEY_HAS_BUILTIN(__builtin_shufflevector) && !defined(HEDLEY_IBM_VERSION) + switch (imm8 & 0x01) { + case 0x00: + a_lo_.u64 = __builtin_shufflevector(a_.u64, a_.u64, 0, 2); + break; + case 0x01: + a_lo_.u64 = __builtin_shufflevector(a_.u64, a_.u64, 1, 3); + break; + } + switch (imm8 & 0x10) { + case 0x00: + b_lo_.u64 = __builtin_shufflevector(b_.u64, b_.u64, 0, 2); + break; + case 0x10: + b_lo_.u64 = __builtin_shufflevector(b_.u64, b_.u64, 1, 3); + break; + } + #else + a_lo_.u64[0] = a_.u64[((imm8 >> 0) & 1) + 0]; + a_lo_.u64[1] = a_.u64[((imm8 >> 0) & 1) + 2]; + b_lo_.u64[0] = b_.u64[((imm8 >> 4) & 1) + 0]; + b_lo_.u64[1] = b_.u64[((imm8 >> 4) & 1) + 2]; + #endif + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_hi_.u64) / sizeof(r_hi_.u64[0])) ; i++) { + a_hi_.u64[i] = simde_x_bitreverse_u64(a_lo_.u64[i]); + b_hi_.u64[i] = simde_x_bitreverse_u64(b_lo_.u64[i]); + + r_lo_.u64[i] = simde_x_clmul_u64(a_lo_.u64[i], b_lo_.u64[i]); + r_hi_.u64[i] = simde_x_clmul_u64(a_hi_.u64[i], b_hi_.u64[i]); + + r_hi_.u64[i] = simde_x_bitreverse_u64(r_hi_.u64[i]) >> 1; + } + + #if HEDLEY_HAS_BUILTIN(__builtin_shufflevector) && !defined(HEDLEY_IBM_VERSION) + r_.u64 = __builtin_shufflevector(r_lo_.u64, r_hi_.u64, 0, 2, 1, 3); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_ = simde__m256i_to_private(simde_mm256_set_m128i(simde__m128i_from_private(r_hi_), simde__m128i_from_private(r_lo_))); + r_.u64 = SIMDE_SHUFFLE_VECTOR_(64, 32, r_.u64, r_.u64, 0, 2, 1, 3); + #else + r_.u64[0] = r_lo_.u64[0]; + r_.u64[1] = r_hi_.u64[0]; + r_.u64[2] = r_lo_.u64[1]; + r_.u64[3] = r_hi_.u64[1]; + #endif + + return simde__m256i_from_private(r_); +} +#if defined(SIMDE_X86_VPCLMULQDQ_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_clmulepi64_epi128(a, b, imm8) _mm256_clmulepi64_epi128(a, b, imm8) +#endif +#if defined(SIMDE_X86_VPCLMULQDQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_clmulepi64_epi128 + #define _mm256_clmulepi64_epi128(a, b, imm8) simde_mm256_clmulepi64_epi128(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_clmulepi64_epi128 (simde__m512i a, simde__m512i b, const int imm8) + SIMDE_REQUIRE_CONSTANT(imm8) { + simde__m512i_private + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b), + r_; + + #if defined(HEDLEY_MSVC_VERSION) + r_ = simde__m512i_to_private(simde_mm512_setzero_si512()); + #endif + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + switch (imm8 & 0x11) { + case 0x00: + r_.m256i[0] = simde_mm256_clmulepi64_epi128(a_.m256i[0], b_.m256i[0], 0x00); + r_.m256i[1] = simde_mm256_clmulepi64_epi128(a_.m256i[1], b_.m256i[1], 0x00); + break; + case 0x01: + r_.m256i[0] = simde_mm256_clmulepi64_epi128(a_.m256i[0], b_.m256i[0], 0x01); + r_.m256i[1] = simde_mm256_clmulepi64_epi128(a_.m256i[1], b_.m256i[1], 0x01); + break; + case 0x10: + r_.m256i[0] = simde_mm256_clmulepi64_epi128(a_.m256i[0], b_.m256i[0], 0x10); + r_.m256i[1] = simde_mm256_clmulepi64_epi128(a_.m256i[1], b_.m256i[1], 0x10); + break; + case 0x11: + r_.m256i[0] = simde_mm256_clmulepi64_epi128(a_.m256i[0], b_.m256i[0], 0x11); + r_.m256i[1] = simde_mm256_clmulepi64_epi128(a_.m256i[1], b_.m256i[1], 0x11); + break; + } + #else + simde__m256i_private a_lo_, b_lo_, r_lo_, a_hi_, b_hi_, r_hi_; + + #if HEDLEY_HAS_BUILTIN(__builtin_shufflevector) && !defined(HEDLEY_IBM_VERSION) + switch (imm8 & 0x01) { + case 0x00: + a_lo_.u64 = __builtin_shufflevector(a_.u64, a_.u64, 0, 2, 4, 6); + break; + case 0x01: + a_lo_.u64 = __builtin_shufflevector(a_.u64, a_.u64, 1, 3, 5, 7); + break; + } + switch (imm8 & 0x10) { + case 0x00: + b_lo_.u64 = __builtin_shufflevector(b_.u64, b_.u64, 0, 2, 4, 6); + break; + case 0x10: + b_lo_.u64 = __builtin_shufflevector(b_.u64, b_.u64, 1, 3, 5, 7); + break; + } + #else + a_lo_.u64[0] = a_.u64[((imm8 >> 0) & 1) + 0]; + a_lo_.u64[1] = a_.u64[((imm8 >> 0) & 1) + 2]; + a_lo_.u64[2] = a_.u64[((imm8 >> 0) & 1) + 4]; + a_lo_.u64[3] = a_.u64[((imm8 >> 0) & 1) + 6]; + b_lo_.u64[0] = b_.u64[((imm8 >> 4) & 1) + 0]; + b_lo_.u64[1] = b_.u64[((imm8 >> 4) & 1) + 2]; + b_lo_.u64[2] = b_.u64[((imm8 >> 4) & 1) + 4]; + b_lo_.u64[3] = b_.u64[((imm8 >> 4) & 1) + 6]; + #endif + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_hi_.u64) / sizeof(r_hi_.u64[0])) ; i++) { + a_hi_.u64[i] = simde_x_bitreverse_u64(a_lo_.u64[i]); + b_hi_.u64[i] = simde_x_bitreverse_u64(b_lo_.u64[i]); + + r_lo_.u64[i] = simde_x_clmul_u64(a_lo_.u64[i], b_lo_.u64[i]); + r_hi_.u64[i] = simde_x_clmul_u64(a_hi_.u64[i], b_hi_.u64[i]); + + r_hi_.u64[i] = simde_x_bitreverse_u64(r_hi_.u64[i]) >> 1; + } + + #if HEDLEY_HAS_BUILTIN(__builtin_shufflevector) && !defined(HEDLEY_IBM_VERSION) + r_.u64 = __builtin_shufflevector(r_lo_.u64, r_hi_.u64, 0, 4, 1, 5, 2, 6, 3, 7); + #else + r_.u64[0] = r_lo_.u64[0]; + r_.u64[1] = r_hi_.u64[0]; + r_.u64[2] = r_lo_.u64[1]; + r_.u64[3] = r_hi_.u64[1]; + r_.u64[4] = r_lo_.u64[2]; + r_.u64[5] = r_hi_.u64[2]; + r_.u64[6] = r_lo_.u64[3]; + r_.u64[7] = r_hi_.u64[3]; + #endif + #endif + + return simde__m512i_from_private(r_); +} +#if defined(SIMDE_X86_VPCLMULQDQ_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_clmulepi64_epi128(a, b, imm8) _mm512_clmulepi64_epi128(a, b, imm8) +#endif +#if defined(SIMDE_X86_VPCLMULQDQ_ENABLE_NATIVE_ALIASES) && defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_clmulepi64_epi128 + #define _mm512_clmulepi64_epi128(a, b, imm8) simde_mm512_clmulepi64_epi128(a, b, imm8) +#endif + +SIMDE_END_DECLS_ + +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_CLMUL_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/f16c.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/f16c.h new file mode 100644 index 00000000..9522bf6f --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/f16c.h @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2021 Evan Nemerson + */ + +#include "../simde-common.h" +#include "../simde-math.h" +#include "../simde-f16.h" + +#if !defined(SIMDE_X86_F16C_H) +#define SIMDE_X86_F16C_H + +#include "avx.h" + +#if !defined(SIMDE_X86_PF16C_NATIVE) && defined(SIMDE_ENABLE_NATIVE_ALIASES) +# define SIMDE_X86_PF16C_ENABLE_NATIVE_ALIASES +#endif + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtps_ph(simde__m128 a, const int imm8) { + simde__m128_private a_ = simde__m128_to_private(a); + simde__m128i_private r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + + HEDLEY_STATIC_CAST(void, imm8); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) + r_.neon_f16 = vcombine_f16(vcvt_f16_f32(a_.neon_f32), vdup_n_f16(SIMDE_FLOAT16_C(0.0))); + #elif defined(SIMDE_FLOAT16_VECTOR) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + r_.f16[i] = simde_float16_from_float32(a_.f32[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + r_.u16[i] = simde_float16_as_uint16(simde_float16_from_float32(a_.f32[i])); + } + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_F16C_NATIVE) + #define simde_mm_cvtps_ph(a, imm8) _mm_cvtps_ph(a, imm8) +#endif +#if defined(SIMDE_X86_F16C_ENABLE_NATIVE_ALIASES) + #define _mm_cvtps_ph(a, sae) simde_mm_cvtps_ph(a, sae) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cvtph_ps(simde__m128i a) { + #if defined(SIMDE_X86_F16C_NATIVE) + return _mm_cvtph_ps(a); + #else + simde__m128i_private a_ = simde__m128i_to_private(a); + simde__m128_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) + r_.neon_f32 = vcvt_f32_f16(vget_low_f16(a_.neon_f16)); + #elif defined(SIMDE_FLOAT16_VECTOR) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + r_.f32[i] = simde_float16_to_float32(a_.f16[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + r_.f32[i] = simde_float16_to_float32(simde_uint16_as_float16(a_.u16[i])); + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_F16C_ENABLE_NATIVE_ALIASES) + #define _mm_cvtph_ps(a) simde_mm_cvtph_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm256_cvtps_ph(simde__m256 a, const int imm8) { + simde__m256_private a_ = simde__m256_to_private(a); + simde__m128i_private r_; + + HEDLEY_STATIC_CAST(void, imm8); + + #if defined(SIMDE_FLOAT16_VECTOR) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + r_.f16[i] = simde_float16_from_float32(a_.f32[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + r_.u16[i] = simde_float16_as_uint16(simde_float16_from_float32(a_.f32[i])); + } + #endif + + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_F16C_NATIVE) + #define simde_mm256_cvtps_ph(a, imm8) _mm256_cvtps_ph(a, imm8) +#endif +#if defined(SIMDE_X86_F16C_ENABLE_NATIVE_ALIASES) + #define _mm256_cvtps_ph(a, imm8) simde_mm256_cvtps_ph(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_cvtph_ps(simde__m128i a) { + #if defined(SIMDE_X86_F16C_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cvtph_ps(a); + #elif defined(SIMDE_X86_F16C_NATIVE) + return _mm256_setr_m128( + _mm_cvtph_ps(a), + _mm_cvtph_ps(_mm_castps_si128(_mm_permute_ps(_mm_castsi128_ps(a), 0xee))) + ); + #else + simde__m128i_private a_ = simde__m128i_to_private(a); + simde__m256_private r_; + + #if defined(SIMDE_FLOAT16_VECTOR) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_float16_to_float32(a_.f16[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_float16_to_float32(simde_uint16_as_float16(a_.u16[i])); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_F16C_ENABLE_NATIVE_ALIASES) + #define _mm256_cvtph_ps(a) simde_mm256_cvtph_ps(a) +#endif + +SIMDE_END_DECLS_ + +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_F16C_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/fma.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/fma.h new file mode 100644 index 00000000..630efc54 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/fma.h @@ -0,0 +1,732 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2019 Evan Nemerson + */ + +#if !defined(SIMDE_X86_FMA_H) +#define SIMDE_X86_FMA_H + +#include "avx.h" + +#if !defined(SIMDE_X86_FMA_NATIVE) && defined(SIMDE_ENABLE_NATIVE_ALIASES) +# define SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES +#endif + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_fmadd_pd (simde__m128d a, simde__m128d b, simde__m128d c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm_fmadd_pd(a, b, c); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b), + c_ = simde__m128d_to_private(c), + r_; + + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = vec_madd(a_.altivec_f64, b_.altivec_f64, c_.altivec_f64); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vfmaq_f64(c_.neon_f64, b_.neon_f64, a_.neon_f64); + #elif defined(simde_math_fma) && (defined(__FP_FAST_FMA) || defined(FP_FAST_FMA)) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_fma(a_.f64[i], b_.f64[i], c_.f64[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = (a_.f64[i] * b_.f64[i]) + c_.f64[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fmadd_pd + #define _mm_fmadd_pd(a, b, c) simde_mm_fmadd_pd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_fmadd_pd (simde__m256d a, simde__m256d b, simde__m256d c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm256_fmadd_pd(a, b, c); + #else + return simde_mm256_add_pd(simde_mm256_mul_pd(a, b), c); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm256_fmadd_pd + #define _mm256_fmadd_pd(a, b, c) simde_mm256_fmadd_pd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_fmadd_ps (simde__m128 a, simde__m128 b, simde__m128 c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm_fmadd_ps(a, b, c); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b), + c_ = simde__m128_to_private(c), + r_; + + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + r_.altivec_f32 = vec_madd(a_.altivec_f32, b_.altivec_f32, c_.altivec_f32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) && defined(SIMDE_ARCH_ARM_FMA) + r_.neon_f32 = vfmaq_f32(c_.neon_f32, b_.neon_f32, a_.neon_f32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vmlaq_f32(c_.neon_f32, b_.neon_f32, a_.neon_f32); + #elif defined(simde_math_fmaf) && (defined(__FP_FAST_FMAF) || defined(FP_FAST_FMAF)) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_fmaf(a_.f32[i], b_.f32[i], c_.f32[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = (a_.f32[i] * b_.f32[i]) + c_.f32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fmadd_ps + #define _mm_fmadd_ps(a, b, c) simde_mm_fmadd_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_fmadd_ps (simde__m256 a, simde__m256 b, simde__m256 c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm256_fmadd_ps(a, b, c); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(128) + simde__m256_private + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b), + c_ = simde__m256_to_private(c), + r_; + + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_fmadd_ps(a_.m128[i], b_.m128[i], c_.m128[i]); + } + + return simde__m256_from_private(r_); + #else + return simde_mm256_add_ps(simde_mm256_mul_ps(a, b), c); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm256_fmadd_ps + #define _mm256_fmadd_ps(a, b, c) simde_mm256_fmadd_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_fmadd_sd (simde__m128d a, simde__m128d b, simde__m128d c) { + #if defined(SIMDE_X86_FMA_NATIVE) && !defined(SIMDE_BUG_MCST_LCC_FMA_WRONG_RESULT) + return _mm_fmadd_sd(a, b, c); + #else + return simde_mm_add_sd(simde_mm_mul_sd(a, b), c); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fmadd_sd + #define _mm_fmadd_sd(a, b, c) simde_mm_fmadd_sd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_fmadd_ss (simde__m128 a, simde__m128 b, simde__m128 c) { + #if defined(SIMDE_X86_FMA_NATIVE) && !defined(SIMDE_BUG_MCST_LCC_FMA_WRONG_RESULT) + return _mm_fmadd_ss(a, b, c); + #else + return simde_mm_add_ss(simde_mm_mul_ss(a, b), c); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fmadd_ss + #define _mm_fmadd_ss(a, b, c) simde_mm_fmadd_ss(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_fmaddsub_pd (simde__m128d a, simde__m128d b, simde__m128d c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm_fmaddsub_pd(a, b, c); + #else + return simde_mm_addsub_pd(simde_mm_mul_pd(a, b), c); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fmaddsub_pd + #define _mm_fmaddsub_pd(a, b, c) simde_mm_fmaddsub_pd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_fmaddsub_pd (simde__m256d a, simde__m256d b, simde__m256d c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm256_fmaddsub_pd(a, b, c); + #else + return simde_mm256_addsub_pd(simde_mm256_mul_pd(a, b), c); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm256_fmaddsub_pd + #define _mm256_fmaddsub_pd(a, b, c) simde_mm256_fmaddsub_pd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_fmaddsub_ps (simde__m128 a, simde__m128 b, simde__m128 c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm_fmaddsub_ps(a, b, c); + #else + return simde_mm_addsub_ps(simde_mm_mul_ps(a, b), c); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fmaddsub_ps + #define _mm_fmaddsub_ps(a, b, c) simde_mm_fmaddsub_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_fmaddsub_ps (simde__m256 a, simde__m256 b, simde__m256 c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm256_fmaddsub_ps(a, b, c); + #else + return simde_mm256_addsub_ps(simde_mm256_mul_ps(a, b), c); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm256_fmaddsub_ps + #define _mm256_fmaddsub_ps(a, b, c) simde_mm256_fmaddsub_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_fmsub_pd (simde__m128d a, simde__m128d b, simde__m128d c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm_fmsub_pd(a, b, c); + #else + return simde_mm_sub_pd(simde_mm_mul_pd(a, b), c); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fmsub_pd + #define _mm_fmsub_pd(a, b, c) simde_mm_fmsub_pd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_fmsub_pd (simde__m256d a, simde__m256d b, simde__m256d c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm256_fmsub_pd(a, b, c); + #else + return simde_mm256_sub_pd(simde_mm256_mul_pd(a, b), c); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm256_fmsub_pd + #define _mm256_fmsub_pd(a, b, c) simde_mm256_fmsub_pd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_fmsub_ps (simde__m128 a, simde__m128 b, simde__m128 c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm_fmsub_ps(a, b, c); + #else + return simde_mm_sub_ps(simde_mm_mul_ps(a, b), c); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fmsub_ps + #define _mm_fmsub_ps(a, b, c) simde_mm_fmsub_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_fmsub_ps (simde__m256 a, simde__m256 b, simde__m256 c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm256_fmsub_ps(a, b, c); + #else + return simde_mm256_sub_ps(simde_mm256_mul_ps(a, b), c); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm256_fmsub_ps + #define _mm256_fmsub_ps(a, b, c) simde_mm256_fmsub_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_fmsub_sd (simde__m128d a, simde__m128d b, simde__m128d c) { + #if defined(SIMDE_X86_FMA_NATIVE) && !defined(SIMDE_BUG_MCST_LCC_FMA_WRONG_RESULT) + return _mm_fmsub_sd(a, b, c); + #else + return simde_mm_sub_sd(simde_mm_mul_sd(a, b), c); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fmsub_sd + #define _mm_fmsub_sd(a, b, c) simde_mm_fmsub_sd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_fmsub_ss (simde__m128 a, simde__m128 b, simde__m128 c) { + #if defined(SIMDE_X86_FMA_NATIVE) && !defined(SIMDE_BUG_MCST_LCC_FMA_WRONG_RESULT) + return _mm_fmsub_ss(a, b, c); + #else + return simde_mm_sub_ss(simde_mm_mul_ss(a, b), c); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fmsub_ss + #define _mm_fmsub_ss(a, b, c) simde_mm_fmsub_ss(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_fmsubadd_pd (simde__m128d a, simde__m128d b, simde__m128d c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm_fmsubadd_pd(a, b, c); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b), + c_ = simde__m128d_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i += 2) { + r_.f64[ i ] = (a_.f64[ i ] * b_.f64[ i ]) + c_.f64[ i ]; + r_.f64[i + 1] = (a_.f64[i + 1] * b_.f64[i + 1]) - c_.f64[i + 1]; + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fmsubadd_pd + #define _mm_fmsubadd_pd(a, b, c) simde_mm_fmsubadd_pd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_fmsubadd_pd (simde__m256d a, simde__m256d b, simde__m256d c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm256_fmsubadd_pd(a, b, c); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b), + c_ = simde__m256d_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i += 2) { + r_.f64[ i ] = (a_.f64[ i ] * b_.f64[ i ]) + c_.f64[ i ]; + r_.f64[i + 1] = (a_.f64[i + 1] * b_.f64[i + 1]) - c_.f64[i + 1]; + } + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm256_fmsubadd_pd + #define _mm256_fmsubadd_pd(a, b, c) simde_mm256_fmsubadd_pd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_fmsubadd_ps (simde__m128 a, simde__m128 b, simde__m128 c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm_fmsubadd_ps(a, b, c); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b), + c_ = simde__m128_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i += 2) { + r_.f32[ i ] = (a_.f32[ i ] * b_.f32[ i ]) + c_.f32[ i ]; + r_.f32[i + 1] = (a_.f32[i + 1] * b_.f32[i + 1]) - c_.f32[i + 1]; + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fmsubadd_ps + #define _mm_fmsubadd_ps(a, b, c) simde_mm_fmsubadd_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_fmsubadd_ps (simde__m256 a, simde__m256 b, simde__m256 c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm256_fmsubadd_ps(a, b, c); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b), + c_ = simde__m256_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i += 2) { + r_.f32[ i ] = (a_.f32[ i ] * b_.f32[ i ]) + c_.f32[ i ]; + r_.f32[i + 1] = (a_.f32[i + 1] * b_.f32[i + 1]) - c_.f32[i + 1]; + } + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm256_fmsubadd_ps + #define _mm256_fmsubadd_ps(a, b, c) simde_mm256_fmsubadd_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_fnmadd_pd (simde__m128d a, simde__m128d b, simde__m128d c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm_fnmadd_pd(a, b, c); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b), + c_ = simde__m128d_to_private(c); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vfmsq_f64(c_.neon_f64, a_.neon_f64, b_.neon_f64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = -(a_.f64[i] * b_.f64[i]) + c_.f64[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fnmadd_pd + #define _mm_fnmadd_pd(a, b, c) simde_mm_fnmadd_pd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_fnmadd_pd (simde__m256d a, simde__m256d b, simde__m256d c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm256_fnmadd_pd(a, b, c); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b), + c_ = simde__m256d_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = -(a_.f64[i] * b_.f64[i]) + c_.f64[i]; + } + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm256_fnmadd_pd + #define _mm256_fnmadd_pd(a, b, c) simde_mm256_fnmadd_pd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_fnmadd_ps (simde__m128 a, simde__m128 b, simde__m128 c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm_fnmadd_ps(a, b, c); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b), + c_ = simde__m128_to_private(c); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) && defined(SIMDE_ARCH_ARM_FMA) + r_.neon_f32 = vfmsq_f32(c_.neon_f32, a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vmlsq_f32(c_.neon_f32, a_.neon_f32, b_.neon_f32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = -(a_.f32[i] * b_.f32[i]) + c_.f32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fnmadd_ps + #define _mm_fnmadd_ps(a, b, c) simde_mm_fnmadd_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_fnmadd_ps (simde__m256 a, simde__m256 b, simde__m256 c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm256_fnmadd_ps(a, b, c); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b), + c_ = simde__m256_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = -(a_.f32[i] * b_.f32[i]) + c_.f32[i]; + } + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm256_fnmadd_ps + #define _mm256_fnmadd_ps(a, b, c) simde_mm256_fnmadd_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_fnmadd_sd (simde__m128d a, simde__m128d b, simde__m128d c) { + #if defined(SIMDE_X86_FMA_NATIVE) && !defined(SIMDE_BUG_MCST_LCC_FMA_WRONG_RESULT) + return _mm_fnmadd_sd(a, b, c); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b), + c_ = simde__m128d_to_private(c); + + r_ = a_; + r_.f64[0] = -(a_.f64[0] * b_.f64[0]) + c_.f64[0]; + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fnmadd_sd + #define _mm_fnmadd_sd(a, b, c) simde_mm_fnmadd_sd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_fnmadd_ss (simde__m128 a, simde__m128 b, simde__m128 c) { + #if defined(SIMDE_X86_FMA_NATIVE) && !defined(SIMDE_BUG_MCST_LCC_FMA_WRONG_RESULT) + return _mm_fnmadd_ss(a, b, c); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b), + c_ = simde__m128_to_private(c); + + r_ = a_; + r_.f32[0] = -(a_.f32[0] * b_.f32[0]) + c_.f32[0]; + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fnmadd_ss + #define _mm_fnmadd_ss(a, b, c) simde_mm_fnmadd_ss(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_fnmsub_pd (simde__m128d a, simde__m128d b, simde__m128d c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm_fnmsub_pd(a, b, c); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b), + c_ = simde__m128d_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = -(a_.f64[i] * b_.f64[i]) - c_.f64[i]; + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fnmsub_pd + #define _mm_fnmsub_pd(a, b, c) simde_mm_fnmsub_pd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_fnmsub_pd (simde__m256d a, simde__m256d b, simde__m256d c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm256_fnmsub_pd(a, b, c); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b), + c_ = simde__m256d_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = -(a_.f64[i] * b_.f64[i]) - c_.f64[i]; + } + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm256_fnmsub_pd + #define _mm256_fnmsub_pd(a, b, c) simde_mm256_fnmsub_pd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_fnmsub_ps (simde__m128 a, simde__m128 b, simde__m128 c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm_fnmsub_ps(a, b, c); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b), + c_ = simde__m128_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = -(a_.f32[i] * b_.f32[i]) - c_.f32[i]; + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fnmsub_ps + #define _mm_fnmsub_ps(a, b, c) simde_mm_fnmsub_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_fnmsub_ps (simde__m256 a, simde__m256 b, simde__m256 c) { + #if defined(SIMDE_X86_FMA_NATIVE) + return _mm256_fnmsub_ps(a, b, c); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b), + c_ = simde__m256_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = -(a_.f32[i] * b_.f32[i]) - c_.f32[i]; + } + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm256_fnmsub_ps + #define _mm256_fnmsub_ps(a, b, c) simde_mm256_fnmsub_ps(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_fnmsub_sd (simde__m128d a, simde__m128d b, simde__m128d c) { + #if defined(SIMDE_X86_FMA_NATIVE) && !defined(SIMDE_BUG_MCST_LCC_FMA_WRONG_RESULT) + return _mm_fnmsub_sd(a, b, c); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b), + c_ = simde__m128d_to_private(c); + + r_ = a_; + r_.f64[0] = -(a_.f64[0] * b_.f64[0]) - c_.f64[0]; + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fnmsub_sd + #define _mm_fnmsub_sd(a, b, c) simde_mm_fnmsub_sd(a, b, c) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_fnmsub_ss (simde__m128 a, simde__m128 b, simde__m128 c) { + #if defined(SIMDE_X86_FMA_NATIVE) && !defined(SIMDE_BUG_MCST_LCC_FMA_WRONG_RESULT) + return _mm_fnmsub_ss(a, b, c); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b), + c_ = simde__m128_to_private(c); + + r_ = simde__m128_to_private(a); + r_.f32[0] = -(a_.f32[0] * b_.f32[0]) - c_.f32[0]; + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_FMA_ENABLE_NATIVE_ALIASES) + #undef _mm_fnmsub_ss + #define _mm_fnmsub_ss(a, b, c) simde_mm_fnmsub_ss(a, b, c) +#endif + +SIMDE_END_DECLS_ + +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_FMA_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/gfni.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/gfni.h new file mode 100644 index 00000000..5982a340 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/gfni.h @@ -0,0 +1,1295 @@ +/* Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020-2021 Christopher Moore + * 2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_GFNI_H) +#define SIMDE_X86_GFNI_H + +#include "avx512/add.h" +#include "avx512/and.h" +#include "avx512/broadcast.h" +#include "avx512/cmpeq.h" +#include "avx512/cmpge.h" +#include "avx512/cmpgt.h" +#include "avx512/cmplt.h" +#include "avx512/extract.h" +#include "avx512/insert.h" +#include "avx512/kshift.h" +#include "avx512/mov.h" +#include "avx512/mov_mask.h" +#include "avx512/permutex2var.h" +#include "avx512/set.h" +#include "avx512/set1.h" +#include "avx512/setzero.h" +#include "avx512/shuffle.h" +#include "avx512/srli.h" +#include "avx512/test.h" +#include "avx512/xor.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +/* In all the *gf2p8affine* intrinsics the argument b must be a compile-time constant so we must use macros and simde_x_mm* helper functions */ + +/* N.B. The _mm*gf2p8affineinv_epi64_epi8 and _mm*gf2p8mul_epi8 intrinsics are for a Field Generator Polynomial (FGP) (aka reduction polynomial) of 0x11B */ +/* Only the _mm*gf2p8affine_epi64_epi8 intrinsics do not assume this specific FGP */ + +/* The field generator polynomial is 0x11B but we make the 0x100 bit implicit to fit inside 8 bits */ +#define SIMDE_X86_GFNI_FGP 0x1B + +/* Computing the inverse of a GF element is expensive so use this LUT for an FGP of 0x11B */ + +static const union { + uint8_t u8[256]; + simde__m128i m128i[16]; +} simde_x_gf2p8inverse_lut = { + { + 0x00, 0x01, 0x8d, 0xf6, 0xcb, 0x52, 0x7b, 0xd1, 0xe8, 0x4f, 0x29, 0xc0, 0xb0, 0xe1, 0xe5, 0xc7, + 0x74, 0xb4, 0xaa, 0x4b, 0x99, 0x2b, 0x60, 0x5f, 0x58, 0x3f, 0xfd, 0xcc, 0xff, 0x40, 0xee, 0xb2, + 0x3a, 0x6e, 0x5a, 0xf1, 0x55, 0x4d, 0xa8, 0xc9, 0xc1, 0x0a, 0x98, 0x15, 0x30, 0x44, 0xa2, 0xc2, + 0x2c, 0x45, 0x92, 0x6c, 0xf3, 0x39, 0x66, 0x42, 0xf2, 0x35, 0x20, 0x6f, 0x77, 0xbb, 0x59, 0x19, + 0x1d, 0xfe, 0x37, 0x67, 0x2d, 0x31, 0xf5, 0x69, 0xa7, 0x64, 0xab, 0x13, 0x54, 0x25, 0xe9, 0x09, + 0xed, 0x5c, 0x05, 0xca, 0x4c, 0x24, 0x87, 0xbf, 0x18, 0x3e, 0x22, 0xf0, 0x51, 0xec, 0x61, 0x17, + 0x16, 0x5e, 0xaf, 0xd3, 0x49, 0xa6, 0x36, 0x43, 0xf4, 0x47, 0x91, 0xdf, 0x33, 0x93, 0x21, 0x3b, + 0x79, 0xb7, 0x97, 0x85, 0x10, 0xb5, 0xba, 0x3c, 0xb6, 0x70, 0xd0, 0x06, 0xa1, 0xfa, 0x81, 0x82, + 0x83, 0x7e, 0x7f, 0x80, 0x96, 0x73, 0xbe, 0x56, 0x9b, 0x9e, 0x95, 0xd9, 0xf7, 0x02, 0xb9, 0xa4, + 0xde, 0x6a, 0x32, 0x6d, 0xd8, 0x8a, 0x84, 0x72, 0x2a, 0x14, 0x9f, 0x88, 0xf9, 0xdc, 0x89, 0x9a, + 0xfb, 0x7c, 0x2e, 0xc3, 0x8f, 0xb8, 0x65, 0x48, 0x26, 0xc8, 0x12, 0x4a, 0xce, 0xe7, 0xd2, 0x62, + 0x0c, 0xe0, 0x1f, 0xef, 0x11, 0x75, 0x78, 0x71, 0xa5, 0x8e, 0x76, 0x3d, 0xbd, 0xbc, 0x86, 0x57, + 0x0b, 0x28, 0x2f, 0xa3, 0xda, 0xd4, 0xe4, 0x0f, 0xa9, 0x27, 0x53, 0x04, 0x1b, 0xfc, 0xac, 0xe6, + 0x7a, 0x07, 0xae, 0x63, 0xc5, 0xdb, 0xe2, 0xea, 0x94, 0x8b, 0xc4, 0xd5, 0x9d, 0xf8, 0x90, 0x6b, + 0xb1, 0x0d, 0xd6, 0xeb, 0xc6, 0x0e, 0xcf, 0xad, 0x08, 0x4e, 0xd7, 0xe3, 0x5d, 0x50, 0x1e, 0xb3, + 0x5b, 0x23, 0x38, 0x34, 0x68, 0x46, 0x03, 0x8c, 0xdd, 0x9c, 0x7d, 0xa0, 0xcd, 0x1a, 0x41, 0x1c + } +}; + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_gf2p8matrix_multiply_epi64_epi8 (simde__m128i x, simde__m128i A) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + const __m128i byte_select = _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1); + const __m128i zero = _mm_setzero_si128(); + __m128i r, a, p, X; + + a = _mm_shuffle_epi8(A, _mm_setr_epi8(7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8)); + X = x; + r = zero; + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 8 ; i++) { + p = _mm_insert_epi16(zero, _mm_movemask_epi8(a), 0); + p = _mm_shuffle_epi8(p, byte_select); + p = _mm_and_si128(p, _mm_cmpgt_epi8(zero, X)); + r = _mm_xor_si128(r, p); + a = _mm_add_epi8(a, a); + X = _mm_add_epi8(X, X); + } + + return r; + #elif defined(SIMDE_X86_SSE2_NATIVE) + const __m128i zero = _mm_setzero_si128(); + __m128i r, a, p, X; + + a = _mm_shufflehi_epi16(A, (0 << 6) + (1 << 4) + (2 << 2) + (3 << 0)); + a = _mm_shufflelo_epi16(a, (0 << 6) + (1 << 4) + (2 << 2) + (3 << 0)); + a = _mm_or_si128(_mm_slli_epi16(a, 8), _mm_srli_epi16(a, 8)); + X = _mm_unpacklo_epi8(x, _mm_unpackhi_epi64(x, x)); + r = zero; + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 8 ; i++) { + p = _mm_set1_epi16(HEDLEY_STATIC_CAST(short, _mm_movemask_epi8(a))); + p = _mm_and_si128(p, _mm_cmpgt_epi8(zero, X)); + r = _mm_xor_si128(r, p); + a = _mm_add_epi8(a, a); + X = _mm_add_epi8(X, X); + } + + return _mm_packus_epi16(_mm_srli_epi16(_mm_slli_epi16(r, 8), 8), _mm_srli_epi16(r, 8)); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + static const uint8_t byte_interleave[16] = {0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15}; + static const uint8_t byte_deinterleave[16] = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15}; + static const uint8_t mask_d[16] = {128, 128, 64, 64, 32, 32, 16, 16, 8, 8, 4, 4, 2, 2, 1, 1}; + const int8x16_t mask = vreinterpretq_s8_u8(vld1q_u8(mask_d)); + int8x16_t r, a, t, X; + + t = simde__m128i_to_neon_i8(A); + a = vqtbl1q_s8(t, vld1q_u8(byte_interleave)); + t = simde__m128i_to_neon_i8(x); + X = vqtbl1q_s8(t, vld1q_u8(byte_interleave)); + r = vdupq_n_s8(0); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 8 ; i++) { + t = vshrq_n_s8(a, 7); + t = vandq_s8(t, mask); + t = vreinterpretq_s8_u16(vdupq_n_u16(vaddvq_u16(vreinterpretq_u16_s8(t)))); + t = vandq_s8(t, vshrq_n_s8(X, 7)); + r = veorq_s8(r, t); + a = vshlq_n_s8(a, 1); + X = vshlq_n_s8(X, 1); + } + + r = vqtbl1q_s8(r, vld1q_u8(byte_deinterleave)); + return simde__m128i_from_neon_i8(r); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + static const uint8_t mask_d[16] = {128, 64, 32, 16, 8, 4, 2, 1, 128, 64, 32, 16, 8, 4, 2, 1}; + const int8x16_t mask = vreinterpretq_s8_u8(vld1q_u8(mask_d)); + int8x16_t r, a, t, X; + int16x8_t t16; + int32x4_t t32; + + a = simde__m128i_to_neon_i8(A); + X = simde__m128i_to_neon_i8(x); + r = vdupq_n_s8(0); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 8 ; i++) { + t = vshrq_n_s8(a, 7); + t = vandq_s8(t, mask); + t16 = vreinterpretq_s16_s8 (vorrq_s8 (t , vrev64q_s8 (t ))); + t32 = vreinterpretq_s32_s16(vorrq_s16(t16, vrev64q_s16(t16))); + t = vreinterpretq_s8_s32 (vorrq_s32(t32, vrev64q_s32(t32))); + t = vandq_s8(t, vshrq_n_s8(X, 7)); + r = veorq_s8(r, t); + a = vshlq_n_s8(a, 1); + X = vshlq_n_s8(X, 1); + } + + return simde__m128i_from_neon_i8(r); + #elif defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + static const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) byte_interleave = {0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15}; + static const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) byte_deinterleave= {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15}; + static const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) bit_select = {0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120}; + static const SIMDE_POWER_ALTIVEC_VECTOR(signed char) zero = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) a, p, r; + SIMDE_POWER_ALTIVEC_VECTOR(signed char) X; + + X = simde__m128i_to_altivec_i8(x); + a = simde__m128i_to_altivec_u8(A); + X = vec_perm(X, X, byte_interleave); + r = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), zero); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 8 ; i++) { + #if defined(SIMDE_BUG_CLANG_50932) + p = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), + vec_bperm(HEDLEY_STATIC_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned __int128), a), bit_select)); + #else + p = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), vec_bperm_u128(a, bit_select)); + #endif + p = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), + vec_splat(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned short), p), 3)); + p &= X < zero; + r ^= p; + a += a; + X += X; + } + + r = vec_perm(r, r, byte_deinterleave); + return simde__m128i_from_altivec_u8(r); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + static const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) mask = {128, 64, 32, 16, 8, 4, 2, 1, 128, 64, 32, 16, 8, 4, 2, 1}; + static const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) byte_select = {7, 7, 7, 7, 7, 7, 7, 7, 15, 15, 15, 15, 15, 15, 15, 15}; + static const SIMDE_POWER_ALTIVEC_VECTOR(signed char) zero = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) p, r; + SIMDE_POWER_ALTIVEC_VECTOR(signed char) a, X; + + X = simde__m128i_to_altivec_i8(x); + a = simde__m128i_to_altivec_i8(A); + r = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), zero); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 8 ; i++) { + p = a < zero; + p &= mask; + p = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), + vec_sum2(vec_sum4(p, HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), zero)), + HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned int), zero))); + p = vec_perm(p, p, byte_select); + p &= X < zero; + r ^= p; + a += a; + X += X; + } + + return simde__m128i_from_altivec_u8(r); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + static const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) byte_interleave = {0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15}; + static const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) byte_deinterleave= {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15}; + static const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) bit_select = {64, 72, 80, 88, 96, 104, 112, 120, 0, 8, 16, 24, 32, 40, 48, 56}; + const SIMDE_POWER_ALTIVEC_VECTOR(signed char) zero = vec_splats(HEDLEY_STATIC_CAST(signed char, 0)); + SIMDE_POWER_ALTIVEC_VECTOR(signed char) X; + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) a, p, r; + + X = simde__m128i_to_altivec_i8(x); + a = simde__m128i_to_altivec_u8(A); + X = vec_perm(X, X, byte_interleave); + r = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), zero); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 8 ; i++) { + #if defined(SIMDE_BUG_CLANG_50932) + p = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), + vec_bperm(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned __int128), a), bit_select)); + #else + p = vec_bperm(a, bit_select); + #endif + p = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), + vec_splat(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned short), p), 4)); + p = vec_and(p, vec_cmplt(X, zero)); + r = vec_xor(r, p); + a = vec_add(a, a); + X = vec_add(X, X); + } + + r = vec_perm(r, r, byte_deinterleave); + return simde__m128i_from_altivec_u8(r); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + static const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) mask = {128, 64, 32, 16, 8, 4, 2, 1, 128, 64, 32, 16, 8, 4, 2, 1}; + static const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) byte_select = {4, 4, 4, 4, 4, 4, 4, 4, 12, 12, 12, 12, 12, 12, 12, 12}; + const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) sevens = vec_splats(HEDLEY_STATIC_CAST(unsigned char, 7)); + const SIMDE_POWER_ALTIVEC_VECTOR(signed char) zero = vec_splats(HEDLEY_STATIC_CAST(signed char, 0)); + SIMDE_POWER_ALTIVEC_VECTOR(signed char) X; + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) a, p, r; + + X = simde__m128i_to_altivec_i8(x); + a = simde__m128i_to_altivec_u8(A); + r = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), zero); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 8 ; i++) { + p = vec_sr(a, sevens); + p = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), + vec_msum(p, + mask, + HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned int), zero))); + p = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), + vec_sum2s(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed int), p), + HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed int), zero))); + p = vec_perm(p, p, byte_select); + p = vec_and(p, vec_cmplt(X, zero)); + r = vec_xor(r, p); + a = vec_add(a, a); + X = vec_add(X, X); + } + + return simde__m128i_from_altivec_u8(r); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + const v128_t zero = wasm_i8x16_splat(0); + v128_t a, p, r, X; + + X = simde__m128i_to_wasm_v128(x); + a = simde__m128i_to_wasm_v128(A); + a = wasm_i8x16_shuffle(a, a, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8); + X = wasm_i8x16_shuffle(X, X, 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15); + r = zero; + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 8 ; i++) { + p = wasm_i16x8_splat(HEDLEY_STATIC_CAST(int16_t, wasm_i8x16_bitmask(a))); + p = wasm_v128_and(p, wasm_i8x16_lt(X, zero)); + r = wasm_v128_xor(r, p); + a = wasm_i8x16_add(a, a); + X = wasm_i8x16_add(X, X); + } + + r = wasm_i8x16_shuffle(r, r, 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15); + return simde__m128i_from_wasm_v128(r); + #else + simde__m128i_private + r_, + x_ = simde__m128i_to_private(x), + A_ = simde__m128i_to_private(A); + + const uint64_t ones = UINT64_C(0x0101010101010101); + const uint64_t mask = UINT64_C(0x0102040810204080); + uint64_t q; + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + q = simde_endian_bswap64_le(A_.u64[i / 8]); + q &= HEDLEY_STATIC_CAST(uint64_t, x_.u8[i]) * ones; + q ^= q >> 4; + q ^= q >> 2; + q ^= q >> 1; + q &= ones; + q *= 255; + q &= mask; + q |= q >> 32; + q |= q >> 16; + q |= q >> 8; + r_.u8[i] = HEDLEY_STATIC_CAST(uint8_t, q); + } + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_gf2p8matrix_multiply_epi64_epi8 (simde__m256i x, simde__m256i A) { + #if defined(SIMDE_X86_AVX2_NATIVE) + simde__m256i r, a, p; + const simde__m256i byte_select = simde_x_mm256_set_epu64x(UINT64_C(0x0303030303030303), UINT64_C(0x0202020202020202), + UINT64_C(0x0101010101010101), UINT64_C(0x0000000000000000)); + a = simde_mm256_shuffle_epi8(A, simde_mm256_broadcastsi128_si256(simde_x_mm_set_epu64x(UINT64_C(0x08090A0B0C0D0E0F), UINT64_C(0x0001020304050607)))); + r = simde_mm256_setzero_si256(); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 8 ; i++) { + p = simde_mm256_set1_epi32(simde_mm256_movemask_epi8(a)); + p = simde_mm256_shuffle_epi8(p, byte_select); + p = simde_mm256_xor_si256(r, p); + r = simde_mm256_blendv_epi8(r, p, x); + a = simde_mm256_add_epi8(a, a); + x = simde_mm256_add_epi8(x, x); + } + + return r; + #else + simde__m256i_private + r_, + x_ = simde__m256i_to_private(x), + A_ = simde__m256i_to_private(A); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_gf2p8matrix_multiply_epi64_epi8(x_.m128i[i], A_.m128i[i]); + } + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_gf2p8matrix_multiply_epi64_epi8 (simde__m512i x, simde__m512i A) { + #if defined(SIMDE_X86_AVX512BW_NATIVE) + simde__m512i r, a, p; + const simde__m512i byte_select = simde_x_mm512_set_epu64(UINT64_C(0x0707070707070707), UINT64_C(0x0606060606060606), UINT64_C(0x0505050505050505), UINT64_C(0x0404040404040404), + UINT64_C(0x0303030303030303), UINT64_C(0x0202020202020202), UINT64_C(0x0101010101010101), UINT64_C(0X0000000000000000)); + a = simde_mm512_shuffle_epi8(A, simde_mm512_broadcast_i32x4(simde_x_mm_set_epu64x(UINT64_C(0x08090A0B0C0D0E0F), UINT64_C(0x0001020304050607)))); + r = simde_mm512_setzero_si512(); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 8 ; i++) { + p = simde_mm512_set1_epi64(HEDLEY_STATIC_CAST(int64_t, simde_mm512_movepi8_mask(a))); + p = simde_mm512_maskz_shuffle_epi8(simde_mm512_movepi8_mask(x), p, byte_select); + r = simde_mm512_xor_si512(r, p); + a = simde_mm512_add_epi8(a, a); + x = simde_mm512_add_epi8(x, x); + } + + return r; + #else + simde__m512i_private + r_, + x_ = simde__m512i_to_private(x), + A_ = simde__m512i_to_private(A); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_gf2p8matrix_multiply_epi64_epi8(x_.m256i[i], A_.m256i[i]); + } + + return simde__m512i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_gf2p8inverse_epi8 (simde__m128i x) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + /* N.B. CM: this fallback may not be faster */ + simde__m128i r, u, t, test; + const simde__m128i sixteens = simde_mm_set1_epi8(16); + const simde__m128i masked_x = simde_mm_and_si128(x, simde_mm_set1_epi8(0x0F)); + + test = simde_mm_set1_epi8(INT8_MIN /* 0x80 */); + x = simde_mm_xor_si128(x, test); + r = simde_mm_shuffle_epi8(simde_x_gf2p8inverse_lut.m128i[0], masked_x); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 1 ; i < 16 ; i++) { + t = simde_mm_shuffle_epi8(simde_x_gf2p8inverse_lut.m128i[i], masked_x); + test = simde_mm_add_epi8(test, sixteens); + u = simde_mm_cmplt_epi8(x, test); + r = simde_mm_blendv_epi8(t, r, u); + } + + return r; + #else + simde__m128i_private + r_, + x_ = simde__m128i_to_private(x); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = simde_x_gf2p8inverse_lut.u8[x_.u8[i]]; + } + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_x_mm256_gf2p8inverse_epi8 (simde__m256i x) { + #if defined(SIMDE_X86_AVX2_NATIVE) + /* N.B. CM: this fallback may not be faster */ + simde__m256i r, u, t, test; + const simde__m256i sixteens = simde_mm256_set1_epi8(16); + const simde__m256i masked_x = simde_mm256_and_si256(x, simde_mm256_set1_epi8(0x0F)); + + test = simde_mm256_set1_epi8(INT8_MIN /* 0x80 */); + x = simde_mm256_xor_si256(x, test); + r = simde_mm256_shuffle_epi8(simde_mm256_broadcastsi128_si256(simde_x_gf2p8inverse_lut.m128i[0]), masked_x); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 1 ; i < 16 ; i++) { + t = simde_mm256_shuffle_epi8(simde_mm256_broadcastsi128_si256(simde_x_gf2p8inverse_lut.m128i[i]), masked_x); + test = simde_mm256_add_epi8(test, sixteens); + u = simde_mm256_cmpgt_epi8(test, x); + r = simde_mm256_blendv_epi8(t, r, u); + } + + return r; + #else + simde__m256i_private + r_, + x_ = simde__m256i_to_private(x); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_x_mm_gf2p8inverse_epi8(x_.m128i[i]); + } + + return simde__m256i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_x_mm512_gf2p8inverse_epi8 (simde__m512i x) { + /* N.B. CM: TODO: later add VBMI version using just two _mm512_permutex2var_epi8 and friends */ + /* But except for Cannon Lake all processors with VBMI also have GFNI */ + #if defined(SIMDE_X86_AVX512BW_NATIVE) + /* N.B. CM: this fallback may not be faster */ + simde__m512i r, test; + const simde__m512i sixteens = simde_mm512_set1_epi8(16); + const simde__m512i masked_x = simde_mm512_and_si512(x, simde_mm512_set1_epi8(0x0F)); + + r = simde_mm512_shuffle_epi8(simde_mm512_broadcast_i32x4(simde_x_gf2p8inverse_lut.m128i[0]), masked_x); + test = sixteens; + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 1 ; i < 16 ; i++) { + r = simde_mm512_mask_shuffle_epi8(r, simde_mm512_cmpge_epu8_mask(x, test), simde_mm512_broadcast_i32x4(simde_x_gf2p8inverse_lut.m128i[i]), masked_x); + test = simde_mm512_add_epi8(test, sixteens); + } + + return r; + #else + simde__m512i_private + r_, + x_ = simde__m512i_to_private(x); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_x_mm256_gf2p8inverse_epi8(x_.m256i[i]); + } + + return simde__m512i_from_private(r_); + #endif +} + +#define simde_x_mm_gf2p8matrix_multiply_inverse_epi64_epi8(x, A) simde_x_mm_gf2p8matrix_multiply_epi64_epi8(simde_x_mm_gf2p8inverse_epi8(x), A) +#define simde_x_mm256_gf2p8matrix_multiply_inverse_epi64_epi8(x, A) simde_x_mm256_gf2p8matrix_multiply_epi64_epi8(simde_x_mm256_gf2p8inverse_epi8(x), A) +#define simde_x_mm512_gf2p8matrix_multiply_inverse_epi64_epi8(x, A) simde_x_mm512_gf2p8matrix_multiply_epi64_epi8(simde_x_mm512_gf2p8inverse_epi8(x), A) + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_gf2p8affine_epi64_epi8 (simde__m128i x, simde__m128i A, int b) + SIMDE_REQUIRE_CONSTANT_RANGE(b, 0, 255) { + return simde_mm_xor_si128(simde_x_mm_gf2p8matrix_multiply_epi64_epi8(x, A), simde_mm_set1_epi8(HEDLEY_STATIC_CAST(int8_t, b))); +} +#if defined(SIMDE_X86_GFNI_NATIVE) + #define simde_mm_gf2p8affine_epi64_epi8(x, A, b) _mm_gf2p8affine_epi64_epi8(x, A, b) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm_gf2p8affine_epi64_epi8 + #define _mm_gf2p8affine_epi64_epi8(x, A, b) simde_mm_gf2p8affine_epi64_epi8(x, A, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_gf2p8affine_epi64_epi8 (simde__m256i x, simde__m256i A, int b) + SIMDE_REQUIRE_CONSTANT_RANGE(b, 0, 255) { + return simde_mm256_xor_si256(simde_x_mm256_gf2p8matrix_multiply_epi64_epi8(x, A), simde_mm256_set1_epi8(HEDLEY_STATIC_CAST(int8_t, b))); +} +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + #define simde_mm256_gf2p8affine_epi64_epi8(x, A, b) _mm256_gf2p8affine_epi64_epi8(x, A, b) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_gf2p8affine_epi64_epi8 + #define _mm256_gf2p8affine_epi64_epi8(x, A, b) simde_mm256_gf2p8affine_epi64_epi8(x, A, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_gf2p8affine_epi64_epi8 (simde__m512i x, simde__m512i A, int b) + SIMDE_REQUIRE_CONSTANT_RANGE(b, 0, 255) { + return simde_mm512_xor_si512(simde_x_mm512_gf2p8matrix_multiply_epi64_epi8(x, A), simde_mm512_set1_epi8(HEDLEY_STATIC_CAST(int8_t, b))); +} +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_gf2p8affine_epi64_epi8(x, A, b) _mm512_gf2p8affine_epi64_epi8(x, A, b) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_gf2p8affine_epi64_epi8 + #define _mm512_gf2p8affine_epi64_epi8(x, A, b) simde_mm512_gf2p8affine_epi64_epi8(x, A, b) +#endif + +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) _mm_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) +#else + #define simde_mm_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) simde_mm_mask_mov_epi8(src, k, simde_mm_gf2p8affine_epi64_epi8(x, A, b)) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_gf2p8affine_epi64_epi8 + #define _mm_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) simde_mm_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) +#endif + +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) _mm256_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) +#else + #define simde_mm256_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) simde_mm256_mask_mov_epi8(src, k, simde_mm256_gf2p8affine_epi64_epi8(x, A, b)) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_gf2p8affine_epi64_epi8 + #define _mm256_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) simde_mm256_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) +#endif + +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) _mm512_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) +#else + #define simde_mm512_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) simde_mm512_mask_mov_epi8(src, k, simde_mm512_gf2p8affine_epi64_epi8(x, A, b)) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_gf2p8affine_epi64_epi8 + #define _mm512_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) simde_mm512_mask_gf2p8affine_epi64_epi8(src, k, x, A, b) +#endif + +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_maskz_gf2p8affine_epi64_epi8(k, x, A, b) _mm_maskz_gf2p8affine_epi64_epi8(k, x, A, b) +#else + #define simde_mm_maskz_gf2p8affine_epi64_epi8(k, x, A, b) simde_mm_maskz_mov_epi8(k, simde_mm_gf2p8affine_epi64_epi8(x, A, b)) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_gf2p8affine_epi64_epi8 + #define _mm_maskz_gf2p8affine_epi64_epi8(k, x, A, b) simde_mm_maskz_gf2p8affine_epi64_epi8(k, x, A, b) +#endif + +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_maskz_gf2p8affine_epi64_epi8(k, x, A, b) _mm256_maskz_gf2p8affine_epi64_epi8(k, x, A, b) +#else + #define simde_mm256_maskz_gf2p8affine_epi64_epi8(k, x, A, b) simde_mm256_maskz_mov_epi8(k, simde_mm256_gf2p8affine_epi64_epi8(x, A, b)) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_gf2p8affine_epi64_epi8 + #define _mm256_maskz_gf2p8affine_epi64_epi8(k, x, A, b) simde_mm256_maskz_gf2p8affine_epi64_epi8(k, x, A, b) +#endif + +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_maskz_gf2p8affine_epi64_epi8(k, x, A, b) _mm512_maskz_gf2p8affine_epi64_epi8(k, x, A, b) +#else + #define simde_mm512_maskz_gf2p8affine_epi64_epi8(k, x, A, b) simde_mm512_maskz_mov_epi8(k, simde_mm512_gf2p8affine_epi64_epi8(x, A, b)) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_gf2p8affine_epi64_epi8 + #define _mm512_maskz_gf2p8affine_epi64_epi8(k, x, A, b) simde_mm512_maskz_gf2p8affine_epi64_epi8(k, x, A, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_gf2p8affineinv_epi64_epi8 (simde__m128i x, simde__m128i A, int b) + SIMDE_REQUIRE_CONSTANT_RANGE(b, 0, 255) { + return simde_mm_xor_si128(simde_x_mm_gf2p8matrix_multiply_inverse_epi64_epi8(x, A), simde_mm_set1_epi8(HEDLEY_STATIC_CAST(int8_t, b))); +} +#if defined(SIMDE_X86_GFNI_NATIVE) + #define simde_mm_gf2p8affineinv_epi64_epi8(x, A, b) _mm_gf2p8affineinv_epi64_epi8(x, A, b) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm_gf2p8affineinv_epi64_epi8 + #define _mm_gf2p8affineinv_epi64_epi8(x, A, b) simde_mm_gf2p8affineinv_epi64_epi8(x, A, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_gf2p8affineinv_epi64_epi8 (simde__m256i x, simde__m256i A, int b) + SIMDE_REQUIRE_CONSTANT_RANGE(b, 0, 255) { + return simde_mm256_xor_si256(simde_x_mm256_gf2p8matrix_multiply_inverse_epi64_epi8(x, A), simde_mm256_set1_epi8(HEDLEY_STATIC_CAST(int8_t, b))); +} +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + #define simde_mm256_gf2p8affineinv_epi64_epi8(x, A, b) _mm256_gf2p8affineinv_epi64_epi8(x, A, b) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_gf2p8affineinv_epi64_epi8 + #define _mm256_gf2p8affineinv_epi64_epi8(x, A, b) simde_mm256_gf2p8affineinv_epi64_epi8(x, A, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_gf2p8affineinv_epi64_epi8 (simde__m512i x, simde__m512i A, int b) + SIMDE_REQUIRE_CONSTANT_RANGE(b, 0, 255) { + return simde_mm512_xor_si512(simde_x_mm512_gf2p8matrix_multiply_inverse_epi64_epi8(x, A), simde_mm512_set1_epi8(HEDLEY_STATIC_CAST(int8_t, b))); +} +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_gf2p8affineinv_epi64_epi8(x, A, b) _mm512_gf2p8affineinv_epi64_epi8(x, A, b) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_gf2p8affineinv_epi64_epi8 + #define _mm512_gf2p8affineinv_epi64_epi8(x, A, b) simde_mm512_gf2p8affineinv_epi64_epi8(x, A, b) +#endif + +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) _mm_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) +#else + #define simde_mm_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) simde_mm_mask_mov_epi8(src, k, simde_mm_gf2p8affineinv_epi64_epi8(x, A, b)) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_gf2p8affineinv_epi64_epi8 + #define _mm_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) simde_mm_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) +#endif + +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) _mm256_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) +#else + #define simde_mm256_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) simde_mm256_mask_mov_epi8(src, k, simde_mm256_gf2p8affineinv_epi64_epi8(x, A, b)) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_gf2p8affineinv_epi64_epi8 + #define _mm256_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) simde_mm256_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) +#endif + +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) _mm512_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) +#else + #define simde_mm512_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) simde_mm512_mask_mov_epi8(src, k, simde_mm512_gf2p8affineinv_epi64_epi8(x, A, b)) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_gf2p8affineinv_epi64_epi8 + #define _mm512_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) simde_mm512_mask_gf2p8affineinv_epi64_epi8(src, k, x, A, b) +#endif + +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) _mm_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) +#else + #define simde_mm_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) simde_mm_maskz_mov_epi8(k, simde_mm_gf2p8affineinv_epi64_epi8(x, A, b)) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_gf2p8affineinv_epi64_epi8 + #define _mm_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) simde_mm_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) +#endif + +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + #define simde_mm256_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) _mm256_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) +#else + #define simde_mm256_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) simde_mm256_maskz_mov_epi8(k, simde_mm256_gf2p8affineinv_epi64_epi8(x, A, b)) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_gf2p8affineinv_epi64_epi8 + #define _mm256_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) simde_mm256_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) +#endif + +#if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + #define simde_mm512_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) _mm512_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) +#else + #define simde_mm512_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) simde_mm512_maskz_mov_epi8(k, simde_mm512_gf2p8affineinv_epi64_epi8(x, A, b)) +#endif +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_gf2p8affineinv_epi64_epi8 + #define _mm512_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) simde_mm512_maskz_gf2p8affineinv_epi64_epi8(k, x, A, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i simde_mm_gf2p8mul_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_GFNI_NATIVE) && (defined(SIMDE_X86_AVX512VL_NATIVE) || !defined(SIMDE_X86_AVX512F_NATIVE)) + return _mm_gf2p8mul_epi8(a, b); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const poly8x16_t pa = vreinterpretq_p8_u8(simde__m128i_to_neon_u8(a)); + const poly8x16_t pb = vreinterpretq_p8_u8(simde__m128i_to_neon_u8(b)); + const uint8x16_t lo = vreinterpretq_u8_p16(vmull_p8(vget_low_p8(pa), vget_low_p8(pb))); + #if defined (SIMDE_ARM_NEON_A64V8_NATIVE) + uint8x16_t hi = vreinterpretq_u8_p16(vmull_high_p8(pa, pb)); + #else + uint8x16_t hi = vreinterpretq_u8_p16(vmull_p8(vget_high_p8(pa), vget_high_p8(pb))); + #endif + uint8x16x2_t hilo = vuzpq_u8(lo, hi); + uint8x16_t r = hilo.val[0]; + hi = hilo.val[1]; + const uint8x16_t idxHi = vshrq_n_u8(hi, 4); + const uint8x16_t idxLo = vandq_u8(hi, vdupq_n_u8(0xF)); + + #if defined (SIMDE_ARM_NEON_A64V8_NATIVE) + static const uint8_t reduceLutHiData[] = { + 0x00, 0xab, 0x4d, 0xe6, 0x9a, 0x31, 0xd7, 0x7c, + 0x2f, 0x84, 0x62, 0xc9, 0xb5, 0x1e, 0xf8, 0x53 + }; + static const uint8_t reduceLutLoData[] = { + 0x00, 0x1b, 0x36, 0x2d, 0x6c, 0x77, 0x5a, 0x41, + 0xd8, 0xc3, 0xee, 0xf5, 0xb4, 0xaf, 0x82, 0x99 + }; + const uint8x16_t reduceLutHi = vld1q_u8(reduceLutHiData); + const uint8x16_t reduceLutLo = vld1q_u8(reduceLutLoData); + r = veorq_u8(r, vqtbl1q_u8(reduceLutHi, idxHi)); + r = veorq_u8(r, vqtbl1q_u8(reduceLutLo, idxLo)); + #else + static const uint8_t reduceLutHiData[] = { + 0x00, 0x2f, + 0xab, 0x84, + 0x4d, 0x62, + 0xe6, 0xc9, + 0x9a, 0xb5, + 0x31, 0x1e, + 0xd7, 0xf8, + 0x7c, 0x53 + }; + static const uint8_t reduceLutLoData[] = { + 0x00, 0xd8, + 0x1b, 0xc3, + 0x36, 0xee, + 0x2d, 0xf5, + 0x6c, 0xb4, + 0x77, 0xaf, + 0x5a, 0x82, + 0x41, 0x99 + }; + const uint8x8x2_t reduceLutHi = vld2_u8(reduceLutHiData); + const uint8x8x2_t reduceLutLo = vld2_u8(reduceLutLoData); + r = veorq_u8(r, vcombine_u8(vtbl2_u8(reduceLutHi, vget_low_u8(idxHi)), vtbl2_u8(reduceLutHi, vget_high_u8(idxHi)))); + r = veorq_u8(r, vcombine_u8(vtbl2_u8(reduceLutLo, vget_low_u8(idxLo)), vtbl2_u8(reduceLutLo, vget_high_u8(idxLo)))); + #endif + return simde__m128i_from_neon_u8(r); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) x, y, lo, hi; + SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) even, odd, mask0x00FF; + x = simde__m128i_to_altivec_u8(a); + y = simde__m128i_to_altivec_u8(b); + mask0x00FF = vec_splats(HEDLEY_STATIC_CAST(unsigned short, 0x00FF)); + lo = y & HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), mask0x00FF); + hi = y ^ lo; + even = vec_gfmsum(x, lo); + odd = vec_gfmsum(x, hi); + lo = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), vec_sel(vec_rli(odd, 8), even, mask0x00FF)); + hi = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), vec_sel(odd, vec_rli(even, 8), mask0x00FF)); + const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) reduceLutHi = {0x00, 0xab, 0x4d, 0xe6, 0x9a, 0x31, 0xd7, 0x7c, 0x2f, 0x84, 0x62, 0xc9, 0xb5, 0x1e, 0xf8, 0x53}; + const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) reduceLutLo = {0x00, 0x1b, 0x36, 0x2d, 0x6c, 0x77, 0x5a, 0x41, 0xd8, 0xc3, 0xee, 0xf5, 0xb4, 0xaf, 0x82, 0x99}; + lo = lo ^ vec_perm(reduceLutHi, reduceLutHi, vec_rli(hi, 4)); + lo = lo ^ vec_perm(reduceLutLo, reduceLutLo, hi); + return simde__m128i_from_altivec_u8(lo); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) x, y, r, t, m; + x = simde__m128i_to_altivec_u8(a); + y = simde__m128i_to_altivec_u8(b); + + const SIMDE_POWER_ALTIVEC_VECTOR(signed char) zero = vec_splat_s8(0); + + m = vec_splat_u8(0x01); + + const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) fgp = vec_splats(HEDLEY_STATIC_CAST(unsigned char, SIMDE_X86_GFNI_FGP)); + t = vec_and(y, m); + t = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), vec_cmpeq(t, m)); + r = vec_and(x, t); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 7 ; i++) { + t = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), vec_cmplt(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed char), x), zero)); + x = vec_add(x, x); + t = vec_and(fgp, t); + x = vec_xor(x, t); + m = vec_add(m, m); + t = vec_and(y, m); + t = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), vec_cmpeq(t, m)); + t = vec_and(x, t); + r = vec_xor(r, t); + } + + return simde__m128i_from_altivec_u8(r); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t x, y, r, t, m; + x = simde__m128i_to_wasm_v128(a); + y = simde__m128i_to_wasm_v128(b); + + m = wasm_i8x16_splat(0x01); + + const v128_t fgp = wasm_i8x16_splat(SIMDE_X86_GFNI_FGP); + + t = wasm_v128_and(y, m); + t = wasm_i8x16_eq(t, m); + r = wasm_v128_and(x, t); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 7 ; i++) { + t = wasm_i8x16_shr(x, 7); + x = wasm_i8x16_add(x, x); + t = wasm_v128_and(fgp, t); + x = wasm_v128_xor(x, t); + m = wasm_i8x16_add(m, m); + t = wasm_v128_and(y, m); + t = wasm_i8x16_eq(t, m); + t = wasm_v128_and(x, t); + r = wasm_v128_xor(r, t); + } + + return simde__m128i_from_wasm_v128(r); + #elif defined(SIMDE_X86_AVX512BW_NATIVE) + simde__m512i r4, t4, u4; + simde__mmask64 ma, mb; + + simde__m512i a4 = simde_mm512_broadcast_i32x4(a); + const simde__m512i zero = simde_mm512_setzero_si512(); + simde__mmask16 m8 = simde_mm512_cmpeq_epi32_mask(zero, zero); + + const simde__m512i b4 = simde_mm512_broadcast_i32x4(b); + + simde__m512i bits = simde_mm512_set_epi64(0x4040404040404040, + 0x4040404040404040, + 0x1010101010101010, + 0x1010101010101010, + 0x0404040404040404, + 0x0404040404040404, + 0x0101010101010101, + 0x0101010101010101); + + const simde__m512i fgp = simde_mm512_set1_epi8(SIMDE_X86_GFNI_FGP); + + for (int i = 0 ; i < 3 ; i++) { + m8 = simde_kshiftli_mask16(m8, 4); + + ma = simde_mm512_cmplt_epi8_mask(a4, zero); + u4 = simde_mm512_add_epi8(a4, a4); + t4 = simde_mm512_maskz_mov_epi8(ma, fgp); + u4 = simde_mm512_xor_epi32(u4, t4); + + ma = simde_mm512_cmplt_epi8_mask(u4, zero); + u4 = simde_mm512_add_epi8(u4, u4); + t4 = simde_mm512_maskz_mov_epi8(ma, fgp); + a4 = simde_mm512_mask_xor_epi32(a4, m8, u4, t4); + } + + mb = simde_mm512_test_epi8_mask(b4, bits); + bits = simde_mm512_add_epi8(bits, bits); + ma = simde_mm512_cmplt_epi8_mask(a4, zero); + r4 = simde_mm512_maskz_mov_epi8(mb, a4); + mb = simde_mm512_test_epi8_mask(b4, bits); + a4 = simde_mm512_add_epi8(a4, a4); + t4 = simde_mm512_maskz_mov_epi8(ma, fgp); + a4 = simde_mm512_xor_si512(a4, t4); + t4 = simde_mm512_maskz_mov_epi8(mb, a4); + r4 = simde_mm512_xor_si512(r4, t4); + + r4 = simde_mm512_xor_si512(r4, simde_mm512_shuffle_i32x4(r4, r4, (1 << 6) + (0 << 4) + (3 << 2) + 2)); + r4 = simde_mm512_xor_si512(r4, simde_mm512_shuffle_i32x4(r4, r4, (0 << 6) + (3 << 4) + (2 << 2) + 1)); + + return simde_mm512_extracti32x4_epi32(r4, 0); + #elif defined(SIMDE_X86_AVX2_NATIVE) + simde__m256i r2, t2; + simde__m256i a2 = simde_mm256_broadcastsi128_si256(a); + const simde__m256i zero = simde_mm256_setzero_si256(); + const simde__m256i fgp = simde_mm256_set1_epi8(SIMDE_X86_GFNI_FGP); + const simde__m256i ones = simde_mm256_set1_epi8(0x01); + simde__m256i b2 = simde_mm256_set_m128i(simde_mm_srli_epi64(b, 4), b); + + for (int i = 0 ; i < 4 ; i++) { + t2 = simde_mm256_cmpgt_epi8(zero, a2); + t2 = simde_mm256_and_si256(fgp, t2); + a2 = simde_mm256_add_epi8(a2, a2); + a2 = simde_mm256_xor_si256(a2, t2); + } + + a2 = simde_mm256_inserti128_si256(a2, a, 0); + + t2 = simde_mm256_and_si256(b2, ones); + t2 = simde_mm256_cmpeq_epi8(t2, ones); + r2 = simde_mm256_and_si256(a2, t2); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 3 ; i++) { + t2 = simde_mm256_cmpgt_epi8(zero, a2); + t2 = simde_mm256_and_si256(fgp, t2); + a2 = simde_mm256_add_epi8(a2, a2); + a2 = simde_mm256_xor_si256(a2, t2); + b2 = simde_mm256_srli_epi64(b2, 1); + t2 = simde_mm256_and_si256(b2, ones); + t2 = simde_mm256_cmpeq_epi8(t2, ones); + t2 = simde_mm256_and_si256(a2, t2); + r2 = simde_mm256_xor_si256(r2, t2); + } + + return simde_mm_xor_si128(simde_mm256_extracti128_si256(r2, 1), + simde_mm256_extracti128_si256(r2, 0)); + #elif defined(SIMDE_X86_SSE2_NATIVE) + simde__m128i r, t; + const simde__m128i zero = simde_mm_setzero_si128(); + const simde__m128i ones = simde_mm_set1_epi8(0x01); + + const simde__m128i fgp = simde_mm_set1_epi8(SIMDE_X86_GFNI_FGP); + + t = simde_mm_and_si128(b, ones); + t = simde_mm_cmpeq_epi8(t, ones); + r = simde_mm_and_si128(a, t); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 7 ; i++) { + t = simde_mm_cmpgt_epi8(zero, a); + t = simde_mm_and_si128(fgp, t); + a = simde_mm_add_epi8(a, a); + a = simde_mm_xor_si128(a, t); + b = simde_mm_srli_epi64(b, 1); + t = simde_mm_and_si128(b, ones); + t = simde_mm_cmpeq_epi8(t, ones); + t = simde_mm_and_si128(a, t); + r = simde_mm_xor_si128(r, t); + } + + return r; + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + const uint8_t fgp = SIMDE_X86_GFNI_FGP; + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = 0; + while ((a_.u8[i] != 0) && (b_.u8[i] != 0)) { + if (b_.u8[i] & 1) + r_.u8[i] ^= a_.u8[i]; + + if (a_.u8[i] & 0x80) + a_.u8[i] = HEDLEY_STATIC_CAST(uint8_t, (a_.u8[i] << 1) ^ fgp); + else + a_.u8[i] <<= 1; + + b_.u8[i] >>= 1; + } + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_gf2p8mul_epi8 + #define _mm_gf2p8mul_epi8(a, b) simde_mm_gf2p8mul_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_gf2p8mul_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_GFNI_NATIVE) && (defined(SIMDE_X86_AVX512VL_NATIVE) || (defined(SIMDE_X86_AVX_NATIVE) && !defined(SIMDE_X86_AVX512F_NATIVE))) + return _mm256_gf2p8mul_epi8(a, b); + #elif !defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + simde__mmask64 ma, mb; + simde__m512i r, t, s; + simde__m512i a2 = simde_mm512_broadcast_i64x4(a); + const simde__m512i zero = simde_mm512_setzero_si512(); + + const simde__m512i fgp = simde_mm512_set1_epi8(SIMDE_X86_GFNI_FGP); + + s = simde_mm512_set1_epi8(0x01); + + for (int i = 0 ; i < 4 ; i++) { + ma = simde_mm512_cmplt_epi8_mask(a2, zero); + a2 = simde_mm512_add_epi8(a2, a2); + t = simde_mm512_xor_si512(a2, fgp); + a2 = simde_mm512_mask_mov_epi8(a2, ma, t); + } + + simde__m512i b2 = simde_mm512_inserti64x4(zero, simde_mm256_srli_epi64(b, 4), 1); + b2 = simde_mm512_inserti64x4(b2, b, 0); + a2 = simde_mm512_inserti64x4(a2, a, 0); + + mb = simde_mm512_test_epi8_mask(b2, s); + r = simde_mm512_maskz_mov_epi8(mb, a2); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 3 ; i++) { + ma = simde_mm512_cmplt_epi8_mask(a2, zero); + s = simde_mm512_add_epi8(s, s); + mb = simde_mm512_test_epi8_mask(b2, s); + a2 = simde_mm512_add_epi8(a2, a2); + t = simde_mm512_maskz_mov_epi8(ma, fgp); + a2 = simde_mm512_xor_si512(a2, t); + t = simde_mm512_maskz_mov_epi8(mb, a2); + r = simde_mm512_xor_si512(r, t); + } + + return simde_mm256_xor_si256(simde_mm512_extracti64x4_epi64(r, 1), + simde_mm512_extracti64x4_epi64(r, 0)); + #elif !defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX2_NATIVE) + simde__m256i r, t; + const simde__m256i zero = simde_mm256_setzero_si256(); + const simde__m256i ones = simde_mm256_set1_epi8(0x01); + + const simde__m256i fgp = simde_mm256_set1_epi8(SIMDE_X86_GFNI_FGP); + + t = simde_mm256_and_si256(b, ones); + t = simde_mm256_cmpeq_epi8(t, ones); + r = simde_mm256_and_si256(a, t); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 7 ; i++) { + t = simde_mm256_cmpgt_epi8(zero, a); + t = simde_mm256_and_si256(fgp, t); + a = simde_mm256_add_epi8(a, a); + a = simde_mm256_xor_si256(a, t); + b = simde_mm256_srli_epi64(b, 1); + t = simde_mm256_and_si256(b, ones); + t = simde_mm256_cmpeq_epi8(t, ones); + t = simde_mm256_and_si256(a, t); + r = simde_mm256_xor_si256(r, t); + } + + return r; + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_gf2p8mul_epi8(a_.m128i[i], b_.m128i[i]); + } + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX_ENABLE_NATIVE_ALIASES) + #undef _mm256_gf2p8mul_epi8 + #define _mm256_gf2p8mul_epi8(a, b) simde_mm256_gf2p8mul_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_gf2p8mul_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_gf2p8mul_epi8(a, b); + #elif !defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) + simde__m512i r, s, t; + simde__mmask64 ma, mb; + const simde__m512i zero = simde_mm512_setzero_si512(); + + const simde__m512i fgp = simde_mm512_set1_epi8(SIMDE_X86_GFNI_FGP); + + s = simde_mm512_set1_epi8(0x01); + + mb = simde_mm512_test_epi8_mask(b, s); + r = simde_mm512_maskz_mov_epi8(mb, a); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (int i = 0 ; i < 7 ; i++) { + ma = simde_mm512_cmplt_epi8_mask(a, zero); + s = simde_mm512_add_epi8(s, s); + mb = simde_mm512_test_epi8_mask(b, s); + a = simde_mm512_add_epi8(a, a); + t = simde_mm512_maskz_mov_epi8(ma, fgp); + a = simde_mm512_xor_si512(a, t); + t = simde_mm512_maskz_mov_epi8(mb, a); + r = simde_mm512_xor_si512(r, t); + } + + return r; + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if !defined(__INTEL_COMPILER) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_gf2p8mul_epi8(a_.m128i[i], b_.m128i[i]); + } + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_gf2p8mul_epi8 + #define _mm512_gf2p8mul_epi8(a, b) simde_mm512_gf2p8mul_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mask_gf2p8mul_epi8 (simde__m128i src, simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_mask_gf2p8mul_epi8(src, k, a, b); + #else + return simde_mm_mask_mov_epi8(src, k, simde_mm_gf2p8mul_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_mask_gf2p8mul_epi8 + #define _mm_mask_gf2p8mul_epi8(src, k, a, b) simde_mm_mask_gf2p8mul_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_mask_gf2p8mul_epi8 (simde__m256i src, simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_mask_gf2p8mul_epi8(src, k, a, b); + #else + return simde_mm256_mask_mov_epi8(src, k, simde_mm256_gf2p8mul_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_mask_gf2p8mul_epi8 + #define _mm256_mask_gf2p8mul_epi8(src, k, a, b) simde_mm256_mask_gf2p8mul_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_gf2p8mul_epi8 (simde__m512i src, simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_gf2p8mul_epi8(src, k, a, b); + #else + return simde_mm512_mask_mov_epi8(src, k, simde_mm512_gf2p8mul_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_gf2p8mul_epi8 + #define _mm512_mask_gf2p8mul_epi8(src, k, a, b) simde_mm512_mask_gf2p8mul_epi8(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maskz_gf2p8mul_epi8 (simde__mmask16 k, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_maskz_gf2p8mul_epi8(k, a, b); + #else + return simde_mm_maskz_mov_epi8(k, simde_mm_gf2p8mul_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm_maskz_gf2p8mul_epi8 + #define _mm_maskz_gf2p8mul_epi8(k, a, b) simde_mm_maskz_gf2p8mul_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_maskz_gf2p8mul_epi8 (simde__mmask32 k, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_maskz_gf2p8mul_epi8(k, a, b); + #else + return simde_mm256_maskz_mov_epi8(k, simde_mm256_gf2p8mul_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) + #undef _mm256_maskz_gf2p8mul_epi8 + #define _mm256_maskz_gf2p8mul_epi8(k, a, b) simde_mm256_maskz_gf2p8mul_epi8(k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_maskz_gf2p8mul_epi8 (simde__mmask64 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_GFNI_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_maskz_gf2p8mul_epi8(k, a, b); + #else + return simde_mm512_maskz_mov_epi8(k, simde_mm512_gf2p8mul_epi8(a, b)); + #endif +} +#if defined(SIMDE_X86_GFNI_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512F_ENABLE_NATIVE_ALIASES) + #undef _mm512_maskz_gf2p8mul_epi8 + #define _mm512_maskz_gf2p8mul_epi8(k, a, b) simde_mm512_maskz_gf2p8mul_epi8(k, a, b) +#endif + +SIMDE_END_DECLS_ + +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_GFNI_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/mmx.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/mmx.h new file mode 100644 index 00000000..e294af8e --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/mmx.h @@ -0,0 +1,2398 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2017-2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_MMX_H) +#define SIMDE_X86_MMX_H + +#include "../simde-common.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS + +#if defined(SIMDE_X86_MMX_NATIVE) + #define SIMDE_X86_MMX_USE_NATIVE_TYPE +#elif defined(SIMDE_X86_SSE_NATIVE) + #define SIMDE_X86_MMX_USE_NATIVE_TYPE +#endif + +#if defined(SIMDE_X86_MMX_USE_NATIVE_TYPE) + #include +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #include +#elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + #include +#endif + +#include +#include + +SIMDE_BEGIN_DECLS_ + +typedef union { + #if defined(SIMDE_VECTOR_SUBSCRIPT) + SIMDE_ALIGN_TO_8 int8_t i8 SIMDE_VECTOR(8) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_8 int16_t i16 SIMDE_VECTOR(8) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_8 int32_t i32 SIMDE_VECTOR(8) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_8 int64_t i64 SIMDE_VECTOR(8) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_8 uint8_t u8 SIMDE_VECTOR(8) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_8 uint16_t u16 SIMDE_VECTOR(8) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_8 uint32_t u32 SIMDE_VECTOR(8) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_8 uint64_t u64 SIMDE_VECTOR(8) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_8 simde_float32 f32 SIMDE_VECTOR(8) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_8 int_fast32_t i32f SIMDE_VECTOR(8) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_8 uint_fast32_t u32f SIMDE_VECTOR(8) SIMDE_MAY_ALIAS; + #else + SIMDE_ALIGN_TO_8 int8_t i8[8]; + SIMDE_ALIGN_TO_8 int16_t i16[4]; + SIMDE_ALIGN_TO_8 int32_t i32[2]; + SIMDE_ALIGN_TO_8 int64_t i64[1]; + SIMDE_ALIGN_TO_8 uint8_t u8[8]; + SIMDE_ALIGN_TO_8 uint16_t u16[4]; + SIMDE_ALIGN_TO_8 uint32_t u32[2]; + SIMDE_ALIGN_TO_8 uint64_t u64[1]; + SIMDE_ALIGN_TO_8 simde_float32 f32[2]; + SIMDE_ALIGN_TO_8 int_fast32_t i32f[8 / sizeof(int_fast32_t)]; + SIMDE_ALIGN_TO_8 uint_fast32_t u32f[8 / sizeof(uint_fast32_t)]; + #endif + + #if defined(SIMDE_X86_MMX_USE_NATIVE_TYPE) + __m64 n; + #endif + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int8x8_t neon_i8; + int16x4_t neon_i16; + int32x2_t neon_i32; + int64x1_t neon_i64; + uint8x8_t neon_u8; + uint16x4_t neon_u16; + uint32x2_t neon_u32; + uint64x1_t neon_u64; + float32x2_t neon_f32; + #endif + #if defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + int8x8_t mmi_i8; + int16x4_t mmi_i16; + int32x2_t mmi_i32; + int64_t mmi_i64; + uint8x8_t mmi_u8; + uint16x4_t mmi_u16; + uint32x2_t mmi_u32; + uint64_t mmi_u64; + #endif +} simde__m64_private; + +#if defined(SIMDE_X86_MMX_USE_NATIVE_TYPE) + typedef __m64 simde__m64; +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + typedef int32x2_t simde__m64; +#elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + typedef int32x2_t simde__m64; +#elif defined(SIMDE_VECTOR_SUBSCRIPT) + typedef int32_t simde__m64 SIMDE_ALIGN_TO_8 SIMDE_VECTOR(8) SIMDE_MAY_ALIAS; +#else + typedef simde__m64_private simde__m64; +#endif + +#if !defined(SIMDE_X86_MMX_USE_NATIVE_TYPE) && defined(SIMDE_ENABLE_NATIVE_ALIASES) + #define SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES + typedef simde__m64 __m64; +#endif + +HEDLEY_STATIC_ASSERT(8 == sizeof(simde__m64), "__m64 size incorrect"); +HEDLEY_STATIC_ASSERT(8 == sizeof(simde__m64_private), "__m64 size incorrect"); +#if defined(SIMDE_CHECK_ALIGNMENT) && defined(SIMDE_ALIGN_OF) +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m64) == 8, "simde__m64 is not 8-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m64_private) == 8, "simde__m64_private is not 8-byte aligned"); +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde__m64_from_private(simde__m64_private v) { + simde__m64 r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64_private +simde__m64_to_private(simde__m64 v) { + simde__m64_private r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +#define SIMDE_X86_GENERATE_CONVERSION_FUNCTION(simde_type, source_type, isax, fragment) \ + SIMDE_FUNCTION_ATTRIBUTES \ + simde__##simde_type \ + simde__##simde_type##_from_##isax##_##fragment(source_type value) { \ + simde__##simde_type##_private r_; \ + r_.isax##_##fragment = value; \ + return simde__##simde_type##_from_private(r_); \ + } \ + \ + SIMDE_FUNCTION_ATTRIBUTES \ + source_type \ + simde__##simde_type##_to_##isax##_##fragment(simde__##simde_type value) { \ + simde__##simde_type##_private r_ = simde__##simde_type##_to_private(value); \ + return r_.isax##_##fragment; \ + } + +#if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, int8x8_t, neon, i8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, int16x4_t, neon, i16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, int32x2_t, neon, i32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, int64x1_t, neon, i64) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, uint8x8_t, neon, u8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, uint16x4_t, neon, u16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, uint32x2_t, neon, u32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, uint64x1_t, neon, u64) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, float32x2_t, neon, f32) +#endif /* defined(SIMDE_ARM_NEON_A32V7_NATIVE) */ + +#if defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, int8x8_t, mmi, i8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, int16x4_t, mmi, i16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, int32x2_t, mmi, i32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, int64_t, mmi, i64) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, uint8x8_t, mmi, u8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, uint16x4_t, mmi, u16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, uint32x2_t, mmi, u32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m64, uint64_t, mmi, u64) +#endif /* defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) */ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_add_pi8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_add_pi8(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vadd_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i8 = paddb_s(a_.mmi_i8, b_.mmi_i8); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = a_.i8 + b_.i8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] + b_.i8[i]; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_paddb(a, b) simde_mm_add_pi8(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_add_pi8(a, b) simde_mm_add_pi8(a, b) +# define _m_paddb(a, b) simde_m_paddb(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_add_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_add_pi16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vadd_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i16 = paddh_s(a_.mmi_i16, b_.mmi_i16); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = a_.i16 + b_.i16; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] + b_.i16[i]; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_paddw(a, b) simde_mm_add_pi16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_add_pi16(a, b) simde_mm_add_pi16(a, b) +# define _m_paddw(a, b) simde_mm_add_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_add_pi32 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_add_pi32(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vadd_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i32 = paddw_s(a_.mmi_i32, b_.mmi_i32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 + b_.i32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] + b_.i32[i]; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_paddd(a, b) simde_mm_add_pi32(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_add_pi32(a, b) simde_mm_add_pi32(a, b) +# define _m_paddd(a, b) simde_mm_add_pi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_adds_pi8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_adds_pi8(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vqadd_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i8 = paddsb(a_.mmi_i8, b_.mmi_i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + if ((((b_.i8[i]) > 0) && ((a_.i8[i]) > (INT8_MAX - (b_.i8[i]))))) { + r_.i8[i] = INT8_MAX; + } else if ((((b_.i8[i]) < 0) && ((a_.i8[i]) < (INT8_MIN - (b_.i8[i]))))) { + r_.i8[i] = INT8_MIN; + } else { + r_.i8[i] = (a_.i8[i]) + (b_.i8[i]); + } + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_paddsb(a, b) simde_mm_adds_pi8(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_adds_pi8(a, b) simde_mm_adds_pi8(a, b) +# define _m_paddsb(a, b) simde_mm_adds_pi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_adds_pu8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_adds_pu8(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vqadd_u8(a_.neon_u8, b_.neon_u8); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_u8 = paddusb(a_.mmi_u8, b_.mmi_u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + const uint_fast16_t x = HEDLEY_STATIC_CAST(uint_fast16_t, a_.u8[i]) + HEDLEY_STATIC_CAST(uint_fast16_t, b_.u8[i]); + if (x > UINT8_MAX) + r_.u8[i] = UINT8_MAX; + else + r_.u8[i] = HEDLEY_STATIC_CAST(uint8_t, x); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_paddusb(a, b) simde_mm_adds_pu8(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_adds_pu8(a, b) simde_mm_adds_pu8(a, b) +# define _m_paddusb(a, b) simde_mm_adds_pu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_adds_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_adds_pi16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vqadd_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i16 = paddsh(a_.mmi_i16, b_.mmi_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + if ((((b_.i16[i]) > 0) && ((a_.i16[i]) > (INT16_MAX - (b_.i16[i]))))) { + r_.i16[i] = INT16_MAX; + } else if ((((b_.i16[i]) < 0) && ((a_.i16[i]) < (SHRT_MIN - (b_.i16[i]))))) { + r_.i16[i] = SHRT_MIN; + } else { + r_.i16[i] = (a_.i16[i]) + (b_.i16[i]); + } + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_paddsw(a, b) simde_mm_adds_pi16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_adds_pi16(a, b) simde_mm_adds_pi16(a, b) +# define _m_paddsw(a, b) simde_mm_adds_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_adds_pu16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_adds_pu16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vqadd_u16(a_.neon_u16, b_.neon_u16); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_u16 = paddush(a_.mmi_u16, b_.mmi_u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + const uint32_t x = a_.u16[i] + b_.u16[i]; + if (x > UINT16_MAX) + r_.u16[i] = UINT16_MAX; + else + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, x); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_paddusw(a, b) simde_mm_adds_pu16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_adds_pu16(a, b) simde_mm_adds_pu16(a, b) +# define _m_paddusw(a, b) simde_mm_adds_pu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_and_si64 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_and_si64(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vand_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 & b_.i64; + #else + r_.i64[0] = a_.i64[0] & b_.i64[0]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pand(a, b) simde_mm_and_si64(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_and_si64(a, b) simde_mm_and_si64(a, b) +# define _m_pand(a, b) simde_mm_and_si64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_andnot_si64 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_andnot_si64(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vbic_s32(b_.neon_i32, a_.neon_i32); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i32 = pandn_sw(a_.mmi_i32, b_.mmi_i32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = ~a_.i32f & b_.i32f; + #else + r_.u64[0] = (~(a_.u64[0])) & (b_.u64[0]); + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pandn(a, b) simde_mm_andnot_si64(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_andnot_si64(a, b) simde_mm_andnot_si64(a, b) +# define _m_pandn(a, b) simde_mm_andnot_si64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cmpeq_pi8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_cmpeq_pi8(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vceq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i8 = pcmpeqb_s(a_.mmi_i8, b_.mmi_i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] == b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pcmpeqb(a, b) simde_mm_cmpeq_pi8(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_cmpeq_pi8(a, b) simde_mm_cmpeq_pi8(a, b) +# define _m_pcmpeqb(a, b) simde_mm_cmpeq_pi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cmpeq_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_cmpeq_pi16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vceq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i16 = pcmpeqh_s(a_.mmi_i16, b_.mmi_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] == b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pcmpeqw(a, b) simde_mm_cmpeq_pi16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_cmpeq_pi16(a, b) simde_mm_cmpeq_pi16(a, b) +# define _m_pcmpeqw(a, b) simde_mm_cmpeq_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cmpeq_pi32 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_cmpeq_pi32(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vceq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i32 = pcmpeqw_s(a_.mmi_i32, b_.mmi_i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] == b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pcmpeqd(a, b) simde_mm_cmpeq_pi32(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_cmpeq_pi32(a, b) simde_mm_cmpeq_pi32(a, b) +# define _m_pcmpeqd(a, b) simde_mm_cmpeq_pi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cmpgt_pi8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_cmpgt_pi8(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcgt_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i8 = pcmpgtb_s(a_.mmi_i8, b_.mmi_i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] > b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pcmpgtb(a, b) simde_mm_cmpgt_pi8(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_cmpgt_pi8(a, b) simde_mm_cmpgt_pi8(a, b) +# define _m_pcmpgtb(a, b) simde_mm_cmpgt_pi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cmpgt_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_cmpgt_pi16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vcgt_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i16 = pcmpgth_s(a_.mmi_i16, b_.mmi_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] > b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pcmpgtw(a, b) simde_mm_cmpgt_pi16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_cmpgt_pi16(a, b) simde_mm_cmpgt_pi16(a, b) +# define _m_pcmpgtw(a, b) simde_mm_cmpgt_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cmpgt_pi32 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_cmpgt_pi32(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcgt_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i32 = pcmpgtw_s(a_.mmi_i32, b_.mmi_i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] > b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pcmpgtd(a, b) simde_mm_cmpgt_pi32(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_cmpgt_pi32(a, b) simde_mm_cmpgt_pi32(a, b) +# define _m_pcmpgtd(a, b) simde_mm_cmpgt_pi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int64_t +simde_mm_cvtm64_si64 (simde__m64 a) { + #if defined(SIMDE_X86_MMX_NATIVE) && defined(SIMDE_ARCH_AMD64) && !defined(__PGI) + return _mm_cvtm64_si64(a); + #else + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + HEDLEY_DIAGNOSTIC_PUSH + #if HEDLEY_HAS_WARNING("-Wvector-conversion") && SIMDE_DETECT_CLANG_VERSION_NOT(10,0,0) + #pragma clang diagnostic ignored "-Wvector-conversion" + #endif + return vget_lane_s64(a_.neon_i64, 0); + HEDLEY_DIAGNOSTIC_POP + #else + return a_.i64[0]; + #endif + #endif +} +#define simde_m_to_int64(a) simde_mm_cvtm64_si64(a) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) +# define _mm_cvtm64_si64(a) simde_mm_cvtm64_si64(a) +# define _m_to_int64(a) simde_mm_cvtm64_si64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cvtsi32_si64 (int32_t a) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvtsi32_si64(a); + #else + simde__m64_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const int32_t av[2] = { a, 0 }; + r_.neon_i32 = vld1_s32(av); + #else + r_.i32[0] = a; + r_.i32[1] = 0; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_from_int(a) simde_mm_cvtsi32_si64(a) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_cvtsi32_si64(a) simde_mm_cvtsi32_si64(a) +# define _m_from_int(a) simde_mm_cvtsi32_si64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cvtsi64_m64 (int64_t a) { + #if defined(SIMDE_X86_MMX_NATIVE) && defined(SIMDE_ARCH_AMD64) && !defined(__PGI) + return _mm_cvtsi64_m64(a); + #else + simde__m64_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vld1_s64(&a); + #else + r_.i64[0] = a; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_from_int64(a) simde_mm_cvtsi64_m64(a) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) +# define _mm_cvtsi64_m64(a) simde_mm_cvtsi64_m64(a) +# define _m_from_int64(a) simde_mm_cvtsi64_m64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm_cvtsi64_si32 (simde__m64 a) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvtsi64_si32(a); + #else + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + HEDLEY_DIAGNOSTIC_PUSH + #if HEDLEY_HAS_WARNING("-Wvector-conversion") && SIMDE_DETECT_CLANG_VERSION_NOT(10,0,0) + #pragma clang diagnostic ignored "-Wvector-conversion" + #endif + return vget_lane_s32(a_.neon_i32, 0); + HEDLEY_DIAGNOSTIC_POP + #else + return a_.i32[0]; + #endif + #endif +} +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_cvtsi64_si32(a) simde_mm_cvtsi64_si32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_empty (void) { + #if defined(SIMDE_X86_MMX_NATIVE) + _mm_empty(); + #else + /* noop */ + #endif +} +#define simde_m_empty() simde_mm_empty() +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_empty() simde_mm_empty() +# define _m_empty() simde_mm_empty() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_madd_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_madd_pi16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int32x4_t i1 = vmull_s16(a_.neon_i16, b_.neon_i16); + r_.neon_i32 = vpadd_s32(vget_low_s32(i1), vget_high_s32(i1)); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i32 = pmaddhw(a_.mmi_i16, b_.mmi_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i += 2) { + r_.i32[i / 2] = (a_.i16[i] * b_.i16[i]) + (a_.i16[i + 1] * b_.i16[i + 1]); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pmaddwd(a, b) simde_mm_madd_pi16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_madd_pi16(a, b) simde_mm_madd_pi16(a, b) +# define _m_pmaddwd(a, b) simde_mm_madd_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_mulhi_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_mulhi_pi16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const int32x4_t t1 = vmull_s16(a_.neon_i16, b_.neon_i16); + const uint32x4_t t2 = vshrq_n_u32(vreinterpretq_u32_s32(t1), 16); + const uint16x4_t t3 = vmovn_u32(t2); + r_.neon_u16 = t3; + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i16 = pmulhh(a_.mmi_i16, b_.mmi_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, ((a_.i16[i] * b_.i16[i]) >> 16)); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pmulhw(a, b) simde_mm_mulhi_pi16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_mulhi_pi16(a, b) simde_mm_mulhi_pi16(a, b) +# define _m_pmulhw(a, b) simde_mm_mulhi_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_mullo_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_mullo_pi16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const int32x4_t t1 = vmull_s16(a_.neon_i16, b_.neon_i16); + const uint16x4_t t2 = vmovn_u32(vreinterpretq_u32_s32(t1)); + r_.neon_u16 = t2; + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i16 = pmullh(a_.mmi_i16, b_.mmi_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, ((a_.i16[i] * b_.i16[i]) & 0xffff)); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pmullw(a, b) simde_mm_mullo_pi16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_mullo_pi16(a, b) simde_mm_mullo_pi16(a, b) +# define _m_pmullw(a, b) simde_mm_mullo_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_or_si64 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_or_si64(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vorr_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 | b_.i64; + #else + r_.i64[0] = a_.i64[0] | b_.i64[0]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_por(a, b) simde_mm_or_si64(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_or_si64(a, b) simde_mm_or_si64(a, b) +# define _m_por(a, b) simde_mm_or_si64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_packs_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_packs_pi16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vqmovn_s16(vcombine_s16(a_.neon_i16, b_.neon_i16)); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i8 = packsshb(a_.mmi_i16, b_.mmi_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + if (a_.i16[i] < INT8_MIN) { + r_.i8[i] = INT8_MIN; + } else if (a_.i16[i] > INT8_MAX) { + r_.i8[i] = INT8_MAX; + } else { + r_.i8[i] = HEDLEY_STATIC_CAST(int8_t, a_.i16[i]); + } + } + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + if (b_.i16[i] < INT8_MIN) { + r_.i8[i + 4] = INT8_MIN; + } else if (b_.i16[i] > INT8_MAX) { + r_.i8[i + 4] = INT8_MAX; + } else { + r_.i8[i + 4] = HEDLEY_STATIC_CAST(int8_t, b_.i16[i]); + } + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_packsswb(a, b) simde_mm_packs_pi16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_packs_pi16(a, b) simde_mm_packs_pi16(a, b) +# define _m_packsswb(a, b) simde_mm_packs_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_packs_pi32 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_packs_pi32(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vqmovn_s32(vcombine_s32(a_.neon_i32, b_.neon_i32)); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i16 = packsswh(a_.mmi_i32, b_.mmi_i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (8 / sizeof(a_.i32[0])) ; i++) { + if (a_.i32[i] < SHRT_MIN) { + r_.i16[i] = SHRT_MIN; + } else if (a_.i32[i] > INT16_MAX) { + r_.i16[i] = INT16_MAX; + } else { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, a_.i32[i]); + } + } + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (8 / sizeof(b_.i32[0])) ; i++) { + if (b_.i32[i] < SHRT_MIN) { + r_.i16[i + 2] = SHRT_MIN; + } else if (b_.i32[i] > INT16_MAX) { + r_.i16[i + 2] = INT16_MAX; + } else { + r_.i16[i + 2] = HEDLEY_STATIC_CAST(int16_t, b_.i32[i]); + } + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_packssdw(a, b) simde_mm_packs_pi32(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_packs_pi32(a, b) simde_mm_packs_pi32(a, b) +# define _m_packssdw(a, b) simde_mm_packs_pi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_packs_pu16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_packs_pu16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + const int16x8_t t1 = vcombine_s16(a_.neon_i16, b_.neon_i16); + + /* Set elements which are < 0 to 0 */ + const int16x8_t t2 = vandq_s16(t1, vreinterpretq_s16_u16(vcgezq_s16(t1))); + + /* Vector with all s16 elements set to UINT8_MAX */ + const int16x8_t vmax = vmovq_n_s16(HEDLEY_STATIC_CAST(int16_t, UINT8_MAX)); + + /* Elements which are within the acceptable range */ + const int16x8_t le_max = vandq_s16(t2, vreinterpretq_s16_u16(vcleq_s16(t2, vmax))); + const int16x8_t gt_max = vandq_s16(vmax, vreinterpretq_s16_u16(vcgtq_s16(t2, vmax))); + + /* Final values as 16-bit integers */ + const int16x8_t values = vorrq_s16(le_max, gt_max); + + r_.neon_u8 = vmovn_u16(vreinterpretq_u16_s16(values)); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_u8 = packushb(a_.mmi_u16, b_.mmi_u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + if (a_.i16[i] > UINT8_MAX) { + r_.u8[i] = UINT8_MAX; + } else if (a_.i16[i] < 0) { + r_.u8[i] = 0; + } else { + r_.u8[i] = HEDLEY_STATIC_CAST(uint8_t, a_.i16[i]); + } + } + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + if (b_.i16[i] > UINT8_MAX) { + r_.u8[i + 4] = UINT8_MAX; + } else if (b_.i16[i] < 0) { + r_.u8[i + 4] = 0; + } else { + r_.u8[i + 4] = HEDLEY_STATIC_CAST(uint8_t, b_.i16[i]); + } + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_packuswb(a, b) simde_mm_packs_pu16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_packs_pu16(a, b) simde_mm_packs_pu16(a, b) +# define _m_packuswb(a, b) simde_mm_packs_pu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_set_pi8 (int8_t e7, int8_t e6, int8_t e5, int8_t e4, int8_t e3, int8_t e2, int8_t e1, int8_t e0) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_set_pi8(e7, e6, e5, e4, e3, e2, e1, e0); + #else + simde__m64_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const int8_t v[sizeof(r_.i8) / sizeof(r_.i8[0])] = { e0, e1, e2, e3, e4, e5, e6, e7 }; + r_.neon_i8 = vld1_s8(v); + #else + r_.i8[0] = e0; + r_.i8[1] = e1; + r_.i8[2] = e2; + r_.i8[3] = e3; + r_.i8[4] = e4; + r_.i8[5] = e5; + r_.i8[6] = e6; + r_.i8[7] = e7; + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_set_pi8(e7, e6, e5, e4, e3, e2, e1, e0) simde_mm_set_pi8(e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_x_mm_set_pu8 (uint8_t e7, uint8_t e6, uint8_t e5, uint8_t e4, uint8_t e3, uint8_t e2, uint8_t e1, uint8_t e0) { + simde__m64_private r_; + + #if defined(SIMDE_X86_MMX_NATIVE) + r_.n = _mm_set_pi8( + HEDLEY_STATIC_CAST(int8_t, e7), + HEDLEY_STATIC_CAST(int8_t, e6), + HEDLEY_STATIC_CAST(int8_t, e5), + HEDLEY_STATIC_CAST(int8_t, e4), + HEDLEY_STATIC_CAST(int8_t, e3), + HEDLEY_STATIC_CAST(int8_t, e2), + HEDLEY_STATIC_CAST(int8_t, e1), + HEDLEY_STATIC_CAST(int8_t, e0)); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const uint8_t v[sizeof(r_.u8) / sizeof(r_.u8[0])] = { e0, e1, e2, e3, e4, e5, e6, e7 }; + r_.neon_u8 = vld1_u8(v); + #else + r_.u8[0] = e0; + r_.u8[1] = e1; + r_.u8[2] = e2; + r_.u8[3] = e3; + r_.u8[4] = e4; + r_.u8[5] = e5; + r_.u8[6] = e6; + r_.u8[7] = e7; + #endif + + return simde__m64_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_set_pi16 (int16_t e3, int16_t e2, int16_t e1, int16_t e0) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_set_pi16(e3, e2, e1, e0); + #else + simde__m64_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const int16_t v[sizeof(r_.i16) / sizeof(r_.i16[0])] = { e0, e1, e2, e3 }; + r_.neon_i16 = vld1_s16(v); + #else + r_.i16[0] = e0; + r_.i16[1] = e1; + r_.i16[2] = e2; + r_.i16[3] = e3; + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_set_pi16(e3, e2, e1, e0) simde_mm_set_pi16(e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_x_mm_set_pu16 (uint16_t e3, uint16_t e2, uint16_t e1, uint16_t e0) { + simde__m64_private r_; + +#if defined(SIMDE_X86_MMX_NATIVE) + r_.n = _mm_set_pi16( + HEDLEY_STATIC_CAST(int16_t, e3), + HEDLEY_STATIC_CAST(int16_t, e2), + HEDLEY_STATIC_CAST(int16_t, e1), + HEDLEY_STATIC_CAST(int16_t, e0) + ); +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const uint16_t v[sizeof(r_.u16) / sizeof(r_.u16[0])] = { e0, e1, e2, e3 }; + r_.neon_u16 = vld1_u16(v); +#else + r_.u16[0] = e0; + r_.u16[1] = e1; + r_.u16[2] = e2; + r_.u16[3] = e3; +#endif + + return simde__m64_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_x_mm_set_pu32 (uint32_t e1, uint32_t e0) { + simde__m64_private r_; + +#if defined(SIMDE_X86_MMX_NATIVE) + r_.n = _mm_set_pi32( + HEDLEY_STATIC_CAST(int32_t, e1), + HEDLEY_STATIC_CAST(int32_t, e0)); +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const uint32_t v[sizeof(r_.u32) / sizeof(r_.u32[0])] = { e0, e1 }; + r_.neon_u32 = vld1_u32(v); +#else + r_.u32[0] = e0; + r_.u32[1] = e1; +#endif + + return simde__m64_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_set_pi32 (int32_t e1, int32_t e0) { + simde__m64_private r_; + +#if defined(SIMDE_X86_MMX_NATIVE) + r_.n = _mm_set_pi32(e1, e0); +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const int32_t v[sizeof(r_.i32) / sizeof(r_.i32[0])] = { e0, e1 }; + r_.neon_i32 = vld1_s32(v); +#else + r_.i32[0] = e0; + r_.i32[1] = e1; +#endif + + return simde__m64_from_private(r_); +} +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_set_pi32(e1, e0) simde_mm_set_pi32(e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_x_mm_set_pi64 (int64_t e0) { + simde__m64_private r_; + +#if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const int64_t v[sizeof(r_.i64) / sizeof(r_.i64[0])] = { e0 }; + r_.neon_i64 = vld1_s64(v); +#else + r_.i64[0] = e0; +#endif + + return simde__m64_from_private(r_); +} + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_x_mm_set_f32x2 (simde_float32 e1, simde_float32 e0) { + simde__m64_private r_; + +#if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const simde_float32 v[sizeof(r_.f32) / sizeof(r_.f32[0])] = { e0, e1 }; + r_.neon_f32 = vld1_f32(v); +#else + r_.f32[0] = e0; + r_.f32[1] = e1; +#endif + + return simde__m64_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_set1_pi8 (int8_t a) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_set1_pi8(a); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + simde__m64_private r_; + r_.neon_i8 = vmov_n_s8(a); + return simde__m64_from_private(r_); + #else + return simde_mm_set_pi8(a, a, a, a, a, a, a, a); + #endif +} +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_set1_pi8(a) simde_mm_set1_pi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_set1_pi16 (int16_t a) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_set1_pi16(a); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + simde__m64_private r_; + r_.neon_i16 = vmov_n_s16(a); + return simde__m64_from_private(r_); + #else + return simde_mm_set_pi16(a, a, a, a); + #endif +} +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_set1_pi16(a) simde_mm_set1_pi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_set1_pi32 (int32_t a) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_set1_pi32(a); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + simde__m64_private r_; + r_.neon_i32 = vmov_n_s32(a); + return simde__m64_from_private(r_); + #else + return simde_mm_set_pi32(a, a); + #endif +} +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_set1_pi32(a) simde_mm_set1_pi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_setr_pi8 (int8_t e7, int8_t e6, int8_t e5, int8_t e4, int8_t e3, int8_t e2, int8_t e1, int8_t e0) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_setr_pi8(e7, e6, e5, e4, e3, e2, e1, e0); + #else + return simde_mm_set_pi8(e0, e1, e2, e3, e4, e5, e6, e7); + #endif +} +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_setr_pi8(e7, e6, e5, e4, e3, e2, e1, e0) simde_mm_setr_pi8(e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_setr_pi16 (int16_t e3, int16_t e2, int16_t e1, int16_t e0) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_setr_pi16(e3, e2, e1, e0); + #else + return simde_mm_set_pi16(e0, e1, e2, e3); + #endif +} +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_setr_pi16(e3, e2, e1, e0) simde_mm_setr_pi16(e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_setr_pi32 (int32_t e1, int32_t e0) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_setr_pi32(e1, e0); + #else + return simde_mm_set_pi32(e0, e1); + #endif +} +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_setr_pi32(e1, e0) simde_mm_setr_pi32(e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_setzero_si64 (void) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_setzero_si64(); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + simde__m64_private r_; + r_.neon_u32 = vmov_n_u32(0); + return simde__m64_from_private(r_); + #else + return simde_mm_set_pi32(0, 0); + #endif +} +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_setzero_si64() simde_mm_setzero_si64() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_x_mm_load_si64 (const void* mem_addr) { + simde__m64 r; + simde_memcpy(&r, SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m64), sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_x_mm_loadu_si64 (const void* mem_addr) { + simde__m64 r; + simde_memcpy(&r, mem_addr, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_x_mm_store_si64 (void* mem_addr, simde__m64 value) { + simde_memcpy(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m64), &value, sizeof(value)); +} + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_x_mm_storeu_si64 (void* mem_addr, simde__m64 value) { + simde_memcpy(mem_addr, &value, sizeof(value)); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_x_mm_setone_si64 (void) { + return simde_mm_set1_pi32(~INT32_C(0)); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_sll_pi16 (simde__m64 a, simde__m64 count) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_sll_pi16(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private count_ = simde__m64_to_private(count); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + HEDLEY_DIAGNOSTIC_PUSH + #if HEDLEY_HAS_WARNING("-Wvector-conversion") && SIMDE_DETECT_CLANG_VERSION_NOT(10,0,0) + #pragma clang diagnostic ignored "-Wvector-conversion" + #endif + r_.neon_i16 = vshl_s16(a_.neon_i16, vmov_n_s16(HEDLEY_STATIC_CAST(int16_t, vget_lane_u64(count_.neon_u64, 0)))); + HEDLEY_DIAGNOSTIC_POP + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && defined(SIMDE_BUG_CLANG_POWER9_16x4_BAD_SHIFT) + if (HEDLEY_UNLIKELY(count_.u64[0] > 15)) + return simde_mm_setzero_si64(); + + r_.i16 = a_.i16 << HEDLEY_STATIC_CAST(int16_t, count_.u64[0]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i16 = a_.i16 << count_.u64[0]; + #else + if (HEDLEY_UNLIKELY(count_.u64[0] > 15)) { + simde_memset(&r_, 0, sizeof(r_)); + return simde__m64_from_private(r_); + } + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, a_.u16[i] << count_.u64[0]); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psllw(a, count) simde_mm_sll_pi16(a, count) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_sll_pi16(a, count) simde_mm_sll_pi16(a, count) +# define _m_psllw(a, count) simde_mm_sll_pi16(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_sll_pi32 (simde__m64 a, simde__m64 count) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_sll_pi32(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private count_ = simde__m64_to_private(count); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + HEDLEY_DIAGNOSTIC_PUSH + #if HEDLEY_HAS_WARNING("-Wvector-conversion") && SIMDE_DETECT_CLANG_VERSION_NOT(10,0,0) + #pragma clang diagnostic ignored "-Wvector-conversion" + #endif + r_.neon_i32 = vshl_s32(a_.neon_i32, vmov_n_s32(HEDLEY_STATIC_CAST(int32_t, vget_lane_u64(count_.neon_u64, 0)))); + HEDLEY_DIAGNOSTIC_POP + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i32 = a_.i32 << count_.u64[0]; + #else + if (HEDLEY_UNLIKELY(count_.u64[0] > 31)) { + simde_memset(&r_, 0, sizeof(r_)); + return simde__m64_from_private(r_); + } + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] << count_.u64[0]; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pslld(a, count) simde_mm_sll_pi32(a, count) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_sll_pi32(a, count) simde_mm_sll_pi32(a, count) +# define _m_pslld(a, count) simde_mm_sll_pi32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_slli_pi16 (simde__m64 a, int count) { + #if defined(SIMDE_X86_MMX_NATIVE) && !defined(__PGI) + return _mm_slli_pi16(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i16 = psllh_s(a_.mmi_i16, count); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && defined(SIMDE_BUG_CLANG_POWER9_16x4_BAD_SHIFT) + if (HEDLEY_UNLIKELY(count > 15)) + return simde_mm_setzero_si64(); + + r_.i16 = a_.i16 << HEDLEY_STATIC_CAST(int16_t, count); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i16 = a_.i16 << count; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vshl_s16(a_.neon_i16, vmov_n_s16((int16_t) count)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, a_.u16[i] << count); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psllwi(a, count) simde_mm_slli_pi16(a, count) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_slli_pi16(a, count) simde_mm_slli_pi16(a, count) +# define _m_psllwi(a, count) simde_mm_slli_pi16(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_slli_pi32 (simde__m64 a, int count) { + #if defined(SIMDE_X86_MMX_NATIVE) && !defined(__PGI) + return _mm_slli_pi32(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i32 = a_.i32 << count; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vshl_s32(a_.neon_i32, vmov_n_s32((int32_t) count)); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i32 = psllw_s(a_.mmi_i32, b_.mmi_i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] << count; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pslldi(a, b) simde_mm_slli_pi32(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_slli_pi32(a, count) simde_mm_slli_pi32(a, count) +# define _m_pslldi(a, count) simde_mm_slli_pi32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_slli_si64 (simde__m64 a, int count) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_slli_si64(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i64 = a_.i64 << count; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vshl_s64(a_.neon_i64, vmov_n_s64((int64_t) count)); + #else + r_.u64[0] = a_.u64[0] << count; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psllqi(a, count) simde_mm_slli_si64(a, count) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_slli_si64(a, count) simde_mm_slli_si64(a, count) +# define _m_psllqi(a, count) simde_mm_slli_si64(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_sll_si64 (simde__m64 a, simde__m64 count) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_sll_si64(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private count_ = simde__m64_to_private(count); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vshl_s64(a_.neon_i64, count_.neon_i64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 << count_.i64; + #else + if (HEDLEY_UNLIKELY(count_.u64[0] > 63)) { + simde_memset(&r_, 0, sizeof(r_)); + return simde__m64_from_private(r_); + } + + r_.u64[0] = a_.u64[0] << count_.u64[0]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psllq(a, count) simde_mm_sll_si64(a, count) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_sll_si64(a, count) simde_mm_sll_si64(a, count) +# define _m_psllq(a, count) simde_mm_sll_si64(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_srl_pi16 (simde__m64 a, simde__m64 count) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_srl_pi16(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private count_ = simde__m64_to_private(count); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && defined(SIMDE_BUG_CLANG_POWER9_16x4_BAD_SHIFT) + if (HEDLEY_UNLIKELY(count_.u64[0] > 15)) + return simde_mm_setzero_si64(); + + r_.u16 = a_.u16 >> HEDLEY_STATIC_CAST(uint16_t, count_.u64[0]); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u16 = a_.u16 >> count_.u64[0]; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vshl_u16(a_.neon_u16, vmov_n_s16(-((int16_t) vget_lane_u64(count_.neon_u64, 0)))); + #else + if (HEDLEY_UNLIKELY(count_.u64[0] > 15)) { + simde_memset(&r_, 0, sizeof(r_)); + return simde__m64_from_private(r_); + } + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < sizeof(r_.u16) / sizeof(r_.u16[0]) ; i++) { + r_.u16[i] = a_.u16[i] >> count_.u64[0]; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psrlw(a, count) simde_mm_srl_pi16(a, count) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_srl_pi16(a, count) simde_mm_srl_pi16(a, count) +# define _m_psrlw(a, count) simde_mm_srl_pi16(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_srl_pi32 (simde__m64 a, simde__m64 count) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_srl_pi32(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private count_ = simde__m64_to_private(count); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = a_.u32 >> count_.u64[0]; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vshl_u32(a_.neon_u32, vmov_n_s32(-((int32_t) vget_lane_u64(count_.neon_u64, 0)))); + #else + if (HEDLEY_UNLIKELY(count_.u64[0] > 31)) { + simde_memset(&r_, 0, sizeof(r_)); + return simde__m64_from_private(r_); + } + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < sizeof(r_.u32) / sizeof(r_.u32[0]) ; i++) { + r_.u32[i] = a_.u32[i] >> count_.u64[0]; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psrld(a, count) simde_mm_srl_pi32(a, count) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_srl_pi32(a, count) simde_mm_srl_pi32(a, count) +# define _m_psrld(a, count) simde_mm_srl_pi32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_srli_pi16 (simde__m64 a, int count) { + #if defined(SIMDE_X86_MMX_NATIVE) && !defined(__PGI) + return _mm_srli_pi16(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u16 = a_.u16 >> count; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vshl_u16(a_.neon_u16, vmov_n_s16(-((int16_t) count))); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i16 = psrlh_s(a_.mmi_i16, b_.mmi_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = a_.u16[i] >> count; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psrlwi(a, count) simde_mm_srli_pi16(a, count) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_srli_pi16(a, count) simde_mm_srli_pi16(a, count) +# define _m_psrlwi(a, count) simde_mm_srli_pi16(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_srli_pi32 (simde__m64 a, int count) { + #if defined(SIMDE_X86_MMX_NATIVE) && !defined(__PGI) + return _mm_srli_pi32(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = a_.u32 >> count; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vshl_u32(a_.neon_u32, vmov_n_s32(-((int32_t) count))); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i32 = psrlw_s(a_.mmi_i32, b_.mmi_i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] >> count; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psrldi(a, count) simde_mm_srli_pi32(a, count) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_srli_pi32(a, count) simde_mm_srli_pi32(a, count) +# define _m_psrldi(a, count) simde_mm_srli_pi32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_srli_si64 (simde__m64 a, int count) { + #if defined(SIMDE_X86_MMX_NATIVE) && !defined(__PGI) + return _mm_srli_si64(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u64 = vshl_u64(a_.neon_u64, vmov_n_s64(-count)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u64 = a_.u64 >> count; + #else + r_.u64[0] = a_.u64[0] >> count; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psrlqi(a, count) simde_mm_srli_si64(a, count) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_srli_si64(a, count) simde_mm_srli_si64(a, count) +# define _m_psrlqi(a, count) simde_mm_srli_si64(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_srl_si64 (simde__m64 a, simde__m64 count) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_srl_si64(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private count_ = simde__m64_to_private(count); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vshl_u64(a_.neon_u64, vneg_s64(count_.neon_i64)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = a_.u64 >> count_.u64; + #else + if (HEDLEY_UNLIKELY(count_.u64[0] > 63)) { + simde_memset(&r_, 0, sizeof(r_)); + return simde__m64_from_private(r_); + } + + r_.u64[0] = a_.u64[0] >> count_.u64[0]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psrlq(a, count) simde_mm_srl_si64(a, count) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_srl_si64(a, count) simde_mm_srl_si64(a, count) +# define _m_psrlq(a, count) simde_mm_srl_si64(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_srai_pi16 (simde__m64 a, int count) { + #if defined(SIMDE_X86_MMX_NATIVE) && !defined(__PGI) + return _mm_srai_pi16(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i16 = a_.i16 >> (count & 0xff); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vshl_s16(a_.neon_i16, vmov_n_s16(-HEDLEY_STATIC_CAST(int16_t, count))); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i16 = psrah_s(a_.mmi_i16, count); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] >> (count & 0xff); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psrawi(a, count) simde_mm_srai_pi16(a, count) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_srai_pi16(a, count) simde_mm_srai_pi16(a, count) +# define _m_psrawi(a, count) simde_mm_srai_pi16(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_srai_pi32 (simde__m64 a, int count) { + #if defined(SIMDE_X86_MMX_NATIVE) && !defined(__PGI) + return _mm_srai_pi32(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i32 = a_.i32 >> (count & 0xff); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vshl_s32(a_.neon_i32, vmov_n_s32(-HEDLEY_STATIC_CAST(int32_t, count))); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i32 = psraw_s(a_.mmi_i32, count); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] >> (count & 0xff); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psradi(a, count) simde_mm_srai_pi32(a, count) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_srai_pi32(a, count) simde_mm_srai_pi32(a, count) +# define _m_psradi(a, count) simde_mm_srai_pi32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_sra_pi16 (simde__m64 a, simde__m64 count) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_sra_pi16(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private count_ = simde__m64_to_private(count); + const int cnt = HEDLEY_STATIC_CAST(int, (count_.i64[0] > 15 ? 15 : count_.i64[0])); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i16 = a_.i16 >> cnt; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vshl_s16(a_.neon_i16, vmov_n_s16(-HEDLEY_STATIC_CAST(int16_t, vget_lane_u64(count_.neon_u64, 0)))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] >> cnt; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psraw(a, count) simde_mm_sra_pi16(a, count) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_sra_pi16(a, count) simde_mm_sra_pi16(a, count) +# define _m_psraw(a, count) simde_mm_sra_pi16(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_sra_pi32 (simde__m64 a, simde__m64 count) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_sra_pi32(a, count); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private count_ = simde__m64_to_private(count); + const int32_t cnt = (count_.u64[0] > 31) ? 31 : HEDLEY_STATIC_CAST(int32_t, count_.u64[0]); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i32 = a_.i32 >> cnt; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vshl_s32(a_.neon_i32, vmov_n_s32(-HEDLEY_STATIC_CAST(int32_t, vget_lane_u64(count_.neon_u64, 0)))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] >> cnt; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psrad(a, b) simde_mm_sra_pi32(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_sra_pi32(a, count) simde_mm_sra_pi32(a, count) +# define _m_psrad(a, count) simde_mm_sra_pi32(a, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_sub_pi8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_sub_pi8(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vsub_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i8 = psubb_s(a_.mmi_i8, b_.mmi_i8); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = a_.i8 - b_.i8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] - b_.i8[i]; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psubb(a, b) simde_mm_sub_pi8(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_sub_pi8(a, b) simde_mm_sub_pi8(a, b) +# define _m_psubb(a, b) simde_mm_sub_pi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_sub_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_sub_pi16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vsub_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i16 = psubh_s(a_.mmi_i16, b_.mmi_i16); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = a_.i16 - b_.i16; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] - b_.i16[i]; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psubw(a, b) simde_mm_sub_pi16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_sub_pi16(a, b) simde_mm_sub_pi16(a, b) +# define _m_psubw(a, b) simde_mm_sub_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_sub_pi32 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_sub_pi32(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vsub_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i32 = psubw_s(a_.mmi_i32, b_.mmi_i32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 - b_.i32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] - b_.i32[i]; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psubd(a, b) simde_mm_sub_pi32(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_sub_pi32(a, b) simde_mm_sub_pi32(a, b) +# define _m_psubd(a, b) simde_mm_sub_pi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_subs_pi8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_subs_pi8(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vqsub_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i8 = psubsb(a_.mmi_i8, b_.mmi_i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + if (((b_.i8[i]) > 0 && (a_.i8[i]) < INT8_MIN + (b_.i8[i]))) { + r_.i8[i] = INT8_MIN; + } else if ((b_.i8[i]) < 0 && (a_.i8[i]) > INT8_MAX + (b_.i8[i])) { + r_.i8[i] = INT8_MAX; + } else { + r_.i8[i] = (a_.i8[i]) - (b_.i8[i]); + } + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psubsb(a, b) simde_mm_subs_pi8(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_subs_pi8(a, b) simde_mm_subs_pi8(a, b) +# define _m_psubsb(a, b) simde_mm_subs_pi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_subs_pu8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_subs_pu8(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vqsub_u8(a_.neon_u8, b_.neon_u8); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_u8 = psubusb(a_.mmi_u8, b_.mmi_u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + const int32_t x = a_.u8[i] - b_.u8[i]; + if (x < 0) { + r_.u8[i] = 0; + } else if (x > UINT8_MAX) { + r_.u8[i] = UINT8_MAX; + } else { + r_.u8[i] = HEDLEY_STATIC_CAST(uint8_t, x); + } + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psubusb(a, b) simde_mm_subs_pu8(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_subs_pu8(a, b) simde_mm_subs_pu8(a, b) +# define _m_psubusb(a, b) simde_mm_subs_pu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_subs_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_subs_pi16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vqsub_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i16 = psubsh(a_.mmi_i16, b_.mmi_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + if (((b_.i16[i]) > 0 && (a_.i16[i]) < SHRT_MIN + (b_.i16[i]))) { + r_.i16[i] = SHRT_MIN; + } else if ((b_.i16[i]) < 0 && (a_.i16[i]) > INT16_MAX + (b_.i16[i])) { + r_.i16[i] = INT16_MAX; + } else { + r_.i16[i] = (a_.i16[i]) - (b_.i16[i]); + } + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psubsw(a, b) simde_mm_subs_pi16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_subs_pi16(a, b) simde_mm_subs_pi16(a, b) +# define _m_psubsw(a, b) simde_mm_subs_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_subs_pu16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_subs_pu16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vqsub_u16(a_.neon_u16, b_.neon_u16); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_u16 = psubush(a_.mmi_u16, b_.mmi_u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + const int x = a_.u16[i] - b_.u16[i]; + if (x < 0) { + r_.u16[i] = 0; + } else if (x > UINT16_MAX) { + r_.u16[i] = UINT16_MAX; + } else { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, x); + } + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psubusw(a, b) simde_mm_subs_pu16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_subs_pu16(a, b) simde_mm_subs_pu16(a, b) +# define _m_psubusw(a, b) simde_mm_subs_pu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_unpackhi_pi8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_unpackhi_pi8(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i8 = vzip2_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i8 = punpckhbh_s(a_.mmi_i8, b_.mmi_i8); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i8 = SIMDE_SHUFFLE_VECTOR_(8, 8, a_.i8, b_.i8, 4, 12, 5, 13, 6, 14, 7, 15); + #else + r_.i8[0] = a_.i8[4]; + r_.i8[1] = b_.i8[4]; + r_.i8[2] = a_.i8[5]; + r_.i8[3] = b_.i8[5]; + r_.i8[4] = a_.i8[6]; + r_.i8[5] = b_.i8[6]; + r_.i8[6] = a_.i8[7]; + r_.i8[7] = b_.i8[7]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_punpckhbw(a, b) simde_mm_unpackhi_pi8(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_unpackhi_pi8(a, b) simde_mm_unpackhi_pi8(a, b) +# define _m_punpckhbw(a, b) simde_mm_unpackhi_pi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_unpackhi_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_unpackhi_pi16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i16 = vzip2_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i16 = punpckhhw_s(a_.mmi_i16, b_.mmi_i16); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i16 = SIMDE_SHUFFLE_VECTOR_(16, 8, a_.i16, b_.i16, 2, 6, 3, 7); + #else + r_.i16[0] = a_.i16[2]; + r_.i16[1] = b_.i16[2]; + r_.i16[2] = a_.i16[3]; + r_.i16[3] = b_.i16[3]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_punpckhwd(a, b) simde_mm_unpackhi_pi16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_unpackhi_pi16(a, b) simde_mm_unpackhi_pi16(a, b) +# define _m_punpckhwd(a, b) simde_mm_unpackhi_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_unpackhi_pi32 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_unpackhi_pi32(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i32 = vzip2_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i32 = punpckhwd_s(a_.mmi_i32, b_.mmi_i32); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i32 = SIMDE_SHUFFLE_VECTOR_(32, 8, a_.i32, b_.i32, 1, 3); + #else + r_.i32[0] = a_.i32[1]; + r_.i32[1] = b_.i32[1]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_punpckhdq(a, b) simde_mm_unpackhi_pi32(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_unpackhi_pi32(a, b) simde_mm_unpackhi_pi32(a, b) +# define _m_punpckhdq(a, b) simde_mm_unpackhi_pi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_unpacklo_pi8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_unpacklo_pi8(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i8 = vzip1_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i8 = punpcklbh_s(a_.mmi_i8, b_.mmi_i8); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i8 = SIMDE_SHUFFLE_VECTOR_(8, 8, a_.i8, b_.i8, 0, 8, 1, 9, 2, 10, 3, 11); + #else + r_.i8[0] = a_.i8[0]; + r_.i8[1] = b_.i8[0]; + r_.i8[2] = a_.i8[1]; + r_.i8[3] = b_.i8[1]; + r_.i8[4] = a_.i8[2]; + r_.i8[5] = b_.i8[2]; + r_.i8[6] = a_.i8[3]; + r_.i8[7] = b_.i8[3]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_punpcklbw(a, b) simde_mm_unpacklo_pi8(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_unpacklo_pi8(a, b) simde_mm_unpacklo_pi8(a, b) +# define _m_punpcklbw(a, b) simde_mm_unpacklo_pi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_unpacklo_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_unpacklo_pi16(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i16 = vzip1_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i16 = punpcklhw_s(a_.mmi_i16, b_.mmi_i16); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i16 = SIMDE_SHUFFLE_VECTOR_(16, 8, a_.i16, b_.i16, 0, 4, 1, 5); + #else + r_.i16[0] = a_.i16[0]; + r_.i16[1] = b_.i16[0]; + r_.i16[2] = a_.i16[1]; + r_.i16[3] = b_.i16[1]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_punpcklwd(a, b) simde_mm_unpacklo_pi16(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_unpacklo_pi16(a, b) simde_mm_unpacklo_pi16(a, b) +# define _m_punpcklwd(a, b) simde_mm_unpacklo_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_unpacklo_pi32 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_unpacklo_pi32(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i32 = vzip1_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) + r_.mmi_i32 = punpcklwd_s(a_.mmi_i32, b_.mmi_i32); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i32 = SIMDE_SHUFFLE_VECTOR_(32, 8, a_.i32, b_.i32, 0, 2); + #else + r_.i32[0] = a_.i32[0]; + r_.i32[1] = b_.i32[0]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_punpckldq(a, b) simde_mm_unpacklo_pi32(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_unpacklo_pi32(a, b) simde_mm_unpacklo_pi32(a, b) +# define _m_punpckldq(a, b) simde_mm_unpacklo_pi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_xor_si64 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _mm_xor_si64(a, b); + #else + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = veor_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f ^ b_.i32f; + #else + r_.u64[0] = a_.u64[0] ^ b_.u64[0]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pxor(a, b) simde_mm_xor_si64(a, b) +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _mm_xor_si64(a, b) simde_mm_xor_si64(a, b) +# define _m_pxor(a, b) simde_mm_xor_si64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_m_to_int (simde__m64 a) { + #if defined(SIMDE_X86_MMX_NATIVE) + return _m_to_int(a); + #else + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + HEDLEY_DIAGNOSTIC_PUSH + #if HEDLEY_HAS_WARNING("-Wvector-conversion") && SIMDE_DETECT_CLANG_VERSION_NOT(10,0,0) + #pragma clang diagnostic ignored "-Wvector-conversion" + #endif + return vget_lane_s32(a_.neon_i32, 0); + HEDLEY_DIAGNOSTIC_POP + #else + return a_.i32[0]; + #endif + #endif +} +#if defined(SIMDE_X86_MMX_ENABLE_NATIVE_ALIASES) +# define _m_to_int(a) simde_m_to_int(a) +#endif + +SIMDE_END_DECLS_ + +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_MMX_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse.h new file mode 100644 index 00000000..a8855f54 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse.h @@ -0,0 +1,4830 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2017-2020 Evan Nemerson + * 2015-2017 John W. Ratcliff + * 2015 Brandon Rowlett + * 2015 Ken Fast + */ + +#if !defined(SIMDE_X86_SSE_H) +#define SIMDE_X86_SSE_H + +#include "mmx.h" +#include "../simde-f16.h" + +#if defined(_WIN32) && !defined(SIMDE_X86_SSE_NATIVE) && defined(_MSC_VER) + #define NOMINMAX + #include +#endif + +#if defined(__ARM_ACLE) + #include +#endif + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +typedef union { + #if defined(SIMDE_VECTOR_SUBSCRIPT) + SIMDE_ALIGN_TO_16 int8_t i8 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int16_t i16 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int32_t i32 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int64_t i64 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint8_t u8 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint16_t u16 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint32_t u32 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint64_t u64 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_ALIGN_TO_16 simde_int128 i128 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 simde_uint128 u128 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + #endif + #if defined(SIMDE_FLOAT16_VECTOR) + SIMDE_ALIGN_TO_16 simde_float16 f16 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + #else + SIMDE_ALIGN_TO_16 simde_float16 f16[8]; + #endif + SIMDE_ALIGN_TO_16 simde_float32 f32 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int_fast32_t i32f SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint_fast32_t u32f SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + #else + SIMDE_ALIGN_TO_16 int8_t i8[16]; + SIMDE_ALIGN_TO_16 int16_t i16[8]; + SIMDE_ALIGN_TO_16 int32_t i32[4]; + SIMDE_ALIGN_TO_16 int64_t i64[2]; + SIMDE_ALIGN_TO_16 uint8_t u8[16]; + SIMDE_ALIGN_TO_16 uint16_t u16[8]; + SIMDE_ALIGN_TO_16 uint32_t u32[4]; + SIMDE_ALIGN_TO_16 uint64_t u64[2]; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_ALIGN_TO_16 simde_int128 i128[1]; + SIMDE_ALIGN_TO_16 simde_uint128 u128[1]; + #endif + SIMDE_ALIGN_TO_16 simde_float16 f16[8]; + SIMDE_ALIGN_TO_16 simde_float32 f32[4]; + SIMDE_ALIGN_TO_16 int_fast32_t i32f[16 / sizeof(int_fast32_t)]; + SIMDE_ALIGN_TO_16 uint_fast32_t u32f[16 / sizeof(uint_fast32_t)]; + #endif + + SIMDE_ALIGN_TO_16 simde__m64_private m64_private[2]; + SIMDE_ALIGN_TO_16 simde__m64 m64[2]; + + #if defined(SIMDE_X86_SSE_NATIVE) + SIMDE_ALIGN_TO_16 __m128 n; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_ALIGN_TO_16 int8x16_t neon_i8; + SIMDE_ALIGN_TO_16 int16x8_t neon_i16; + SIMDE_ALIGN_TO_16 int32x4_t neon_i32; + SIMDE_ALIGN_TO_16 int64x2_t neon_i64; + SIMDE_ALIGN_TO_16 uint8x16_t neon_u8; + SIMDE_ALIGN_TO_16 uint16x8_t neon_u16; + SIMDE_ALIGN_TO_16 uint32x4_t neon_u32; + SIMDE_ALIGN_TO_16 uint64x2_t neon_u64; + SIMDE_ALIGN_TO_16 float32x4_t neon_f32; + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + SIMDE_ALIGN_TO_16 float64x2_t neon_f64; + #endif + #elif defined(SIMDE_MIPS_MSA_NATIVE) + v16i8 msa_i8; + v8i16 msa_i16; + v4i32 msa_i32; + v2i64 msa_i64; + v16u8 msa_u8; + v8u16 msa_u16; + v4u32 msa_u32; + v2u64 msa_u64; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + SIMDE_ALIGN_TO_16 v128_t wasm_v128; + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) altivec_u8; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) altivec_u16; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed char) altivec_i8; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed short) altivec_i16; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed int) altivec_i32; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(float) altivec_f32; + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long) altivec_u64; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed long long) altivec_i64; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(double) altivec_f64; + #endif + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + v16i8 lsx_i8; + v8i16 lsx_i16; + v4i32 lsx_i32; + v2i64 lsx_i64; + v16u8 lsx_u8; + v8u16 lsx_u16; + v4u32 lsx_u32; + v2u64 lsx_u64; + v4f32 lsx_f32; + v2f64 lsx_f64; + #endif +} simde__m128_private; + +#if defined(SIMDE_X86_SSE_NATIVE) + typedef __m128 simde__m128; +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + typedef float32x4_t simde__m128; +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + typedef v128_t simde__m128; +#elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + typedef SIMDE_POWER_ALTIVEC_VECTOR(float) simde__m128; +#elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + typedef v4f32 simde__m128; +#elif defined(SIMDE_VECTOR_SUBSCRIPT) + typedef simde_float32 simde__m128 SIMDE_ALIGN_TO_16 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; +#else + typedef simde__m128_private simde__m128; +#endif + +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + typedef simde__m128 __m128; +#endif + +HEDLEY_STATIC_ASSERT(16 == sizeof(simde__m128), "simde__m128 size incorrect"); +HEDLEY_STATIC_ASSERT(16 == sizeof(simde__m128_private), "simde__m128_private size incorrect"); +#if defined(SIMDE_CHECK_ALIGNMENT) && defined(SIMDE_ALIGN_OF) +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m128) == 16, "simde__m128 is not 16-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m128_private) == 16, "simde__m128_private is not 16-byte aligned"); +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde__m128_from_private(simde__m128_private v) { + simde__m128 r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128_private +simde__m128_to_private(simde__m128 v) { + simde__m128_private r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +#if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, int8x16_t, neon, i8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, int16x8_t, neon, i16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, int32x4_t, neon, i32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, int64x2_t, neon, i64) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, uint8x16_t, neon, u8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, uint16x8_t, neon, u16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, uint32x4_t, neon, u32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, uint64x2_t, neon, u64) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, float32x4_t, neon, f32) + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, float64x2_t, neon, f64) + #endif +#endif /* defined(SIMDE_ARM_NEON_A32V7_NATIVE) */ + +#if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, SIMDE_POWER_ALTIVEC_VECTOR(signed char), altivec, i8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, SIMDE_POWER_ALTIVEC_VECTOR(signed short), altivec, i16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, SIMDE_POWER_ALTIVEC_VECTOR(signed int), altivec, i32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), altivec, u8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, SIMDE_POWER_ALTIVEC_VECTOR(unsigned short), altivec, u16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, SIMDE_POWER_ALTIVEC_VECTOR(unsigned int), altivec, u32) + + #if defined(SIMDE_BUG_GCC_95782) + SIMDE_FUNCTION_ATTRIBUTES + SIMDE_POWER_ALTIVEC_VECTOR(float) + simde__m128_to_altivec_f32(simde__m128 value) { + simde__m128_private r_ = simde__m128_to_private(value); + return r_.altivec_f32; + } + + SIMDE_FUNCTION_ATTRIBUTES + simde__m128 + simde__m128_from_altivec_f32(SIMDE_POWER_ALTIVEC_VECTOR(float) value) { + simde__m128_private r_; + r_.altivec_f32 = value; + return simde__m128_from_private(r_); + } + #else + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, SIMDE_POWER_ALTIVEC_VECTOR(float), altivec, f32) + #endif + + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, SIMDE_POWER_ALTIVEC_VECTOR(signed long long), altivec, i64) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long), altivec, u64) + #endif +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, v128_t, wasm, v128); +#endif /* defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) */ + +#if defined(SIMDE_LOONGARCH_LSX_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, v16i8, lsx, i8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, v8i16, lsx, i16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, v4i32, lsx, i32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, v2i64, lsx, i64) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, v16u8, lsx, u8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, v8u16, lsx, u16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, v4u32, lsx, u32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, v2u64, lsx, u64) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, v4f32, lsx, f32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128, v2f64, lsx, f64) +#endif /* defined(SIMDE_LOONGARCH_LSX_NATIVE) */ + +enum { + #if defined(SIMDE_X86_SSE_NATIVE) + SIMDE_MM_ROUND_NEAREST = _MM_ROUND_NEAREST, + SIMDE_MM_ROUND_DOWN = _MM_ROUND_DOWN, + SIMDE_MM_ROUND_UP = _MM_ROUND_UP, + SIMDE_MM_ROUND_TOWARD_ZERO = _MM_ROUND_TOWARD_ZERO + #else + SIMDE_MM_ROUND_NEAREST = 0x0000, + SIMDE_MM_ROUND_DOWN = 0x2000, + SIMDE_MM_ROUND_UP = 0x4000, + SIMDE_MM_ROUND_TOWARD_ZERO = 0x6000 + #endif +}; +#if defined(_MM_ROUND_MASK) +# define SIMDE_MM_ROUND_MASK _MM_ROUND_MASK +#else +# define SIMDE_MM_ROUND_MASK (0x6000) +#endif +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + #define _MM_ROUND_MASK SIMDE_MM_ROUND_MASK +#endif + +#if defined(_MM_FROUND_TO_NEAREST_INT) +# define SIMDE_MM_FROUND_TO_NEAREST_INT _MM_FROUND_TO_NEAREST_INT +# define SIMDE_MM_FROUND_TO_NEG_INF _MM_FROUND_TO_NEG_INF +# define SIMDE_MM_FROUND_TO_POS_INF _MM_FROUND_TO_POS_INF +# define SIMDE_MM_FROUND_TO_ZERO _MM_FROUND_TO_ZERO +# define SIMDE_MM_FROUND_CUR_DIRECTION _MM_FROUND_CUR_DIRECTION + +# define SIMDE_MM_FROUND_RAISE_EXC _MM_FROUND_RAISE_EXC +# define SIMDE_MM_FROUND_NO_EXC _MM_FROUND_NO_EXC +#else +# define SIMDE_MM_FROUND_TO_NEAREST_INT 0x00 +# define SIMDE_MM_FROUND_TO_NEG_INF 0x01 +# define SIMDE_MM_FROUND_TO_POS_INF 0x02 +# define SIMDE_MM_FROUND_TO_ZERO 0x03 +# define SIMDE_MM_FROUND_CUR_DIRECTION 0x04 + +# define SIMDE_MM_FROUND_RAISE_EXC 0x00 +# define SIMDE_MM_FROUND_NO_EXC 0x08 +#endif + +#define SIMDE_MM_FROUND_NINT \ + (SIMDE_MM_FROUND_TO_NEAREST_INT | SIMDE_MM_FROUND_RAISE_EXC) +#define SIMDE_MM_FROUND_FLOOR \ + (SIMDE_MM_FROUND_TO_NEG_INF | SIMDE_MM_FROUND_RAISE_EXC) +#define SIMDE_MM_FROUND_CEIL \ + (SIMDE_MM_FROUND_TO_POS_INF | SIMDE_MM_FROUND_RAISE_EXC) +#define SIMDE_MM_FROUND_TRUNC \ + (SIMDE_MM_FROUND_TO_ZERO | SIMDE_MM_FROUND_RAISE_EXC) +#define SIMDE_MM_FROUND_RINT \ + (SIMDE_MM_FROUND_CUR_DIRECTION | SIMDE_MM_FROUND_RAISE_EXC) +#define SIMDE_MM_FROUND_NEARBYINT \ + (SIMDE_MM_FROUND_CUR_DIRECTION | SIMDE_MM_FROUND_NO_EXC) + +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) && !defined(_MM_FROUND_TO_NEAREST_INT) +# define _MM_FROUND_TO_NEAREST_INT SIMDE_MM_FROUND_TO_NEAREST_INT +# define _MM_FROUND_TO_NEG_INF SIMDE_MM_FROUND_TO_NEG_INF +# define _MM_FROUND_TO_POS_INF SIMDE_MM_FROUND_TO_POS_INF +# define _MM_FROUND_TO_ZERO SIMDE_MM_FROUND_TO_ZERO +# define _MM_FROUND_CUR_DIRECTION SIMDE_MM_FROUND_CUR_DIRECTION +# define _MM_FROUND_RAISE_EXC SIMDE_MM_FROUND_RAISE_EXC +# define _MM_FROUND_NINT SIMDE_MM_FROUND_NINT +# define _MM_FROUND_FLOOR SIMDE_MM_FROUND_FLOOR +# define _MM_FROUND_CEIL SIMDE_MM_FROUND_CEIL +# define _MM_FROUND_TRUNC SIMDE_MM_FROUND_TRUNC +# define _MM_FROUND_RINT SIMDE_MM_FROUND_RINT +# define _MM_FROUND_NEARBYINT SIMDE_MM_FROUND_NEARBYINT +#endif + +#if defined(_MM_EXCEPT_INVALID) +# define SIMDE_MM_EXCEPT_INVALID _MM_EXCEPT_INVALID +#else +# define SIMDE_MM_EXCEPT_INVALID (0x0001) +#endif +#if defined(_MM_EXCEPT_DENORM) +# define SIMDE_MM_EXCEPT_DENORM _MM_EXCEPT_DENORM +#else +# define SIMDE_MM_EXCEPT_DENORM (0x0002) +#endif +#if defined(_MM_EXCEPT_DIV_ZERO) +# define SIMDE_MM_EXCEPT_DIV_ZERO _MM_EXCEPT_DIV_ZERO +#else +# define SIMDE_MM_EXCEPT_DIV_ZERO (0x0004) +#endif +#if defined(_MM_EXCEPT_OVERFLOW) +# define SIMDE_MM_EXCEPT_OVERFLOW _MM_EXCEPT_OVERFLOW +#else +# define SIMDE_MM_EXCEPT_OVERFLOW (0x0008) +#endif +#if defined(_MM_EXCEPT_UNDERFLOW) +# define SIMDE_MM_EXCEPT_UNDERFLOW _MM_EXCEPT_UNDERFLOW +#else +# define SIMDE_MM_EXCEPT_UNDERFLOW (0x0010) +#endif +#if defined(_MM_EXCEPT_INEXACT) +# define SIMDE_MM_EXCEPT_INEXACT _MM_EXCEPT_INEXACT +#else +# define SIMDE_MM_EXCEPT_INEXACT (0x0020) +#endif +#if defined(_MM_EXCEPT_MASK) +# define SIMDE_MM_EXCEPT_MASK _MM_EXCEPT_MASK +#else +# define SIMDE_MM_EXCEPT_MASK \ + (SIMDE_MM_EXCEPT_INVALID | SIMDE_MM_EXCEPT_DENORM | \ + SIMDE_MM_EXCEPT_DIV_ZERO | SIMDE_MM_EXCEPT_OVERFLOW | \ + SIMDE_MM_EXCEPT_UNDERFLOW | SIMDE_MM_EXCEPT_INEXACT) +#endif +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + #define _MM_EXCEPT_INVALID SIMDE_MM_EXCEPT_INVALID + #define _MM_EXCEPT_DENORM SIMDE_MM_EXCEPT_DENORM + #define _MM_EXCEPT_DIV_ZERO SIMDE_MM_EXCEPT_DIV_ZERO + #define _MM_EXCEPT_OVERFLOW SIMDE_MM_EXCEPT_OVERFLOW + #define _MM_EXCEPT_UNDERFLOW SIMDE_MM_EXCEPT_UNDERFLOW + #define _MM_EXCEPT_INEXACT SIMDE_MM_EXCEPT_INEXACT + #define _MM_EXCEPT_MASK SIMDE_MM_EXCEPT_MASK +#endif + +#if defined(_MM_MASK_INVALID) +# define SIMDE_MM_MASK_INVALID _MM_MASK_INVALID +#else +# define SIMDE_MM_MASK_INVALID (0x0080) +#endif +#if defined(_MM_MASK_DENORM) +# define SIMDE_MM_MASK_DENORM _MM_MASK_DENORM +#else +# define SIMDE_MM_MASK_DENORM (0x0100) +#endif +#if defined(_MM_MASK_DIV_ZERO) +# define SIMDE_MM_MASK_DIV_ZERO _MM_MASK_DIV_ZERO +#else +# define SIMDE_MM_MASK_DIV_ZERO (0x0200) +#endif +#if defined(_MM_MASK_OVERFLOW) +# define SIMDE_MM_MASK_OVERFLOW _MM_MASK_OVERFLOW +#else +# define SIMDE_MM_MASK_OVERFLOW (0x0400) +#endif +#if defined(_MM_MASK_UNDERFLOW) +# define SIMDE_MM_MASK_UNDERFLOW _MM_MASK_UNDERFLOW +#else +# define SIMDE_MM_MASK_UNDERFLOW (0x0800) +#endif +#if defined(_MM_MASK_INEXACT) +# define SIMDE_MM_MASK_INEXACT _MM_MASK_INEXACT +#else +# define SIMDE_MM_MASK_INEXACT (0x1000) +#endif +#if defined(_MM_MASK_MASK) +# define SIMDE_MM_MASK_MASK _MM_MASK_MASK +#else +# define SIMDE_MM_MASK_MASK \ + (SIMDE_MM_MASK_INVALID | SIMDE_MM_MASK_DENORM | \ + SIMDE_MM_MASK_DIV_ZERO | SIMDE_MM_MASK_OVERFLOW | \ + SIMDE_MM_MASK_UNDERFLOW | SIMDE_MM_MASK_INEXACT) +#endif +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + #define _MM_MASK_INVALID SIMDE_MM_MASK_INVALID + #define _MM_MASK_DENORM SIMDE_MM_MASK_DENORM + #define _MM_MASK_DIV_ZERO SIMDE_MM_MASK_DIV_ZERO + #define _MM_MASK_OVERFLOW SIMDE_MM_MASK_OVERFLOW + #define _MM_MASK_UNDERFLOW SIMDE_MM_MASK_UNDERFLOW + #define _MM_MASK_INEXACT SIMDE_MM_MASK_INEXACT + #define _MM_MASK_MASK SIMDE_MM_MASK_MASK +#endif + +#if defined(_MM_FLUSH_ZERO_MASK) +# define SIMDE_MM_FLUSH_ZERO_MASK _MM_FLUSH_ZERO_MASK +#else +# define SIMDE_MM_FLUSH_ZERO_MASK (0x8000) +#endif +#if defined(_MM_FLUSH_ZERO_ON) +# define SIMDE_MM_FLUSH_ZERO_ON _MM_FLUSH_ZERO_ON +#else +# define SIMDE_MM_FLUSH_ZERO_ON (0x8000) +#endif +#if defined(_MM_FLUSH_ZERO_OFF) +# define SIMDE_MM_FLUSH_ZERO_OFF _MM_FLUSH_ZERO_OFF +#else +# define SIMDE_MM_FLUSH_ZERO_OFF (0x0000) +#endif +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + #define _MM_FLUSH_ZERO_MASK SIMDE_MM_FLUSH_ZERO_MASK + #define _MM_FLUSH_ZERO_ON SIMDE_MM_FLUSH_ZERO_ON + #define _MM_FLUSH_ZERO_OFF SIMDE_MM_FLUSH_ZERO_OFF +#endif + +SIMDE_FUNCTION_ATTRIBUTES +uint32_t +SIMDE_MM_GET_ROUNDING_MODE(void) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _MM_GET_ROUNDING_MODE(); + #elif defined(SIMDE_HAVE_FENV_H) + unsigned int vfe_mode; + + switch (fegetround()) { + #if defined(FE_TONEAREST) + case FE_TONEAREST: + vfe_mode = SIMDE_MM_ROUND_NEAREST; + break; + #endif + + #if defined(FE_TOWARDZERO) + case FE_TOWARDZERO: + vfe_mode = SIMDE_MM_ROUND_DOWN; + break; + #endif + + #if defined(FE_UPWARD) + case FE_UPWARD: + vfe_mode = SIMDE_MM_ROUND_UP; + break; + #endif + + #if defined(FE_DOWNWARD) + case FE_DOWNWARD: + vfe_mode = SIMDE_MM_ROUND_TOWARD_ZERO; + break; + #endif + + default: + vfe_mode = SIMDE_MM_ROUND_NEAREST; + break; + } + + return vfe_mode; + #else + return SIMDE_MM_ROUND_NEAREST; + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + #define _MM_GET_ROUNDING_MODE() SIMDE_MM_GET_ROUNDING_MODE() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +SIMDE_MM_SET_ROUNDING_MODE(uint32_t a) { + #if defined(SIMDE_X86_SSE_NATIVE) + _MM_SET_ROUNDING_MODE(a); + #elif defined(SIMDE_HAVE_FENV_H) + int fe_mode = FE_TONEAREST; + + switch (a) { + #if defined(FE_TONEAREST) + case SIMDE_MM_ROUND_NEAREST: + fe_mode = FE_TONEAREST; + break; + #endif + + #if defined(FE_TOWARDZERO) + case SIMDE_MM_ROUND_TOWARD_ZERO: + fe_mode = FE_TOWARDZERO; + break; + #endif + + #if defined(FE_DOWNWARD) + case SIMDE_MM_ROUND_DOWN: + fe_mode = FE_DOWNWARD; + break; + #endif + + #if defined(FE_UPWARD) + case SIMDE_MM_ROUND_UP: + fe_mode = FE_UPWARD; + break; + #endif + + default: + return; + } + + fesetround(fe_mode); + #else + (void) a; + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + #define _MM_SET_ROUNDING_MODE(a) SIMDE_MM_SET_ROUNDING_MODE(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +uint32_t +SIMDE_MM_GET_FLUSH_ZERO_MODE (void) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_getcsr() & _MM_FLUSH_ZERO_MASK; + #else + return SIMDE_MM_FLUSH_ZERO_OFF; + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + #define _MM_GET_FLUSH_ZERO_MODE(a) SIMDE_MM_GET_FLUSH_ZERO_MODE(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +SIMDE_MM_SET_FLUSH_ZERO_MODE (uint32_t a) { + #if defined(SIMDE_X86_SSE_NATIVE) + _MM_SET_FLUSH_ZERO_MODE(a); + #else + (void) a; + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + #define _MM_SET_FLUSH_ZERO_MODE(a) SIMDE_MM_SET_FLUSH_ZERO_MODE(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +uint32_t +simde_mm_getcsr (void) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_getcsr(); + #else + return SIMDE_MM_GET_ROUNDING_MODE(); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + #define _mm_getcsr() simde_mm_getcsr() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_setcsr (uint32_t a) { + #if defined(SIMDE_X86_SSE_NATIVE) + _mm_setcsr(a); + #else + SIMDE_MM_SET_ROUNDING_MODE(HEDLEY_STATIC_CAST(uint32_t, a & SIMDE_MM_ROUND_MASK)); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + #define _mm_setcsr(a) simde_mm_setcsr(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_x_mm_round_ps (simde__m128 a, int rounding, int lax_rounding) + SIMDE_REQUIRE_CONSTANT_RANGE(rounding, 0, 15) + SIMDE_REQUIRE_CONSTANT_RANGE(lax_rounding, 0, 1) { + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + (void) lax_rounding; + + /* For architectures which lack a current direction SIMD instruction. + * + * Note that NEON actually has a current rounding mode instruction, + * but in ARMv8+ the rounding mode is ignored and nearest is always + * used, so we treat ARMv7 as having a rounding mode but ARMv8 as + * not. */ + #if \ + defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || \ + defined(SIMDE_ARM_NEON_A32V8) + if ((rounding & 7) == SIMDE_MM_FROUND_CUR_DIRECTION) + rounding = HEDLEY_STATIC_CAST(int, SIMDE_MM_GET_ROUNDING_MODE()) << 13; + #endif + + switch (rounding & ~SIMDE_MM_FROUND_NO_EXC) { + case SIMDE_MM_FROUND_CUR_DIRECTION: + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_round(a_.altivec_f32)); + #elif defined(SIMDE_ARM_NEON_A32V8_NATIVE) && !defined(SIMDE_BUG_GCC_95399) + r_.neon_f32 = vrndiq_f32(a_.neon_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_nearest(a_.wasm_v128); + #elif defined(simde_math_nearbyintf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_nearbyintf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm_undefined_pd()); + #endif + break; + + case SIMDE_MM_FROUND_TO_NEAREST_INT: + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_rint(a_.altivec_f32)); + #elif defined(SIMDE_ARM_NEON_A32V8_NATIVE) + r_.neon_f32 = vrndnq_f32(a_.neon_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vfrintrne_s(a_.lsx_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_nearest(a_.wasm_v128); + #elif defined(simde_math_roundevenf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_roundevenf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm_undefined_pd()); + #endif + break; + + case SIMDE_MM_FROUND_TO_NEG_INF: + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_floor(a_.altivec_f32)); + #elif defined(SIMDE_ARM_NEON_A32V8_NATIVE) + r_.neon_f32 = vrndmq_f32(a_.neon_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vfrintrm_s(a_.lsx_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_floor(a_.wasm_v128); + #elif defined(simde_math_floorf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_floorf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm_undefined_pd()); + #endif + break; + + case SIMDE_MM_FROUND_TO_POS_INF: + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_ceil(a_.altivec_f32)); + #elif defined(SIMDE_ARM_NEON_A32V8_NATIVE) + r_.neon_f32 = vrndpq_f32(a_.neon_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vfrintrp_s(a_.lsx_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_ceil(a_.wasm_v128); + #elif defined(simde_math_ceilf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_ceilf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm_undefined_pd()); + #endif + break; + + case SIMDE_MM_FROUND_TO_ZERO: + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_trunc(a_.altivec_f32)); + #elif defined(SIMDE_ARM_NEON_A32V8_NATIVE) + r_.neon_f32 = vrndq_f32(a_.neon_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vfrintrz_s(a_.lsx_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_trunc(a_.wasm_v128); + #elif defined(simde_math_truncf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_truncf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm_undefined_pd()); + #endif + break; + + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm_undefined_pd()); + } + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) + #define simde_mm_round_ps(a, rounding) _mm_round_ps((a), (rounding)) +#else + #define simde_mm_round_ps(a, rounding) simde_x_mm_round_ps((a), (rounding), 0) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #define _mm_round_ps(a, rounding) simde_mm_round_ps((a), (rounding)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_set_ps (simde_float32 e3, simde_float32 e2, simde_float32 e1, simde_float32 e0) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_set_ps(e3, e2, e1, e0); + #else + simde__m128_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_ALIGN_TO_16 simde_float32 data[4] = { e0, e1, e2, e3 }; + r_.neon_f32 = vld1q_f32(data); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_make(e0, e1, e2, e3); + #else + r_.f32[0] = e0; + r_.f32[1] = e1; + r_.f32[2] = e2; + r_.f32[3] = e3; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_set_ps(e3, e2, e1, e0) simde_mm_set_ps(e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_set_ps1 (simde_float32 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_set_ps1(a); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + return vdupq_n_f32(a); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + (void) a; + return vec_splats(a); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + return (simde__m128)__lsx_vldrepl_w(&a, 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f32x4_splat(a); + #else + return simde_mm_set_ps(a, a, a, a); + #endif +} +#define simde_mm_set1_ps(a) simde_mm_set_ps1(a) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_set_ps1(a) simde_mm_set_ps1(a) +# define _mm_set1_ps(a) simde_mm_set1_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_move_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_move_ss(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vsetq_lane_f32(vgetq_lane_f32(b_.neon_f32, 0), a_.neon_f32, 0); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + static const SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) m = { ~0U, 0U, 0U, 0U }; + r_.altivec_f32 = vec_sel(a_.altivec_f32, b_.altivec_f32, m); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_shuffle(b_.wasm_v128, a_.wasm_v128, 0, 1, 2, 3, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vextrins_w(a_.lsx_i64, b_.lsx_i64, 0); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.f32, b_.f32, 4, 1, 2, 3); + #else + r_.f32[0] = b_.f32[0]; + r_.f32[1] = a_.f32[1]; + r_.f32[2] = a_.f32[2]; + r_.f32[3] = a_.f32[3]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_move_ss(a, b) simde_mm_move_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_x_mm_broadcastlow_ps(simde__m128 a) { + /* This function broadcasts the first element in the input vector to + * all lanes. It is used to avoid generating spurious exceptions in + * *_ss functions since there may be garbage in the upper lanes. */ + + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_shuffle_ps(a, a, 0); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f32 = vdupq_laneq_f32(a_.neon_f32, 0); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_f32 = vec_splat(a_.altivec_f32, 0); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vreplvei_w(a_.lsx_i64, 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_splat(a_.f32[0]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.f32, a_.f32, 0, 0, 0, 0); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[0]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_add_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_add_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vaddq_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_add(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_f32 = vec_add(a_.altivec_f32, b_.altivec_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_f32 = __lsx_vfadd_s(a_.lsx_f32, b_.lsx_f32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = a_.f32 + b_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[i] + b_.f32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_add_ps(a, b) simde_mm_add_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_add_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_add_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_add_ps(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_add_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32_t b0 = vgetq_lane_f32(b_.neon_f32, 0); + float32x4_t value = vsetq_lane_f32(b0, vdupq_n_f32(0), 0); + // the upper values in the result must be the remnants of . + r_.neon_f32 = vaddq_f32(a_.neon_f32, value); + #else + r_.f32[0] = a_.f32[0] + b_.f32[0]; + r_.f32[1] = a_.f32[1]; + r_.f32[2] = a_.f32[2]; + r_.f32[3] = a_.f32[3]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_add_ss(a, b) simde_mm_add_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_and_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_and_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vandq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_and(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vand_v(a_.lsx_i64, b_.lsx_i64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 & b_.i32; + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_f32 = vec_and(a_.altivec_f32, b_.altivec_f32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] & b_.i32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_and_ps(a, b) simde_mm_and_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_andnot_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_andnot_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vbicq_s32(b_.neon_i32, a_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_andnot(b_.wasm_v128, a_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + r_.altivec_f32 = vec_andc(b_.altivec_f32, a_.altivec_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vandn_v(a_.lsx_i64, b_.lsx_i64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = ~a_.i32 & b_.i32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ~(a_.i32[i]) & b_.i32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_andnot_ps(a, b) simde_mm_andnot_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_xor_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_xor_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = veorq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_xor(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i32 = vec_xor(a_.altivec_i32, b_.altivec_i32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vxor_v(a_.lsx_i64, b_.lsx_i64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f ^ b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] ^ b_.u32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_xor_ps(a, b) simde_mm_xor_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_or_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_or_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vorrq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_or(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i32 = vec_or(a_.altivec_i32, b_.altivec_i32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vor_v(a_.lsx_i64, b_.lsx_i64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f | b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] | b_.u32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_or_ps(a, b) simde_mm_or_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_x_mm_not_ps(simde__m128 a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + __m128i ai = _mm_castps_si128(a); + return _mm_castsi128_ps(_mm_ternarylogic_epi32(ai, ai, ai, 0x55)); + #elif defined(SIMDE_X86_SSE2_NATIVE) + /* Note: we use ints instead of floats because we don't want cmpeq + * to return false for (NaN, NaN) */ + __m128i ai = _mm_castps_si128(a); + return _mm_castsi128_ps(_mm_andnot_si128(ai, _mm_cmpeq_epi32(ai, ai))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vmvnq_s32(a_.neon_i32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i32 = vec_nor(a_.altivec_i32, a_.altivec_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_not(a_.wasm_v128); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vnor_v(a_.lsx_i64, a_.lsx_i64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = ~a_.i32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = ~(a_.i32[i]); + } + #endif + + return simde__m128_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_x_mm_select_ps(simde__m128 a, simde__m128 b, simde__m128 mask) { + /* This function is for when you want to blend two elements together + * according to a mask. It is similar to _mm_blendv_ps, except that + * it is undefined whether the blend is based on the highest bit in + * each lane (like blendv) or just bitwise operations. This allows + * us to implement the function efficiently everywhere. + * + * Basically, you promise that all the lanes in mask are either 0 or + * ~0. */ + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_blendv_ps(a, b, mask); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b), + mask_ = simde__m128_to_private(mask); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vbslq_s32(mask_.neon_u32, b_.neon_i32, a_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_bitselect(b_.wasm_v128, a_.wasm_v128, mask_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i32 = vec_sel(a_.altivec_i32, b_.altivec_i32, mask_.altivec_u32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vbitsel_v(a_.lsx_i64, b_.lsx_i64, mask_.lsx_i64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 ^ ((a_.i32 ^ b_.i32) & mask_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] ^ ((a_.i32[i] ^ b_.i32[i]) & mask_.i32[i]); + } + #endif + + return simde__m128_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_avg_pu16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_avg_pu16(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vrhadd_u16(b_.neon_u16, a_.neon_u16); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && defined(SIMDE_CONVERT_VECTOR_) && !defined(SIMDE_BUG_GCC_100761) + uint32_t wa SIMDE_VECTOR(16); + uint32_t wb SIMDE_VECTOR(16); + uint32_t wr SIMDE_VECTOR(16); + SIMDE_CONVERT_VECTOR_(wa, a_.u16); + SIMDE_CONVERT_VECTOR_(wb, b_.u16); + wr = (wa + wb + 1) >> 1; + SIMDE_CONVERT_VECTOR_(r_.u16, wr); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] + b_.u16[i] + 1) >> 1; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pavgw(a, b) simde_mm_avg_pu16(a, b) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_avg_pu16(a, b) simde_mm_avg_pu16(a, b) +# define _m_pavgw(a, b) simde_mm_avg_pu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_avg_pu8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_avg_pu8(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vrhadd_u8(b_.neon_u8, a_.neon_u8); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && defined(SIMDE_CONVERT_VECTOR_) && !defined(SIMDE_BUG_GCC_100761) + uint16_t wa SIMDE_VECTOR(16); + uint16_t wb SIMDE_VECTOR(16); + uint16_t wr SIMDE_VECTOR(16); + SIMDE_CONVERT_VECTOR_(wa, a_.u8); + SIMDE_CONVERT_VECTOR_(wb, b_.u8); + wr = (wa + wb + 1) >> 1; + SIMDE_CONVERT_VECTOR_(r_.u8, wr); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] + b_.u8[i] + 1) >> 1; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pavgb(a, b) simde_mm_avg_pu8(a, b) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_avg_pu8(a, b) simde_mm_avg_pu8(a, b) +# define _m_pavgb(a, b) simde_mm_avg_pu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_x_mm_abs_ps(simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + simde_float32 mask_; + uint32_t u32_ = UINT32_C(0x7FFFFFFF); + simde_memcpy(&mask_, &u32_, sizeof(u32_)); + return _mm_and_ps(_mm_set1_ps(mask_), a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vabsq_f32(a_.neon_f32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + r_.altivec_f32 = vec_abs(a_.altivec_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_abs(a_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_fabsf(a_.f32[i]); + } + #endif + + return simde__m128_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpeq_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cmpeq_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vceqq_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_eq(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_cmpeq(a_.altivec_f32, b_.altivec_f32)); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vfcmp_ceq_s(a_.lsx_f32, b_.lsx_f32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.f32 == b_.f32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = (a_.f32[i] == b_.f32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpeq_ps(a, b) simde_mm_cmpeq_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpeq_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cmpeq_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_cmpeq_ps(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_cmpeq_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + r_.u32[0] = (a_.f32[0] == b_.f32[0]) ? ~UINT32_C(0) : UINT32_C(0); + SIMDE_VECTORIZE + for (size_t i = 1 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = a_.u32[i]; + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpeq_ss(a, b) simde_mm_cmpeq_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpge_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cmpge_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcgeq_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_ge(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_cmpge(a_.altivec_f32, b_.altivec_f32)); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vfcmp_cle_s(b_.lsx_f32, a_.lsx_f32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 >= b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = (a_.f32[i] >= b_.f32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpge_ps(a, b) simde_mm_cmpge_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpge_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) && !defined(__PGI) + return _mm_cmpge_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_cmpge_ps(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_cmpge_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + r_.u32[0] = (a_.f32[0] >= b_.f32[0]) ? ~UINT32_C(0) : UINT32_C(0); + SIMDE_VECTORIZE + for (size_t i = 1 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = a_.u32[i]; + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpge_ss(a, b) simde_mm_cmpge_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpgt_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cmpgt_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcgtq_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_gt(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_cmpgt(a_.altivec_f32, b_.altivec_f32)); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vfcmp_clt_s(b_.lsx_f32, a_.lsx_f32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 > b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = (a_.f32[i] > b_.f32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpgt_ps(a, b) simde_mm_cmpgt_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpgt_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) && !defined(__PGI) + return _mm_cmpgt_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_cmpgt_ps(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_cmpgt_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + r_.u32[0] = (a_.f32[0] > b_.f32[0]) ? ~UINT32_C(0) : UINT32_C(0); + SIMDE_VECTORIZE + for (size_t i = 1 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = a_.u32[i]; + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpgt_ss(a, b) simde_mm_cmpgt_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmple_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cmple_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcleq_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_le(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_cmple(a_.altivec_f32, b_.altivec_f32)); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vfcmp_cle_s(a_.lsx_f32, b_.lsx_f32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 <= b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = (a_.f32[i] <= b_.f32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmple_ps(a, b) simde_mm_cmple_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmple_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cmple_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_cmple_ps(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_cmple_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + r_.u32[0] = (a_.f32[0] <= b_.f32[0]) ? ~UINT32_C(0) : UINT32_C(0); + SIMDE_VECTORIZE + for (size_t i = 1 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = a_.u32[i]; + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmple_ss(a, b) simde_mm_cmple_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmplt_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cmplt_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcltq_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_lt(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_cmplt(a_.altivec_f32, b_.altivec_f32)); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vfcmp_clt_s(a_.lsx_f32, b_.lsx_f32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 < b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = (a_.f32[i] < b_.f32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmplt_ps(a, b) simde_mm_cmplt_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmplt_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cmplt_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_cmplt_ps(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_cmplt_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + r_.u32[0] = (a_.f32[0] < b_.f32[0]) ? ~UINT32_C(0) : UINT32_C(0); + SIMDE_VECTORIZE + for (size_t i = 1 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = a_.u32[i]; + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmplt_ss(a, b) simde_mm_cmplt_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpneq_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cmpneq_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vmvnq_u32(vceqq_f32(a_.neon_f32, b_.neon_f32)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_ne(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_cmpeq(a_.altivec_f32, b_.altivec_f32)); + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), vec_nor(r_.altivec_f32, r_.altivec_f32)); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vfcmp_cune_s(a_.lsx_f32, b_.lsx_f32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.f32 != b_.f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = (a_.f32[i] != b_.f32[i]) ? ~UINT32_C(0) : UINT32_C(0); + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpneq_ps(a, b) simde_mm_cmpneq_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpneq_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cmpneq_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_cmpneq_ps(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_cmpneq_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + r_.u32[0] = (a_.f32[0] != b_.f32[0]) ? ~UINT32_C(0) : UINT32_C(0); + SIMDE_VECTORIZE + for (size_t i = 1 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = a_.u32[i]; + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpneq_ss(a, b) simde_mm_cmpneq_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpnge_ps (simde__m128 a, simde__m128 b) { + return simde_mm_cmplt_ps(a, b); +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpnge_ps(a, b) simde_mm_cmpnge_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpnge_ss (simde__m128 a, simde__m128 b) { + return simde_mm_cmplt_ss(a, b); +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpnge_ss(a, b) simde_mm_cmpnge_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpngt_ps (simde__m128 a, simde__m128 b) { + return simde_mm_cmple_ps(a, b); +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpngt_ps(a, b) simde_mm_cmpngt_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpngt_ss (simde__m128 a, simde__m128 b) { + return simde_mm_cmple_ss(a, b); +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpngt_ss(a, b) simde_mm_cmpngt_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpnle_ps (simde__m128 a, simde__m128 b) { + return simde_mm_cmpgt_ps(a, b); +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpnle_ps(a, b) simde_mm_cmpnle_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpnle_ss (simde__m128 a, simde__m128 b) { + return simde_mm_cmpgt_ss(a, b); +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpnle_ss(a, b) simde_mm_cmpnle_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpnlt_ps (simde__m128 a, simde__m128 b) { + return simde_mm_cmpge_ps(a, b); +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpnlt_ps(a, b) simde_mm_cmpnlt_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpnlt_ss (simde__m128 a, simde__m128 b) { + return simde_mm_cmpge_ss(a, b); +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpnlt_ss(a, b) simde_mm_cmpnlt_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpord_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cmpord_ps(a, b); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_v128_and(wasm_f32x4_eq(a, a), wasm_f32x4_eq(b, b)); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + /* Note: NEON does not have ordered compare builtin + Need to compare a eq a and b eq b to check for NaN + Do AND of results to get final */ + uint32x4_t ceqaa = vceqq_f32(a_.neon_f32, a_.neon_f32); + uint32x4_t ceqbb = vceqq_f32(b_.neon_f32, b_.neon_f32); + r_.neon_u32 = vandq_u32(ceqaa, ceqbb); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_and(wasm_f32x4_eq(a_.wasm_v128, a_.wasm_v128), wasm_f32x4_eq(b_.wasm_v128, b_.wasm_v128)); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), + vec_and(vec_cmpeq(a_.altivec_f32, a_.altivec_f32), vec_cmpeq(b_.altivec_f32, b_.altivec_f32))); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vfcmp_cun_s(a_.lsx_f32, b_.lsx_f32); + r_.lsx_i64 = __lsx_vnor_v(r_.lsx_i64, r_.lsx_i64); + #elif defined(simde_math_isnanf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = (simde_math_isnanf(a_.f32[i]) || simde_math_isnanf(b_.f32[i])) ? UINT32_C(0) : ~UINT32_C(0); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpord_ps(a, b) simde_mm_cmpord_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpunord_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cmpunord_ps(a, b); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_v128_or(wasm_f32x4_ne(a, a), wasm_f32x4_ne(b, b)); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x4_t ceqaa = vceqq_f32(a_.neon_f32, a_.neon_f32); + uint32x4_t ceqbb = vceqq_f32(b_.neon_f32, b_.neon_f32); + r_.neon_u32 = vmvnq_u32(vandq_u32(ceqaa, ceqbb)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_or(wasm_f32x4_ne(a_.wasm_v128, a_.wasm_v128), wasm_f32x4_ne(b_.wasm_v128, b_.wasm_v128)); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), + vec_nand(vec_cmpeq(a_.altivec_f32, a_.altivec_f32), vec_cmpeq(b_.altivec_f32, b_.altivec_f32))); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), + vec_and(vec_cmpeq(a_.altivec_f32, a_.altivec_f32), vec_cmpeq(b_.altivec_f32, b_.altivec_f32))); + r_.altivec_f32 = vec_nor(r_.altivec_f32, r_.altivec_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vfcmp_cun_s(a_.lsx_f32, b_.lsx_f32); + #elif defined(simde_math_isnanf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = (simde_math_isnanf(a_.f32[i]) || simde_math_isnanf(b_.f32[i])) ? ~UINT32_C(0) : UINT32_C(0); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpunord_ps(a, b) simde_mm_cmpunord_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpunord_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) && !defined(__PGI) + return _mm_cmpunord_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_cmpunord_ps(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_cmpunord_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(simde_math_isnanf) + r_.u32[0] = (simde_math_isnanf(a_.f32[0]) || simde_math_isnanf(b_.f32[0])) ? ~UINT32_C(0) : UINT32_C(0); + SIMDE_VECTORIZE + for (size_t i = 1 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i]; + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpunord_ss(a, b) simde_mm_cmpunord_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_comieq_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_comieq_ss(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x4_t a_not_nan = vceqq_f32(a_.neon_f32, a_.neon_f32); + uint32x4_t b_not_nan = vceqq_f32(b_.neon_f32, b_.neon_f32); + uint32x4_t a_or_b_nan = vmvnq_u32(vandq_u32(a_not_nan, b_not_nan)); + uint32x4_t a_eq_b = vceqq_f32(a_.neon_f32, b_.neon_f32); + return !!(vgetq_lane_u32(vorrq_u32(a_or_b_nan, a_eq_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f32x4_extract_lane(a_.wasm_v128, 0) == wasm_f32x4_extract_lane(b_.wasm_v128, 0); + #else + return a_.f32[0] == b_.f32[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_comieq_ss(a, b) simde_mm_comieq_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_comige_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_comige_ss(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x4_t a_not_nan = vceqq_f32(a_.neon_f32, a_.neon_f32); + uint32x4_t b_not_nan = vceqq_f32(b_.neon_f32, b_.neon_f32); + uint32x4_t a_and_b_not_nan = vandq_u32(a_not_nan, b_not_nan); + uint32x4_t a_ge_b = vcgeq_f32(a_.neon_f32, b_.neon_f32); + return !!(vgetq_lane_u32(vandq_u32(a_and_b_not_nan, a_ge_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f32x4_extract_lane(a_.wasm_v128, 0) >= wasm_f32x4_extract_lane(b_.wasm_v128, 0); + #else + return a_.f32[0] >= b_.f32[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_comige_ss(a, b) simde_mm_comige_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_comigt_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_comigt_ss(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x4_t a_not_nan = vceqq_f32(a_.neon_f32, a_.neon_f32); + uint32x4_t b_not_nan = vceqq_f32(b_.neon_f32, b_.neon_f32); + uint32x4_t a_and_b_not_nan = vandq_u32(a_not_nan, b_not_nan); + uint32x4_t a_gt_b = vcgtq_f32(a_.neon_f32, b_.neon_f32); + return !!(vgetq_lane_u32(vandq_u32(a_and_b_not_nan, a_gt_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f32x4_extract_lane(a_.wasm_v128, 0) > wasm_f32x4_extract_lane(b_.wasm_v128, 0); + #else + return a_.f32[0] > b_.f32[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_comigt_ss(a, b) simde_mm_comigt_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_comile_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_comile_ss(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x4_t a_not_nan = vceqq_f32(a_.neon_f32, a_.neon_f32); + uint32x4_t b_not_nan = vceqq_f32(b_.neon_f32, b_.neon_f32); + uint32x4_t a_or_b_nan = vmvnq_u32(vandq_u32(a_not_nan, b_not_nan)); + uint32x4_t a_le_b = vcleq_f32(a_.neon_f32, b_.neon_f32); + return !!(vgetq_lane_u32(vorrq_u32(a_or_b_nan, a_le_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f32x4_extract_lane(a_.wasm_v128, 0) <= wasm_f32x4_extract_lane(b_.wasm_v128, 0); + #else + return a_.f32[0] <= b_.f32[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_comile_ss(a, b) simde_mm_comile_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_comilt_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_comilt_ss(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x4_t a_not_nan = vceqq_f32(a_.neon_f32, a_.neon_f32); + uint32x4_t b_not_nan = vceqq_f32(b_.neon_f32, b_.neon_f32); + uint32x4_t a_or_b_nan = vmvnq_u32(vandq_u32(a_not_nan, b_not_nan)); + uint32x4_t a_lt_b = vcltq_f32(a_.neon_f32, b_.neon_f32); + return !!(vgetq_lane_u32(vorrq_u32(a_or_b_nan, a_lt_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f32x4_extract_lane(a_.wasm_v128, 0) < wasm_f32x4_extract_lane(b_.wasm_v128, 0); + #else + return a_.f32[0] < b_.f32[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_comilt_ss(a, b) simde_mm_comilt_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_comineq_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_comineq_ss(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x4_t a_not_nan = vceqq_f32(a_.neon_f32, a_.neon_f32); + uint32x4_t b_not_nan = vceqq_f32(b_.neon_f32, b_.neon_f32); + uint32x4_t a_and_b_not_nan = vandq_u32(a_not_nan, b_not_nan); + uint32x4_t a_neq_b = vmvnq_u32(vceqq_f32(a_.neon_f32, b_.neon_f32)); + return !!(vgetq_lane_u32(vandq_u32(a_and_b_not_nan, a_neq_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f32x4_extract_lane(a_.wasm_v128, 0) != wasm_f32x4_extract_lane(b_.wasm_v128, 0); + #else + return a_.f32[0] != b_.f32[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_comineq_ss(a, b) simde_mm_comineq_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_x_mm_copysign_ps(simde__m128 dest, simde__m128 src) { + simde__m128_private + r_, + dest_ = simde__m128_to_private(dest), + src_ = simde__m128_to_private(src); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const uint32x4_t sign_pos = vreinterpretq_u32_f32(vdupq_n_f32(-SIMDE_FLOAT32_C(0.0))); + r_.neon_u32 = vbslq_u32(sign_pos, src_.neon_u32, dest_.neon_u32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + const v128_t sign_pos = wasm_f32x4_splat(-0.0f); + r_.wasm_v128 = wasm_v128_bitselect(src_.wasm_v128, dest_.wasm_v128, sign_pos); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + #if defined(SIMDE_BUG_VEC_CPSGN_REVERSED_ARGS) + r_.altivec_f32 = vec_cpsgn(dest_.altivec_f32, src_.altivec_f32); + #else + r_.altivec_f32 = vec_cpsgn(src_.altivec_f32, dest_.altivec_f32); + #endif + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + const SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) sign_pos = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned int), vec_splats(-0.0f)); + r_.altivec_f32 = vec_sel(dest_.altivec_f32, src_.altivec_f32, sign_pos); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + const v4f32 sign_pos = {-0.0f, -0.0f, -0.0f, -0.0f}; + r_.lsx_i64 = __lsx_vbitsel_v(dest_.lsx_i64, src_.lsx_i64, (v2i64)sign_pos); + #elif defined(SIMDE_IEEE754_STORAGE) + (void) src_; + (void) dest_; + simde__m128 sign_pos = simde_mm_set1_ps(-0.0f); + r_ = simde__m128_to_private(simde_mm_xor_ps(dest, simde_mm_and_ps(simde_mm_xor_ps(dest, src), sign_pos))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_copysignf(dest_.f32[i], src_.f32[i]); + } + #endif + + return simde__m128_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_x_mm_xorsign_ps(simde__m128 dest, simde__m128 src) { + return simde_mm_xor_ps(simde_mm_and_ps(simde_mm_set1_ps(-0.0f), src), dest); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cvt_pi2ps (simde__m128 a, simde__m64 b) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvt_pi2ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vcombine_f32(vcvt_f32_s32(b_.neon_i32), vget_high_f32(a_.neon_f32)); + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.m64_private[0].f32, b_.i32); + r_.m64_private[1] = a_.m64_private[1]; + #else + r_.f32[0] = (simde_float32) b_.i32[0]; + r_.f32[1] = (simde_float32) b_.i32[1]; + r_.i32[2] = a_.i32[2]; + r_.i32[3] = a_.i32[3]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvt_pi2ps(a, b) simde_mm_cvt_pi2ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cvt_ps2pi (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvt_ps2pi(a); + #else + simde__m64_private r_; + simde__m128_private a_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + a_ = simde__m128_to_private(simde_mm_round_ps(a, SIMDE_MM_FROUND_CUR_DIRECTION)); + r_.neon_i32 = vcvt_s32_f32(vget_low_f32(a_.neon_f32)); + #elif defined(SIMDE_CONVERT_VECTOR_) && SIMDE_NATURAL_VECTOR_SIZE_GE(128) && !defined(SIMDE_BUG_GCC_100761) + a_ = simde__m128_to_private(simde_mm_round_ps(a, SIMDE_MM_FROUND_CUR_DIRECTION)); + SIMDE_CONVERT_VECTOR_(r_.i32, a_.m64_private[0].f32); + #else + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = HEDLEY_STATIC_CAST(int32_t, simde_math_nearbyintf(a_.f32[i])); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvt_ps2pi(a) simde_mm_cvt_ps2pi((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cvt_si2ss (simde__m128 a, int32_t b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cvt_si2ss(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vsetq_lane_f32(HEDLEY_STATIC_CAST(float, b), a_.neon_f32, 0); + #else + r_.f32[0] = HEDLEY_STATIC_CAST(simde_float32, b); + r_.i32[1] = a_.i32[1]; + r_.i32[2] = a_.i32[2]; + r_.i32[3] = a_.i32[3]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvt_si2ss(a, b) simde_mm_cvt_si2ss((a), b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm_cvt_ss2si (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cvt_ss2si(a); + #elif defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_FAST_CONVERSION_RANGE) && !defined(SIMDE_BUG_GCC_95399) + return vgetq_lane_s32(vcvtnq_s32_f32(simde__m128_to_neon_f32(a)), 0); + #else + simde__m128_private a_ = simde__m128_to_private(simde_mm_round_ps(a, SIMDE_MM_FROUND_CUR_DIRECTION)); + #if !defined(SIMDE_FAST_CONVERSION_RANGE) + return ((a_.f32[0] > HEDLEY_STATIC_CAST(simde_float32, INT32_MIN)) && + (a_.f32[0] < HEDLEY_STATIC_CAST(simde_float32, INT32_MAX))) ? + SIMDE_CONVERT_FTOI(int32_t, a_.f32[0]) : INT32_MIN; + #else + return SIMDE_CONVERT_FTOI(int32_t, a_.f32[0]); + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvt_ss2si(a) simde_mm_cvt_ss2si((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cvtpi16_ps (simde__m64 a) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvtpi16_ps(a); + #else + simde__m128_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vcvtq_f32_s32(vmovl_s16(a_.neon_i16)); + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.f32, a_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + simde_float32 v = a_.i16[i]; + r_.f32[i] = v; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvtpi16_ps(a) simde_mm_cvtpi16_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cvtpi32_ps (simde__m128 a, simde__m64 b) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvtpi32_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + simde__m64_private b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vcombine_f32(vcvt_f32_s32(b_.neon_i32), vget_high_f32(a_.neon_f32)); + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.m64_private[0].f32, b_.i32); + r_.m64_private[1] = a_.m64_private[1]; + #else + r_.f32[0] = (simde_float32) b_.i32[0]; + r_.f32[1] = (simde_float32) b_.i32[1]; + r_.i32[2] = a_.i32[2]; + r_.i32[3] = a_.i32[3]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvtpi32_ps(a, b) simde_mm_cvtpi32_ps((a), b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cvtpi32x2_ps (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvtpi32x2_ps(a, b); + #else + simde__m128_private r_; + simde__m64_private + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vcvtq_f32_s32(vcombine_s32(a_.neon_i32, b_.neon_i32)); + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.m64_private[0].f32, a_.i32); + SIMDE_CONVERT_VECTOR_(r_.m64_private[1].f32, b_.i32); + #else + r_.f32[0] = (simde_float32) a_.i32[0]; + r_.f32[1] = (simde_float32) a_.i32[1]; + r_.f32[2] = (simde_float32) b_.i32[0]; + r_.f32[3] = (simde_float32) b_.i32[1]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvtpi32x2_ps(a, b) simde_mm_cvtpi32x2_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cvtpi8_ps (simde__m64 a) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvtpi8_ps(a); + #else + simde__m128_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(a_.neon_i8)))); + #else + r_.f32[0] = HEDLEY_STATIC_CAST(simde_float32, a_.i8[0]); + r_.f32[1] = HEDLEY_STATIC_CAST(simde_float32, a_.i8[1]); + r_.f32[2] = HEDLEY_STATIC_CAST(simde_float32, a_.i8[2]); + r_.f32[3] = HEDLEY_STATIC_CAST(simde_float32, a_.i8[3]); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvtpi8_ps(a) simde_mm_cvtpi8_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cvtps_pi16 (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvtps_pi16(a); + #else + simde__m64_private r_; + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && !defined(SIMDE_BUG_GCC_95399) + r_.neon_i16 = vmovn_s32(vcvtq_s32_f32(vrndiq_f32(a_.neon_f32))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = SIMDE_CONVERT_FTOI(int16_t, simde_math_roundf(a_.f32[i])); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvtps_pi16(a) simde_mm_cvtps_pi16((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cvtps_pi32 (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvtps_pi32(a); + #else + simde__m64_private r_; + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_FAST_CONVERSION_RANGE) && !defined(SIMDE_BUG_GCC_95399) + r_.neon_i32 = vcvt_s32_f32(vget_low_f32(vrndiq_f32(a_.neon_f32))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + simde_float32 v = simde_math_roundf(a_.f32[i]); + #if !defined(SIMDE_FAST_CONVERSION_RANGE) + r_.i32[i] = ((v > HEDLEY_STATIC_CAST(simde_float32, INT32_MIN)) && (v < HEDLEY_STATIC_CAST(simde_float32, INT32_MAX))) ? + SIMDE_CONVERT_FTOI(int32_t, v) : INT32_MIN; + #else + r_.i32[i] = SIMDE_CONVERT_FTOI(int32_t, v); + #endif + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvtps_pi32(a) simde_mm_cvtps_pi32((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cvtps_pi8 (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvtps_pi8(a); + #else + simde__m64_private r_; + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && !defined(SIMDE_BUG_GCC_95471) + /* Clamp the input to [INT8_MIN, INT8_MAX], round, convert to i32, narrow to + * i16, combine with an all-zero vector of i16 (which will become the upper + * half), narrow to i8. */ + float32x4_t max = vdupq_n_f32(HEDLEY_STATIC_CAST(simde_float32, INT8_MAX)); + float32x4_t min = vdupq_n_f32(HEDLEY_STATIC_CAST(simde_float32, INT8_MIN)); + float32x4_t values = vrndnq_f32(vmaxq_f32(vminq_f32(max, a_.neon_f32), min)); + r_.neon_i8 = vmovn_s16(vcombine_s16(vmovn_s32(vcvtq_s32_f32(values)), vdup_n_s16(0))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.f32) / sizeof(a_.f32[0])) ; i++) { + if (a_.f32[i] > HEDLEY_STATIC_CAST(simde_float32, INT8_MAX)) + r_.i8[i] = INT8_MAX; + else if (a_.f32[i] < HEDLEY_STATIC_CAST(simde_float32, INT8_MIN)) + r_.i8[i] = INT8_MIN; + else + r_.i8[i] = SIMDE_CONVERT_FTOI(int8_t, simde_math_roundf(a_.f32[i])); + } + /* Note: the upper half is undefined */ + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvtps_pi8(a) simde_mm_cvtps_pi8((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cvtpu16_ps (simde__m64 a) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvtpu16_ps(a); + #else + simde__m128_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vcvtq_f32_u32(vmovl_u16(a_.neon_u16)); + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.f32, a_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = (simde_float32) a_.u16[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvtpu16_ps(a) simde_mm_cvtpu16_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cvtpu8_ps (simde__m64 a) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvtpu8_ps(a); + #else + simde__m128_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(a_.neon_u8)))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = HEDLEY_STATIC_CAST(simde_float32, a_.u8[i]); + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvtpu8_ps(a) simde_mm_cvtpu8_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cvtsi32_ss (simde__m128 a, int32_t b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cvtsi32_ss(a, b); + #else + simde__m128_private r_; + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vsetq_lane_f32(HEDLEY_STATIC_CAST(float32_t, b), a_.neon_f32, 0); + #else + r_ = a_; + r_.f32[0] = HEDLEY_STATIC_CAST(simde_float32, b); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvtsi32_ss(a, b) simde_mm_cvtsi32_ss((a), b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cvtsi64_ss (simde__m128 a, int64_t b) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_ARCH_AMD64) + #if !defined(__PGI) + return _mm_cvtsi64_ss(a, b); + #else + return _mm_cvtsi64x_ss(a, b); + #endif + #else + simde__m128_private r_; + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vsetq_lane_f32(HEDLEY_STATIC_CAST(float32_t, b), a_.neon_f32, 0); + #else + r_ = a_; + r_.f32[0] = HEDLEY_STATIC_CAST(simde_float32, b); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) +# define _mm_cvtsi64_ss(a, b) simde_mm_cvtsi64_ss((a), b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde_float32 +simde_mm_cvtss_f32 (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cvtss_f32(a); + #else + simde__m128_private a_ = simde__m128_to_private(a); + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + return vgetq_lane_f32(a_.neon_f32, 0); + #else + return a_.f32[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvtss_f32(a) simde_mm_cvtss_f32((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm_cvtss_si32 (simde__m128 a) { + return simde_mm_cvt_ss2si(a); +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvtss_si32(a) simde_mm_cvtss_si32((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int64_t +simde_mm_cvtss_si64 (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_ARCH_AMD64) + #if !defined(__PGI) + return _mm_cvtss_si64(a); + #else + return _mm_cvtss_si64x(a); + #endif + #else + simde__m128_private a_ = simde__m128_to_private(a); + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + return SIMDE_CONVERT_FTOI(int64_t, simde_math_roundf(vgetq_lane_f32(a_.neon_f32, 0))); + #else + return SIMDE_CONVERT_FTOI(int64_t, simde_math_roundf(a_.f32[0])); + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) +# define _mm_cvtss_si64(a) simde_mm_cvtss_si64((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cvtt_ps2pi (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvtt_ps2pi(a); + #else + simde__m64_private r_; + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) && defined(SIMDE_FAST_CONVERSION_RANGE) + r_.neon_i32 = vcvt_s32_f32(vget_low_f32(a_.neon_f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + simde_float32 v = a_.f32[i]; + #if !defined(SIMDE_FAST_CONVERSION_RANGE) + r_.i32[i] = ((v > HEDLEY_STATIC_CAST(simde_float32, INT32_MIN)) && (v < HEDLEY_STATIC_CAST(simde_float32, INT32_MAX))) ? + SIMDE_CONVERT_FTOI(int32_t, v) : INT32_MIN; + #else + r_.i32[i] = SIMDE_CONVERT_FTOI(int32_t, v); + #endif + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_mm_cvttps_pi32(a) simde_mm_cvtt_ps2pi(a) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvtt_ps2pi(a) simde_mm_cvtt_ps2pi((a)) +# define _mm_cvttps_pi32(a) simde_mm_cvttps_pi32((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm_cvtt_ss2si (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cvtt_ss2si(a); + #else + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) && defined(SIMDE_FAST_CONVERSION_RANGE) + return SIMDE_CONVERT_FTOI(int32_t, vgetq_lane_f32(a_.neon_f32, 0)); + #else + simde_float32 v = a_.f32[0]; + #if !defined(SIMDE_FAST_CONVERSION_RANGE) + return ((v > HEDLEY_STATIC_CAST(simde_float32, INT32_MIN)) && (v < HEDLEY_STATIC_CAST(simde_float32, INT32_MAX))) ? + SIMDE_CONVERT_FTOI(int32_t, v) : INT32_MIN; + #else + return SIMDE_CONVERT_FTOI(int32_t, v); + #endif + #endif + #endif +} +#define simde_mm_cvttss_si32(a) simde_mm_cvtt_ss2si((a)) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cvtt_ss2si(a) simde_mm_cvtt_ss2si((a)) +# define _mm_cvttss_si32(a) simde_mm_cvtt_ss2si((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int64_t +simde_mm_cvttss_si64 (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_ARCH_AMD64) && !defined(_MSC_VER) + #if defined(__PGI) + return _mm_cvttss_si64x(a); + #else + return _mm_cvttss_si64(a); + #endif + #else + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + return SIMDE_CONVERT_FTOI(int64_t, vgetq_lane_f32(a_.neon_f32, 0)); + #else + return SIMDE_CONVERT_FTOI(int64_t, a_.f32[0]); + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) +# define _mm_cvttss_si64(a) simde_mm_cvttss_si64((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cmpord_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_cmpord_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_cmpord_ps(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_cmpord_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(simde_math_isnanf) + r_.u32[0] = (simde_math_isnanf(simde_mm_cvtss_f32(a)) || simde_math_isnanf(simde_mm_cvtss_f32(b))) ? UINT32_C(0) : ~UINT32_C(0); + SIMDE_VECTORIZE + for (size_t i = 1 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.u32[i] = a_.u32[i]; + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_cmpord_ss(a, b) simde_mm_cmpord_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_div_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_div_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f32 = vdivq_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32x4_t recip0 = vrecpeq_f32(b_.neon_f32); + float32x4_t recip1 = vmulq_f32(recip0, vrecpsq_f32(recip0, b_.neon_f32)); + r_.neon_f32 = vmulq_f32(a_.neon_f32, recip1); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_div(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + r_.altivec_f32 = vec_div(a_.altivec_f32, b_.altivec_f32); + #elif defined(SIMDE_LOONGARCH_LASX_NATIVE) + r_.lsx_f32 = __lsx_vfdiv_s(a_.lsx_f32, b_.lsx_f32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = a_.f32 / b_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[i] / b_.f32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_div_ps(a, b) simde_mm_div_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_div_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_div_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_div_ps(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_div_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32_t value = + vgetq_lane_f32(simde__m128_to_private(simde_mm_div_ps(a, b)).neon_f32, 0); + r_.neon_f32 = vsetq_lane_f32(value, a_.neon_f32, 0); + #else + r_.f32[0] = a_.f32[0] / b_.f32[0]; + SIMDE_VECTORIZE + for (size_t i = 1 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_div_ss(a, b) simde_mm_div_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int16_t +simde_mm_extract_pi16 (simde__m64 a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m64_private a_ = simde__m64_to_private(a); + return a_.i16[imm8]; +} +#if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) && !defined(HEDLEY_PGI_VERSION) && !defined(SIMDE_BUG_CLANG_44589) + #define simde_mm_extract_pi16(a, imm8) HEDLEY_STATIC_CAST(int16_t, _mm_extract_pi16(a, imm8)) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #define simde_mm_extract_pi16(a, imm8) vget_lane_s16(simde__m64_to_private(a).neon_i16, imm8) +#endif +#define simde_m_pextrw(a, imm8) simde_mm_extract_pi16(a, imm8) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_extract_pi16(a, imm8) simde_mm_extract_pi16((a), (imm8)) +# define _m_pextrw(a, imm8) simde_mm_extract_pi16((a), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_insert_pi16 (simde__m64 a, int16_t i, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m64_private + a_ = simde__m64_to_private(a); + + a_.i16[imm8] = i; + + return simde__m64_from_private(a_); +} +#if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) && !defined(__PGI) && !defined(SIMDE_BUG_CLANG_44589) + #define simde_mm_insert_pi16(a, i, imm8) _mm_insert_pi16(a, i, imm8) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #define simde_mm_insert_pi16(a, i, imm8) simde__m64_from_neon_i16(vset_lane_s16((i), simde__m64_to_neon_i16(a), (imm8))) +#endif +#define simde_m_pinsrw(a, i, imm8) (simde_mm_insert_pi16(a, i, imm8)) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_insert_pi16(a, i, imm8) simde_mm_insert_pi16(a, i, imm8) +# define _m_pinsrw(a, i, imm8) simde_mm_insert_pi16(a, i, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_load_ps (simde_float32 const mem_addr[HEDLEY_ARRAY_PARAM(4)]) { +#if defined(SIMDE_X86_SSE_NATIVE) + return _mm_load_ps(mem_addr); +#else + simde__m128_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vld1q_f32(mem_addr); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + r_.altivec_f32 = vec_vsx_ld(0, mem_addr); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_f32 = vec_ld(0, mem_addr); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vld(mem_addr, 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_load(mem_addr); + #else + simde_memcpy(&r_, SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m128), sizeof(r_)); + #endif + + return simde__m128_from_private(r_); +#endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_load_ps(mem_addr) simde_mm_load_ps(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_load1_ps (simde_float32 const* mem_addr) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_load_ps1(mem_addr); + #else + simde__m128_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vld1q_dup_f32(mem_addr); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vldrepl_w(mem_addr, 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_load32_splat(mem_addr); + #else + r_ = simde__m128_to_private(simde_mm_set1_ps(*mem_addr)); + #endif + + return simde__m128_from_private(r_); + #endif +} +#define simde_mm_load_ps1(mem_addr) simde_mm_load1_ps(mem_addr) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_load_ps1(mem_addr) simde_mm_load1_ps(mem_addr) +# define _mm_load1_ps(mem_addr) simde_mm_load1_ps(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_load_ss (simde_float32 const* mem_addr) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_load_ss(mem_addr); + #else + simde__m128_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vsetq_lane_f32(*mem_addr, vdupq_n_f32(0), 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_load32_zero(mem_addr); + #else + r_.f32[0] = *mem_addr; + r_.i32[1] = 0; + r_.i32[2] = 0; + r_.i32[3] = 0; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_load_ss(mem_addr) simde_mm_load_ss(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_loadh_pi (simde__m128 a, simde__m64 const* mem_addr) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_loadh_pi(a, HEDLEY_REINTERPRET_CAST(__m64 const*, mem_addr)); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vcombine_f32(vget_low_f32(a_.neon_f32), vld1_f32(HEDLEY_REINTERPRET_CAST(const float32_t*, mem_addr))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_load64_lane(mem_addr, a_.wasm_v128, 1); + #else + simde__m64_private b_ = *HEDLEY_REINTERPRET_CAST(simde__m64_private const*, mem_addr); + r_.f32[0] = a_.f32[0]; + r_.f32[1] = a_.f32[1]; + r_.f32[2] = b_.f32[0]; + r_.f32[3] = b_.f32[1]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + #if HEDLEY_HAS_WARNING("-Wold-style-cast") + #define _mm_loadh_pi(a, mem_addr) simde_mm_loadh_pi((a), HEDLEY_REINTERPRET_CAST(simde__m64 const*, (mem_addr))) + #else + #define _mm_loadh_pi(a, mem_addr) simde_mm_loadh_pi((a), (simde__m64 const*) (mem_addr)) + #endif +#endif + +/* The SSE documentation says that there are no alignment requirements + for mem_addr. Unfortunately they used the __m64 type for the argument + which is supposed to be 8-byte aligned, so some compilers (like clang + with -Wcast-align) will generate a warning if you try to cast, say, + a simde_float32* to a simde__m64* for this function. + + I think the choice of argument type is unfortunate, but I do think we + need to stick to it here. If there is demand I can always add something + like simde_x_mm_loadl_f32(simde__m128, simde_float32 mem_addr[2]) */ +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_loadl_pi (simde__m128 a, simde__m64 const* mem_addr) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_loadl_pi(a, HEDLEY_REINTERPRET_CAST(__m64 const*, mem_addr)); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vcombine_f32(vld1_f32( + HEDLEY_REINTERPRET_CAST(const float32_t*, mem_addr)), vget_high_f32(a_.neon_f32)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_load64_lane(mem_addr, a_.wasm_v128, 0); + #else + simde__m64_private b_; + simde_memcpy(&b_, mem_addr, sizeof(b_)); + r_.i32[0] = b_.i32[0]; + r_.i32[1] = b_.i32[1]; + r_.i32[2] = a_.i32[2]; + r_.i32[3] = a_.i32[3]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + #if HEDLEY_HAS_WARNING("-Wold-style-cast") + #define _mm_loadl_pi(a, mem_addr) simde_mm_loadl_pi((a), HEDLEY_REINTERPRET_CAST(simde__m64 const*, (mem_addr))) + #else + #define _mm_loadl_pi(a, mem_addr) simde_mm_loadl_pi((a), (simde__m64 const*) (mem_addr)) + #endif +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_loadr_ps (simde_float32 const mem_addr[HEDLEY_ARRAY_PARAM(4)]) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_loadr_ps(mem_addr); + #else + simde__m128_private + r_, + v_ = simde__m128_to_private(simde_mm_load_ps(mem_addr)); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vrev64q_f32(v_.neon_f32); + r_.neon_f32 = vextq_f32(r_.neon_f32, r_.neon_f32, 2); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) && defined(__PPC64__) + r_.altivec_f32 = vec_reve(v_.altivec_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vshuf4i_w(v_.lsx_i64, 0x1b); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, v_.f32, v_.f32, 3, 2, 1, 0); + #else + r_.f32[0] = v_.f32[3]; + r_.f32[1] = v_.f32[2]; + r_.f32[2] = v_.f32[1]; + r_.f32[3] = v_.f32[0]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_loadr_ps(mem_addr) simde_mm_loadr_ps(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_loadu_ps (simde_float32 const mem_addr[HEDLEY_ARRAY_PARAM(4)]) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_loadu_ps(mem_addr); + #else + simde__m128_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vld1q_f32(HEDLEY_REINTERPRET_CAST(const float32_t*, mem_addr)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_load(mem_addr); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) && defined(__PPC64__) + r_.altivec_f32 = vec_vsx_ld(0, mem_addr); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vld(mem_addr, 0); + #else + simde_memcpy(&r_, mem_addr, sizeof(r_)); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_loadu_ps(mem_addr) simde_mm_loadu_ps(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_maskmove_si64 (simde__m64 a, simde__m64 mask, int8_t* mem_addr) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + _mm_maskmove_si64(a, mask, HEDLEY_REINTERPRET_CAST(char*, mem_addr)); + #else + simde__m64_private + a_ = simde__m64_to_private(a), + mask_ = simde__m64_to_private(mask); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(a_.i8) / sizeof(a_.i8[0])) ; i++) + if (mask_.i8[i] < 0) + mem_addr[i] = a_.i8[i]; + #endif +} +#define simde_m_maskmovq(a, mask, mem_addr) simde_mm_maskmove_si64(a, mask, mem_addr) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_maskmove_si64(a, mask, mem_addr) simde_mm_maskmove_si64((a), (mask), SIMDE_CHECKED_REINTERPRET_CAST(int8_t*, char*, (mem_addr))) +# define _m_maskmovq(a, mask, mem_addr) simde_mm_maskmove_si64((a), (mask), SIMDE_CHECKED_REINTERPRET_CAST(int8_t*, char*, (mem_addr))) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_max_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_max_pi16(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vmax_s16(a_.neon_i16, b_.neon_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] > b_.i16[i]) ? a_.i16[i] : b_.i16[i]; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pmaxsw(a, b) simde_mm_max_pi16(a, b) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_max_pi16(a, b) simde_mm_max_pi16(a, b) +# define _m_pmaxsw(a, b) simde_mm_max_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_max_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_max_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) && defined(SIMDE_FAST_NANS) + r_.neon_f32 = vmaxq_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vbslq_f32(vcgtq_f32(a_.neon_f32, b_.neon_f32), a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) && defined(SIMDE_FAST_NANS) + r_.wasm_v128 = wasm_f32x4_max(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_bitselect(a_.wasm_v128, b_.wasm_v128, wasm_f32x4_gt(a_.wasm_v128, b_.wasm_v128)); + #elif (defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE)) && defined(SIMDE_FAST_NANS) + r_.altivec_f32 = vec_max(a_.altivec_f32, b_.altivec_f32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + r_.altivec_f32 = vec_sel(b_.altivec_f32, a_.altivec_f32, vec_cmpgt(a_.altivec_f32, b_.altivec_f32)); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) && defined(SIMDE_FAST_NANS) + r_.lsx_f32 = __lsx_vfmax_s(a_.lsx_f32, b_.lsx_f32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = (a_.f32[i] > b_.f32[i]) ? a_.f32[i] : b_.f32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_max_ps(a, b) simde_mm_max_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_max_pu8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_max_pu8(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vmax_u8(a_.neon_u8, b_.neon_u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] > b_.u8[i]) ? a_.u8[i] : b_.u8[i]; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pmaxub(a, b) simde_mm_max_pu8(a, b) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_max_pu8(a, b) simde_mm_max_pu8(a, b) +# define _m_pmaxub(a, b) simde_mm_max_pu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_max_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_max_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_max_ps(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_max_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32_t value = vgetq_lane_f32(maxq_f32(a_.neon_f32, b_.neon_f32), 0); + r_.neon_f32 = vsetq_lane_f32(value, a_.neon_f32, 0); + #else + r_.f32[0] = (a_.f32[0] > b_.f32[0]) ? a_.f32[0] : b_.f32[0]; + r_.f32[1] = a_.f32[1]; + r_.f32[2] = a_.f32[2]; + r_.f32[3] = a_.f32[3]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_max_ss(a, b) simde_mm_max_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_min_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_min_pi16(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vmin_s16(a_.neon_i16, b_.neon_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] < b_.i16[i]) ? a_.i16[i] : b_.i16[i]; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pminsw(a, b) simde_mm_min_pi16(a, b) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_min_pi16(a, b) simde_mm_min_pi16(a, b) +# define _m_pminsw(a, b) simde_mm_min_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_min_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_min_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_FAST_NANS) && defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vminq_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_pmin(b_.wasm_v128, a_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + #if defined(SIMDE_FAST_NANS) + r_.altivec_f32 = vec_min(a_.altivec_f32, b_.altivec_f32); + #else + r_.altivec_f32 = vec_sel(b_.altivec_f32, a_.altivec_f32, vec_cmpgt(b_.altivec_f32, a_.altivec_f32)); + #endif + #elif defined(SIMDE_FAST_NANS) && defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_f32 = __lsx_vfmin_s(a_.lsx_f32, b_.lsx_f32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + uint32_t SIMDE_VECTOR(16) m = HEDLEY_REINTERPRET_CAST(__typeof__(m), a_.f32 < b_.f32); + r_.f32 = + HEDLEY_REINTERPRET_CAST( + __typeof__(r_.f32), + ( (HEDLEY_REINTERPRET_CAST(__typeof__(m), a_.f32) & m) | + (HEDLEY_REINTERPRET_CAST(__typeof__(m), b_.f32) & ~m) + ) + ); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = (a_.f32[i] < b_.f32[i]) ? a_.f32[i] : b_.f32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_min_ps(a, b) simde_mm_min_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_min_pu8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_min_pu8(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vmin_u8(a_.neon_u8, b_.neon_u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] < b_.u8[i]) ? a_.u8[i] : b_.u8[i]; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pminub(a, b) simde_mm_min_pu8(a, b) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_min_pu8(a, b) simde_mm_min_pu8(a, b) +# define _m_pminub(a, b) simde_mm_min_pu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_min_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_min_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_min_ps(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_min_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32_t value = vgetq_lane_f32(vminq_f32(a_.neon_f32, b_.neon_f32), 0); + r_.neon_f32 = vsetq_lane_f32(value, a_.neon_f32, 0); + #else + r_.f32[0] = (a_.f32[0] < b_.f32[0]) ? a_.f32[0] : b_.f32[0]; + r_.f32[1] = a_.f32[1]; + r_.f32[2] = a_.f32[2]; + r_.f32[3] = a_.f32[3]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_min_ss(a, b) simde_mm_min_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_movehl_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_movehl_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vzip2q_u64(b_.neon_u64, a_.neon_u64); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32x2_t a32 = vget_high_f32(a_.neon_f32); + float32x2_t b32 = vget_high_f32(b_.neon_f32); + r_.neon_f32 = vcombine_f32(b32, a32); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), + vec_mergel(b_.altivec_i64, a_.altivec_i64)); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vilvh_d(a_.lsx_i64, b_.lsx_i64); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.f32, b_.f32, 6, 7, 2, 3); + #else + r_.f32[0] = b_.f32[2]; + r_.f32[1] = b_.f32[3]; + r_.f32[2] = a_.f32[2]; + r_.f32[3] = a_.f32[3]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_movehl_ps(a, b) simde_mm_movehl_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_movelh_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_movelh_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32x2_t a10 = vget_low_f32(a_.neon_f32); + float32x2_t b10 = vget_low_f32(b_.neon_f32); + r_.neon_f32 = vcombine_f32(a10, b10); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), + vec_mergeh(a_.altivec_i64, b_.altivec_i64)); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vilvl_d(b_.lsx_i64, a_.lsx_i64); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.f32, b_.f32, 0, 1, 4, 5); + #else + r_.f32[0] = a_.f32[0]; + r_.f32[1] = a_.f32[1]; + r_.f32[2] = b_.f32[0]; + r_.f32[3] = b_.f32[1]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_movelh_ps(a, b) simde_mm_movelh_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_movemask_pi8 (simde__m64 a) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_movemask_pi8(a); + #else + simde__m64_private a_ = simde__m64_to_private(a); + int r = 0; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint8x8_t input = a_.neon_u8; + const int8_t xr[8] = {-7, -6, -5, -4, -3, -2, -1, 0}; + const uint8x8_t mask_and = vdup_n_u8(0x80); + const int8x8_t mask_shift = vld1_s8(xr); + const uint8x8_t mask_result = vshl_u8(vand_u8(input, mask_and), mask_shift); + uint8x8_t lo = mask_result; + r = vaddv_u8(lo); + #else + const size_t nmemb = sizeof(a_.i8) / sizeof(a_.i8[0]); + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < nmemb ; i++) { + r |= (a_.u8[nmemb - 1 - i] >> 7) << (nmemb - 1 - i); + } + #endif + + return r; + #endif +} +#define simde_m_pmovmskb(a) simde_mm_movemask_pi8(a) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_movemask_pi8(a) simde_mm_movemask_pi8(a) +# define _m_pmovmskb(a) simde_mm_movemask_pi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_movemask_ps (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_movemask_ps(a); + #else + int r = 0; + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + static const int32_t shift[4] = {0, 1, 2, 3}; + uint32x4_t tmp = vshrq_n_u32(a_.neon_u32, 31); + return HEDLEY_STATIC_CAST(int32_t, vaddvq_u32(vshlq_u32(tmp, vld1q_s32(shift)))); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + // Shift out everything but the sign bits with a 32-bit unsigned shift right. + uint64x2_t high_bits = vreinterpretq_u64_u32(vshrq_n_u32(a_.neon_u32, 31)); + // Merge the two pairs together with a 64-bit unsigned shift right + add. + uint8x16_t paired = vreinterpretq_u8_u64(vsraq_n_u64(high_bits, high_bits, 31)); + // Extract the result. + return vgetq_lane_u8(paired, 0) | (vgetq_lane_u8(paired, 8) << 2); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) && defined(SIMDE_BUG_CLANG_50932) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) idx = { 96, 64, 32, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }; + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) res = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), vec_bperm(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned __int128), a_.altivec_u64), idx)); + return HEDLEY_STATIC_CAST(int32_t, vec_extract(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed int), res), 2)); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) idx = { 96, 64, 32, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }; + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) res = vec_bperm(a_.altivec_u8, idx); + return HEDLEY_STATIC_CAST(int32_t, vec_extract(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed int), res), 2)); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + v2i64 t64 = __lsx_vmskltz_w(a_.lsx_i64); + r = __lsx_vpickve2gr_wu(t64, 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return HEDLEY_STATIC_CAST(int32_t, wasm_i32x4_bitmask(a_.wasm_v128)); + #else + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < sizeof(a_.u32) / sizeof(a_.u32[0]) ; i++) { + r |= (a_.u32[i] >> ((sizeof(a_.u32[i]) * CHAR_BIT) - 1)) << i; + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_movemask_ps(a) simde_mm_movemask_ps((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mul_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_mul_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vmulq_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_mul(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = a_.f32 * b_.f32; + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + r_.altivec_f32 = vec_mul(a_.altivec_f32, b_.altivec_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_f32 = __lsx_vfmul_s(a_.lsx_f32, b_.lsx_f32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[i] * b_.f32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_mul_ps(a, b) simde_mm_mul_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_mul_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_mul_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_mul_ps(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_mul_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + r_.f32[0] = a_.f32[0] * b_.f32[0]; + r_.f32[1] = a_.f32[1]; + r_.f32[2] = a_.f32[2]; + r_.f32[3] = a_.f32[3]; + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_mul_ss(a, b) simde_mm_mul_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_mulhi_pu16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_mulhi_pu16(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const uint32x4_t t1 = vmull_u16(a_.neon_u16, b_.neon_u16); + const uint32x4_t t2 = vshrq_n_u32(t1, 16); + const uint16x4_t t3 = vmovn_u32(t2); + r_.neon_u16 = t3; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, ((HEDLEY_STATIC_CAST(uint32_t, a_.u16[i]) * HEDLEY_STATIC_CAST(uint32_t, b_.u16[i])) >> UINT32_C(16))); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_pmulhuw(a, b) simde_mm_mulhi_pu16(a, b) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_mulhi_pu16(a, b) simde_mm_mulhi_pu16(a, b) +# define _m_pmulhuw(a, b) simde_mm_mulhi_pu16(a, b) +#endif + +#if defined(SIMDE_X86_SSE_NATIVE) && defined(HEDLEY_GCC_VERSION) + #define SIMDE_MM_HINT_NTA HEDLEY_STATIC_CAST(enum _mm_hint, 0) + #define SIMDE_MM_HINT_T0 HEDLEY_STATIC_CAST(enum _mm_hint, 1) + #define SIMDE_MM_HINT_T1 HEDLEY_STATIC_CAST(enum _mm_hint, 2) + #define SIMDE_MM_HINT_T2 HEDLEY_STATIC_CAST(enum _mm_hint, 3) + #define SIMDE_MM_HINT_ENTA HEDLEY_STATIC_CAST(enum _mm_hint, 4) + #define SIMDE_MM_HINT_ET0 HEDLEY_STATIC_CAST(enum _mm_hint, 5) + #define SIMDE_MM_HINT_ET1 HEDLEY_STATIC_CAST(enum _mm_hint, 6) + #define SIMDE_MM_HINT_ET2 HEDLEY_STATIC_CAST(enum _mm_hint, 7) +#else + #define SIMDE_MM_HINT_NTA 0 + #define SIMDE_MM_HINT_T0 1 + #define SIMDE_MM_HINT_T1 2 + #define SIMDE_MM_HINT_T2 3 + #define SIMDE_MM_HINT_ENTA 4 + #define SIMDE_MM_HINT_ET0 5 + #define SIMDE_MM_HINT_ET1 6 + #define SIMDE_MM_HINT_ET2 7 +#endif + +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + HEDLEY_DIAGNOSTIC_PUSH + #if HEDLEY_HAS_WARNING("-Wreserved-id-macro") + _Pragma("clang diagnostic ignored \"-Wreserved-id-macro\"") + #endif + #undef _MM_HINT_NTA + #define _MM_HINT_NTA SIMDE_MM_HINT_NTA + #undef _MM_HINT_T0 + #define _MM_HINT_T0 SIMDE_MM_HINT_T0 + #undef _MM_HINT_T1 + #define _MM_HINT_T1 SIMDE_MM_HINT_T1 + #undef _MM_HINT_T2 + #define _MM_HINT_T2 SIMDE_MM_HINT_T2 + #undef _MM_HINT_ENTA + #define _MM_HINT_ETNA SIMDE_MM_HINT_ENTA + #undef _MM_HINT_ET0 + #define _MM_HINT_ET0 SIMDE_MM_HINT_ET0 + #undef _MM_HINT_ET1 + #define _MM_HINT_ET1 SIMDE_MM_HINT_ET1 + #undef _MM_HINT_ET1 + #define _MM_HINT_ET2 SIMDE_MM_HINT_ET2 + HEDLEY_DIAGNOSTIC_POP +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_prefetch (const void* p, int i) { + #if \ + HEDLEY_HAS_BUILTIN(__builtin_prefetch) || \ + HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + HEDLEY_INTEL_VERSION_CHECK(13,0,0) + switch(i) { + case SIMDE_MM_HINT_NTA: + __builtin_prefetch(p, 0, 0); + break; + case SIMDE_MM_HINT_T0: + __builtin_prefetch(p, 0, 3); + break; + case SIMDE_MM_HINT_T1: + __builtin_prefetch(p, 0, 2); + break; + case SIMDE_MM_HINT_T2: + __builtin_prefetch(p, 0, 1); + break; + case SIMDE_MM_HINT_ENTA: + __builtin_prefetch(p, 1, 0); + break; + case SIMDE_MM_HINT_ET0: + __builtin_prefetch(p, 1, 3); + break; + case SIMDE_MM_HINT_ET1: + __builtin_prefetch(p, 1, 2); + break; + case SIMDE_MM_HINT_ET2: + __builtin_prefetch(p, 0, 1); + break; + } + #elif defined(__ARM_ACLE) + #if (__ARM_ACLE >= 101) + switch(i) { + case SIMDE_MM_HINT_NTA: + __pldx(0, 0, 1, p); + break; + case SIMDE_MM_HINT_T0: + __pldx(0, 0, 0, p); + break; + case SIMDE_MM_HINT_T1: + __pldx(0, 1, 0, p); + break; + case SIMDE_MM_HINT_T2: + __pldx(0, 2, 0, p); + break; + case SIMDE_MM_HINT_ENTA: + __pldx(1, 0, 1, p); + break; + case SIMDE_MM_HINT_ET0: + __pldx(1, 0, 0, p); + break; + case SIMDE_MM_HINT_ET1: + __pldx(1, 1, 0, p); + break; + case SIMDE_MM_HINT_ET2: + __pldx(1, 2, 0, p); + break; + } + #else + (void) i; + __pld(p) + #endif + #elif HEDLEY_PGI_VERSION_CHECK(10,0,0) + (void) i; + #pragma mem prefetch p + #elif HEDLEY_CRAY_VERSION_CHECK(8,1,0) + switch (i) { + case SIMDE_MM_HINT_NTA: + #pragma _CRI prefetch (nt) p + break; + case SIMDE_MM_HINT_T0: + case SIMDE_MM_HINT_T1: + case SIMDE_MM_HINT_T2: + #pragma _CRI prefetch p + break; + case SIMDE_MM_HINT_ENTA: + #pragma _CRI prefetch (write, nt) p + break; + case SIMDE_MM_HINT_ET0: + case SIMDE_MM_HINT_ET1: + case SIMDE_MM_HINT_ET2: + #pragma _CRI prefetch (write) p + break; + } + #elif HEDLEY_IBM_VERSION_CHECK(11,0,0) + switch(i) { + case SIMDE_MM_HINT_NTA: + __prefetch_by_load(p, 0, 0); + break; + case SIMDE_MM_HINT_T0: + __prefetch_by_load(p, 0, 3); + break; + case SIMDE_MM_HINT_T1: + __prefetch_by_load(p, 0, 2); + break; + case SIMDE_MM_HINT_T2: + __prefetch_by_load(p, 0, 1); + break; + case SIMDE_MM_HINT_ENTA: + __prefetch_by_load(p, 1, 0); + break; + case SIMDE_MM_HINT_ET0: + __prefetch_by_load(p, 1, 3); + break; + case SIMDE_MM_HINT_ET1: + __prefetch_by_load(p, 1, 2); + break; + case SIMDE_MM_HINT_ET2: + __prefetch_by_load(p, 0, 1); + break; + } + #elif HEDLEY_MSVC_VERSION + (void) i; + (void) p; + #endif +} +#if defined(SIMDE_X86_SSE_NATIVE) + #if defined(__clang__) && !SIMDE_DETECT_CLANG_VERSION_CHECK(10,0,0) /* https://reviews.llvm.org/D71718 */ + #define simde_mm_prefetch(p, i) \ + (__extension__({ \ + HEDLEY_DIAGNOSTIC_PUSH \ + HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL \ + _mm_prefetch((p), (i)); \ + HEDLEY_DIAGNOSTIC_POP \ + })) + #else + #define simde_mm_prefetch(p, i) _mm_prefetch(p, i) + #endif +#endif +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) + #define _mm_prefetch(p, i) simde_mm_prefetch(p, i) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_x_mm_negate_ps(simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + return simde_mm_xor_ps(a, _mm_set1_ps(SIMDE_FLOAT32_C(-0.0))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vnegq_f32(a_.neon_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_neg(a_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + r_.altivec_f32 = vec_neg(a_.altivec_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + const v4f32 f32 = {0.0f, 0.0f, 0.0f, 0.0f}; + r_.lsx_f32 = __lsx_vfsub_s(f32, a_.lsx_f32); + #elif defined(SIMDE_VECTOR_NEGATE) + r_.f32 = -a_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = -a_.f32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_rcp_ps (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_rcp_ps(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32x4_t recip = vrecpeq_f32(a_.neon_f32); + + #if SIMDE_ACCURACY_PREFERENCE > 0 + for (int i = 0; i < SIMDE_ACCURACY_PREFERENCE ; ++i) { + recip = vmulq_f32(recip, vrecpsq_f32(recip, a_.neon_f32)); + } + #endif + + r_.neon_f32 = recip; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_div(simde_mm_set1_ps(1.0f), a_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_f32 = vec_re(a_.altivec_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_f32 = __lsx_vfrecip_s(a_.lsx_f32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.f32 = 1.0f / a_.f32; + #elif defined(SIMDE_IEEE754_STORAGE) + /* https://stackoverflow.com/questions/12227126/division-as-multiply-and-lut-fast-float-division-reciprocal/12228234#12228234 */ + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + int32_t ix; + simde_float32 fx = a_.f32[i]; + simde_memcpy(&ix, &fx, sizeof(ix)); + int32_t x = INT32_C(0x7EF311C3) - ix; + simde_float32 temp; + simde_memcpy(&temp, &x, sizeof(temp)); + r_.f32[i] = temp * (SIMDE_FLOAT32_C(2.0) - temp * fx); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = 1.0f / a_.f32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_rcp_ps(a) simde_mm_rcp_ps((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_rcp_ss (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_rcp_ss(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_rcp_ps(a)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_rcp_ps(simde_x_mm_broadcastlow_ps(a))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + r_.f32[0] = 1.0f / a_.f32[0]; + r_.f32[1] = a_.f32[1]; + r_.f32[2] = a_.f32[2]; + r_.f32[3] = a_.f32[3]; + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_rcp_ss(a) simde_mm_rcp_ss((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_rsqrt_ps (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_rsqrt_ps(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vrsqrteq_f32(a_.neon_f32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_f32 = vec_rsqrte(a_.altivec_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_f32 = __lsx_vfrsqrt_s(a_.lsx_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_div(simde_mm_set1_ps(1.0f), wasm_f32x4_sqrt(a_.wasm_v128)); + #elif defined(SIMDE_IEEE754_STORAGE) + /* https://basesandframes.files.wordpress.com/2020/04/even_faster_math_functions_green_2020.pdf + Pages 100 - 103 */ + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + #if SIMDE_ACCURACY_PREFERENCE <= 0 + r_.i32[i] = INT32_C(0x5F37624F) - (a_.i32[i] >> 1); + #else + simde_float32 x = a_.f32[i]; + simde_float32 xhalf = SIMDE_FLOAT32_C(0.5) * x; + int32_t ix; + + simde_memcpy(&ix, &x, sizeof(ix)); + + #if SIMDE_ACCURACY_PREFERENCE == 1 + ix = INT32_C(0x5F375A82) - (ix >> 1); + #else + ix = INT32_C(0x5F37599E) - (ix >> 1); + #endif + + simde_memcpy(&x, &ix, sizeof(x)); + + #if SIMDE_ACCURACY_PREFERENCE >= 2 + x = x * (SIMDE_FLOAT32_C(1.5008909) - xhalf * x * x); + #endif + x = x * (SIMDE_FLOAT32_C(1.5008909) - xhalf * x * x); + + r_.f32[i] = x; + #endif + } + #elif defined(simde_math_sqrtf) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = 1.0f / simde_math_sqrtf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_rsqrt_ps(a) simde_mm_rsqrt_ps((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_rsqrt_ss (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_rsqrt_ss(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_rsqrt_ps(a)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_rsqrt_ps(simde_x_mm_broadcastlow_ps(a))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vsetq_lane_f32(vgetq_lane_f32(simde_mm_rsqrt_ps(a).neon_f32, 0), a_.neon_f32, 0); + #elif defined(SIMDE_IEEE754_STORAGE) + { + #if SIMDE_ACCURACY_PREFERENCE <= 0 + r_.i32[0] = INT32_C(0x5F37624F) - (a_.i32[0] >> 1); + #else + simde_float32 x = a_.f32[0]; + simde_float32 xhalf = SIMDE_FLOAT32_C(0.5) * x; + int32_t ix; + + simde_memcpy(&ix, &x, sizeof(ix)); + + #if SIMDE_ACCURACY_PREFERENCE == 1 + ix = INT32_C(0x5F375A82) - (ix >> 1); + #else + ix = INT32_C(0x5F37599E) - (ix >> 1); + #endif + + simde_memcpy(&x, &ix, sizeof(x)); + + #if SIMDE_ACCURACY_PREFERENCE >= 2 + x = x * (SIMDE_FLOAT32_C(1.5008909) - xhalf * x * x); + #endif + x = x * (SIMDE_FLOAT32_C(1.5008909) - xhalf * x * x); + + r_.f32[0] = x; + #endif + } + r_.f32[1] = a_.f32[1]; + r_.f32[2] = a_.f32[2]; + r_.f32[3] = a_.f32[3]; + #elif defined(simde_math_sqrtf) + r_.f32[0] = 1.0f / simde_math_sqrtf(a_.f32[0]); + r_.f32[1] = a_.f32[1]; + r_.f32[2] = a_.f32[2]; + r_.f32[3] = a_.f32[3]; + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_rsqrt_ss(a) simde_mm_rsqrt_ss((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_sad_pu8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_sad_pu8(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint64x1_t t = vpaddl_u32(vpaddl_u16(vpaddl_u8(vabd_u8(a_.neon_u8, b_.neon_u8)))); + r_.neon_u16 = vset_lane_u16(HEDLEY_STATIC_CAST(uint64_t, vget_lane_u64(t, 0)), vdup_n_u16(0), 0); + #else + uint16_t sum = 0; + + SIMDE_VECTORIZE_REDUCTION(+:sum) + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + sum += HEDLEY_STATIC_CAST(uint8_t, simde_math_abs(a_.u8[i] - b_.u8[i])); + } + + r_.i16[0] = HEDLEY_STATIC_CAST(int16_t, sum); + r_.i16[1] = 0; + r_.i16[2] = 0; + r_.i16[3] = 0; + #endif + + return simde__m64_from_private(r_); + #endif +} +#define simde_m_psadbw(a, b) simde_mm_sad_pu8(a, b) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_sad_pu8(a, b) simde_mm_sad_pu8(a, b) +# define _m_psadbw(a, b) simde_mm_sad_pu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_set_ss (simde_float32 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_set_ss(a); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + return vsetq_lane_f32(a, vdupq_n_f32(SIMDE_FLOAT32_C(0.0)), 0); + #else + return simde_mm_set_ps(SIMDE_FLOAT32_C(0.0), SIMDE_FLOAT32_C(0.0), SIMDE_FLOAT32_C(0.0), a); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_set_ss(a) simde_mm_set_ss(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_setr_ps (simde_float32 e3, simde_float32 e2, simde_float32 e1, simde_float32 e0) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_setr_ps(e3, e2, e1, e0); + #else + return simde_mm_set_ps(e0, e1, e2, e3); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_setr_ps(e3, e2, e1, e0) simde_mm_setr_ps(e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_setzero_ps (void) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_setzero_ps(); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + return vdupq_n_f32(SIMDE_FLOAT32_C(0.0)); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + return vec_splats(SIMDE_FLOAT32_C(0.0)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f32x4_const(0.f, 0.f, 0.f, 0.f); + #else + simde__m128 r; + simde_memset(&r, 0, sizeof(r)); + return r; + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_setzero_ps() simde_mm_setzero_ps() +#endif + +#if defined(SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_) +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_ +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_undefined_ps (void) { + simde__m128_private r_; + + #if defined(SIMDE_HAVE_UNDEFINED128) + r_.n = _mm_undefined_ps(); + #elif !defined(SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_) + r_ = simde__m128_to_private(simde_mm_setzero_ps()); + #endif + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_undefined_ps() simde_mm_undefined_ps() +#endif + +#if defined(SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_) +HEDLEY_DIAGNOSTIC_POP +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_x_mm_setone_ps (void) { + simde__m128 t = simde_mm_setzero_ps(); + return simde_mm_cmpeq_ps(t, t); +} + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_sfence (void) { + /* TODO: Use Hedley. */ + #if defined(SIMDE_X86_SSE_NATIVE) + _mm_sfence(); + #elif defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7)) + __atomic_thread_fence(__ATOMIC_SEQ_CST); + #elif !defined(__INTEL_COMPILER) && defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_ATOMICS__) + #if defined(__GNUC__) && (__GNUC__ == 4) && (__GNUC_MINOR__ < 9) + __atomic_thread_fence(__ATOMIC_SEQ_CST); + #else + atomic_thread_fence(memory_order_seq_cst); + #endif + #elif defined(_MSC_VER) + MemoryBarrier(); + #elif HEDLEY_HAS_EXTENSION(c_atomic) + __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); + #elif defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) + __sync_synchronize(); + #elif defined(_OPENMP) + #pragma omp critical(simde_mm_sfence_) + { } + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_sfence() simde_mm_sfence() +#endif + +#define SIMDE_MM_SHUFFLE(z, y, x, w) (((z) << 6) | ((y) << 4) | ((x) << 2) | (w)) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _MM_SHUFFLE(z, y, x, w) SIMDE_MM_SHUFFLE(z, y, x, w) +#endif + +#if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) && !defined(__PGI) +# define simde_mm_shuffle_pi16(a, imm8) _mm_shuffle_pi16(a, imm8) +#elif defined(SIMDE_SHUFFLE_VECTOR_) +# define simde_mm_shuffle_pi16(a, imm8) (__extension__ ({ \ + const simde__m64_private simde_tmp_a_ = simde__m64_to_private(a); \ + simde__m64_from_private((simde__m64_private) { .i16 = \ + SIMDE_SHUFFLE_VECTOR_(16, 8, \ + (simde_tmp_a_).i16, \ + (simde_tmp_a_).i16, \ + (((imm8) ) & 3), \ + (((imm8) >> 2) & 3), \ + (((imm8) >> 4) & 3), \ + (((imm8) >> 6) & 3)) }); })) +#else +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_shuffle_pi16 (simde__m64 a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m64_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + for (size_t i = 0 ; i < sizeof(r_.i16) / sizeof(r_.i16[0]) ; i++) { + r_.i16[i] = a_.i16[(imm8 >> (i * 2)) & 3]; + } + +HEDLEY_DIAGNOSTIC_PUSH +#if HEDLEY_HAS_WARNING("-Wconditional-uninitialized") +# pragma clang diagnostic ignored "-Wconditional-uninitialized" +#endif + return simde__m64_from_private(r_); +HEDLEY_DIAGNOSTIC_POP +} +#endif +#if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) && !defined(__PGI) +# define simde_m_pshufw(a, imm8) _m_pshufw(a, imm8) +#else +# define simde_m_pshufw(a, imm8) simde_mm_shuffle_pi16(a, imm8) +#endif +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_shuffle_pi16(a, imm8) simde_mm_shuffle_pi16(a, imm8) +# define _m_pshufw(a, imm8) simde_mm_shuffle_pi16(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_shuffle_ps (simde__m128 a, simde__m128 b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + r_.f32[0] = a_.f32[(imm8 >> 0) & 3]; + r_.f32[1] = a_.f32[(imm8 >> 2) & 3]; + r_.f32[2] = b_.f32[(imm8 >> 4) & 3]; + r_.f32[3] = b_.f32[(imm8 >> 6) & 3]; + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_SSE_NATIVE) && !defined(__PGI) +# define simde_mm_shuffle_ps(a, b, imm8) _mm_shuffle_ps(a, b, imm8) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_shuffle_ps(a, b, imm8) (__extension__ ({ \ + simde__m128_from_private((simde__m128_private) { .wasm_v128 = \ + wasm_i32x4_shuffle( \ + simde__m128_to_private(a).wasm_v128, \ + simde__m128_to_private(b).wasm_v128, \ + (((imm8) ) & 3), \ + (((imm8) >> 2) & 3), \ + (((imm8) >> 4) & 3) + 4, \ + (((imm8) >> 6) & 3) + 4) }); })) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm_shuffle_ps(a, b, imm8) \ + (__extension__({ \ + float32x4_t simde_mm_shuffle_ps_a_ = simde__m128_to_neon_f32(a); \ + float32x4_t simde_mm_shuffle_ps_b_ = simde__m128_to_neon_f32(b); \ + float32x4_t simde_mm_shuffle_ps_r_; \ + \ + simde_mm_shuffle_ps_r_ = vmovq_n_f32(vgetq_lane_f32(simde_mm_shuffle_ps_a_, (imm8) & (0x3))); \ + simde_mm_shuffle_ps_r_ = vsetq_lane_f32(vgetq_lane_f32(simde_mm_shuffle_ps_a_, ((imm8) >> 2) & 0x3), simde_mm_shuffle_ps_r_, 1); \ + simde_mm_shuffle_ps_r_ = vsetq_lane_f32(vgetq_lane_f32(simde_mm_shuffle_ps_b_, ((imm8) >> 4) & 0x3), simde_mm_shuffle_ps_r_, 2); \ + vsetq_lane_f32(vgetq_lane_f32(simde_mm_shuffle_ps_b_, ((imm8) >> 6) & 0x3), simde_mm_shuffle_ps_r_, 3); \ + })) +#elif defined(SIMDE_SHUFFLE_VECTOR_) + #define simde_mm_shuffle_ps(a, b, imm8) (__extension__ ({ \ + simde__m128_from_private((simde__m128_private) { .f32 = \ + SIMDE_SHUFFLE_VECTOR_(32, 16, \ + simde__m128_to_private(a).f32, \ + simde__m128_to_private(b).f32, \ + (((imm8) ) & 3), \ + (((imm8) >> 2) & 3), \ + (((imm8) >> 4) & 3) + 4, \ + (((imm8) >> 6) & 3) + 4) }); })) +#endif +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_shuffle_ps(a, b, imm8) simde_mm_shuffle_ps((a), (b), imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_sqrt_ps (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_sqrt_ps(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f32 = vsqrtq_f32(a_.neon_f32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32x4_t est = vrsqrteq_f32(a_.neon_f32); + for (int i = 0 ; i <= SIMDE_ACCURACY_PREFERENCE ; i++) { + est = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a_.neon_f32, est), est), est); + } + r_.neon_f32 = vmulq_f32(a_.neon_f32, est); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_sqrt(a_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_14_NATIVE) + r_.altivec_f32 = vec_sqrt(a_.altivec_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_f32 = __lsx_vfsqrt_s(a_.lsx_f32); + #elif defined(simde_math_sqrt) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < sizeof(r_.f32) / sizeof(r_.f32[0]) ; i++) { + r_.f32[i] = simde_math_sqrtf(a_.f32[i]); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_sqrt_ps(a) simde_mm_sqrt_ps((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_sqrt_ss (simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_sqrt_ss(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_sqrt_ps(a)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_sqrt_ps(simde_x_mm_broadcastlow_ps(a))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32_t value = + vgetq_lane_f32(simde__m128_to_private(simde_mm_sqrt_ps(a)).neon_f32, 0); + r_.neon_f32 = vsetq_lane_f32(value, a_.neon_f32, 0); + #elif defined(simde_math_sqrtf) + r_.f32[0] = simde_math_sqrtf(a_.f32[0]); + r_.f32[1] = a_.f32[1]; + r_.f32[2] = a_.f32[2]; + r_.f32[3] = a_.f32[3]; + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_sqrt_ss(a) simde_mm_sqrt_ss((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_store_ps (simde_float32 mem_addr[4], simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + _mm_store_ps(mem_addr, a); + #else + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + vst1q_f32(mem_addr, a_.neon_f32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + vec_st(a_.altivec_f32, 0, mem_addr); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + wasm_v128_store(mem_addr, a_.wasm_v128); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + __lsx_vst(a_.lsx_f32, mem_addr, 0); + #else + simde_memcpy(mem_addr, &a_, sizeof(a)); + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_store_ps(mem_addr, a) simde_mm_store_ps(SIMDE_CHECKED_REINTERPRET_CAST(float*, simde_float32*, mem_addr), (a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_store1_ps (simde_float32 mem_addr[4], simde__m128 a) { + simde_float32* mem_addr_ = SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m128); + + #if defined(SIMDE_X86_SSE_NATIVE) + _mm_store_ps1(mem_addr_, a); + #else + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + vst1q_f32(mem_addr_, vdupq_lane_f32(vget_low_f32(a_.neon_f32), 0)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + wasm_v128_store(mem_addr_, wasm_i32x4_shuffle(a_.wasm_v128, a_.wasm_v128, 0, 0, 0, 0)); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + vec_st(vec_splat(a_.altivec_f32, 0), 0, mem_addr_); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + __lsx_vst(__lsx_vreplvei_w(a_.lsx_f32, 0), mem_addr_, 0); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + simde__m128_private tmp_; + tmp_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.f32, a_.f32, 0, 0, 0, 0); + simde_mm_store_ps(mem_addr_, tmp_.f32); + #else + SIMDE_VECTORIZE_ALIGNED(mem_addr_:16) + for (size_t i = 0 ; i < sizeof(a_.f32) / sizeof(a_.f32[0]) ; i++) { + mem_addr_[i] = a_.f32[0]; + } + #endif + #endif +} +#define simde_mm_store_ps1(mem_addr, a) simde_mm_store1_ps(mem_addr, a) +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_store_ps1(mem_addr, a) simde_mm_store1_ps(SIMDE_CHECKED_REINTERPRET_CAST(float*, simde_float32*, mem_addr), (a)) +# define _mm_store1_ps(mem_addr, a) simde_mm_store1_ps(SIMDE_CHECKED_REINTERPRET_CAST(float*, simde_float32*, mem_addr), (a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_store_ss (simde_float32* mem_addr, simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + _mm_store_ss(mem_addr, a); + #else + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + vst1q_lane_f32(mem_addr, a_.neon_f32, 0); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + __lsx_vstelm_w(a_.lsx_f32, mem_addr, 0, 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + wasm_v128_store32_lane(HEDLEY_REINTERPRET_CAST(void*, mem_addr), a_.wasm_v128, 0); + #else + *mem_addr = a_.f32[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_store_ss(mem_addr, a) simde_mm_store_ss(SIMDE_CHECKED_REINTERPRET_CAST(float*, simde_float32*, mem_addr), (a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_storeh_pi (simde__m64* mem_addr, simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + _mm_storeh_pi(HEDLEY_REINTERPRET_CAST(__m64*, mem_addr), a); + #else + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + vst1_f32(HEDLEY_REINTERPRET_CAST(float32_t*, mem_addr), vget_high_f32(a_.neon_f32)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + wasm_v128_store64_lane(HEDLEY_REINTERPRET_CAST(void*, mem_addr), a_.wasm_v128, 1); + #else + simde_memcpy(mem_addr, &(a_.m64[1]), sizeof(a_.m64[1])); + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_storeh_pi(mem_addr, a) simde_mm_storeh_pi(mem_addr, (a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_storel_pi (simde__m64* mem_addr, simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + _mm_storel_pi(HEDLEY_REINTERPRET_CAST(__m64*, mem_addr), a); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + simde__m128_private a_ = simde__m128_to_private(a); + wasm_v128_store64_lane(HEDLEY_REINTERPRET_CAST(void*, mem_addr), a_.wasm_v128, 0); + #else + simde__m64_private* dest_ = HEDLEY_REINTERPRET_CAST(simde__m64_private*, mem_addr); + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + dest_->neon_f32 = vget_low_f32(a_.neon_f32); + #else + dest_->f32[0] = a_.f32[0]; + dest_->f32[1] = a_.f32[1]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_storel_pi(mem_addr, a) simde_mm_storel_pi(mem_addr, (a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_storer_ps (simde_float32 mem_addr[4], simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + _mm_storer_ps(mem_addr, a); + #else + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + vec_st(vec_reve(a_.altivec_f32), 0, mem_addr); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32x4_t tmp = vrev64q_f32(a_.neon_f32); + vst1q_f32(mem_addr, vextq_f32(tmp, tmp, 2)); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + __lsx_vst(__lsx_vshuf4i_w(a_.lsx_f32, 0x1b), mem_addr, 0); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + a_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.f32, a_.f32, 3, 2, 1, 0); + simde_mm_store_ps(mem_addr, simde__m128_from_private(a_)); + #else + SIMDE_VECTORIZE_ALIGNED(mem_addr:16) + for (size_t i = 0 ; i < sizeof(a_.f32) / sizeof(a_.f32[0]) ; i++) { + mem_addr[i] = a_.f32[((sizeof(a_.f32) / sizeof(a_.f32[0])) - 1) - i]; + } + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_storer_ps(mem_addr, a) simde_mm_storer_ps(SIMDE_CHECKED_REINTERPRET_CAST(float*, simde_float32*, mem_addr), (a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_storeu_ps (simde_float32 mem_addr[4], simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + _mm_storeu_ps(mem_addr, a); + #else + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + vst1q_f32(mem_addr, a_.neon_f32); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + vec_vsx_st(a_.altivec_f32, 0, mem_addr); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + __lsx_vst(a_.lsx_f32, mem_addr, 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + wasm_v128_store(mem_addr, a_.wasm_v128); + #else + simde_memcpy(mem_addr, &a_, sizeof(a_)); + #endif + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_storeu_ps(mem_addr, a) simde_mm_storeu_ps(SIMDE_CHECKED_REINTERPRET_CAST(float*, simde_float32*, mem_addr), (a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_sub_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_sub_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vsubq_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_sub(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_f32 = vec_sub(a_.altivec_f32, b_.altivec_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_f32 = __lsx_vfsub_s(a_.lsx_f32, b_.lsx_f32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f32 = a_.f32 - b_.f32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[i] - b_.f32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_sub_ps(a, b) simde_mm_sub_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_sub_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_sub_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_sub_ps(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_sub_ps(simde_x_mm_broadcastlow_ps(a), simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + r_.f32[0] = a_.f32[0] - b_.f32[0]; + r_.f32[1] = a_.f32[1]; + r_.f32[2] = a_.f32[2]; + r_.f32[3] = a_.f32[3]; + + return simde__m128_from_private(r_); + #endif +} + +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_sub_ss(a, b) simde_mm_sub_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_ucomieq_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_ucomieq_ss(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + int r; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x4_t a_not_nan = vceqq_f32(a_.neon_f32, a_.neon_f32); + uint32x4_t b_not_nan = vceqq_f32(b_.neon_f32, b_.neon_f32); + uint32x4_t a_or_b_nan = vmvnq_u32(vandq_u32(a_not_nan, b_not_nan)); + uint32x4_t a_eq_b = vceqq_f32(a_.neon_f32, b_.neon_f32); + r = !!(vgetq_lane_u32(vorrq_u32(a_or_b_nan, a_eq_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r = wasm_f32x4_extract_lane(a_.wasm_v128, 0) == wasm_f32x4_extract_lane(b_.wasm_v128, 0); + #elif defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = a_.f32[0] == b_.f32[0]; + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = a_.f32[0] == b_.f32[0]; + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_ucomieq_ss(a, b) simde_mm_ucomieq_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_ucomige_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_ucomige_ss(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + int r; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x4_t a_not_nan = vceqq_f32(a_.neon_f32, a_.neon_f32); + uint32x4_t b_not_nan = vceqq_f32(b_.neon_f32, b_.neon_f32); + uint32x4_t a_and_b_not_nan = vandq_u32(a_not_nan, b_not_nan); + uint32x4_t a_ge_b = vcgeq_f32(a_.neon_f32, b_.neon_f32); + r = !!(vgetq_lane_u32(vandq_u32(a_and_b_not_nan, a_ge_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r = wasm_f32x4_extract_lane(a_.wasm_v128, 0) >= wasm_f32x4_extract_lane(b_.wasm_v128, 0); + #elif defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = a_.f32[0] >= b_.f32[0]; + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = a_.f32[0] >= b_.f32[0]; + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_ucomige_ss(a, b) simde_mm_ucomige_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_ucomigt_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_ucomigt_ss(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + int r; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x4_t a_not_nan = vceqq_f32(a_.neon_f32, a_.neon_f32); + uint32x4_t b_not_nan = vceqq_f32(b_.neon_f32, b_.neon_f32); + uint32x4_t a_and_b_not_nan = vandq_u32(a_not_nan, b_not_nan); + uint32x4_t a_gt_b = vcgtq_f32(a_.neon_f32, b_.neon_f32); + r = !!(vgetq_lane_u32(vandq_u32(a_and_b_not_nan, a_gt_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r = wasm_f32x4_extract_lane(a_.wasm_v128, 0) > wasm_f32x4_extract_lane(b_.wasm_v128, 0); + #elif defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = a_.f32[0] > b_.f32[0]; + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = a_.f32[0] > b_.f32[0]; + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_ucomigt_ss(a, b) simde_mm_ucomigt_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_ucomile_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_ucomile_ss(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + int r; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x4_t a_not_nan = vceqq_f32(a_.neon_f32, a_.neon_f32); + uint32x4_t b_not_nan = vceqq_f32(b_.neon_f32, b_.neon_f32); + uint32x4_t a_or_b_nan = vmvnq_u32(vandq_u32(a_not_nan, b_not_nan)); + uint32x4_t a_le_b = vcleq_f32(a_.neon_f32, b_.neon_f32); + r = !!(vgetq_lane_u32(vorrq_u32(a_or_b_nan, a_le_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r = wasm_f32x4_extract_lane(a_.wasm_v128, 0) <= wasm_f32x4_extract_lane(b_.wasm_v128, 0); + #elif defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = a_.f32[0] <= b_.f32[0]; + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = a_.f32[0] <= b_.f32[0]; + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_ucomile_ss(a, b) simde_mm_ucomile_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_ucomilt_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_ucomilt_ss(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + int r; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x4_t a_not_nan = vceqq_f32(a_.neon_f32, a_.neon_f32); + uint32x4_t b_not_nan = vceqq_f32(b_.neon_f32, b_.neon_f32); + uint32x4_t a_or_b_nan = vmvnq_u32(vandq_u32(a_not_nan, b_not_nan)); + uint32x4_t a_lt_b = vcltq_f32(a_.neon_f32, b_.neon_f32); + r = !!(vgetq_lane_u32(vorrq_u32(a_or_b_nan, a_lt_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r = wasm_f32x4_extract_lane(a_.wasm_v128, 0) < wasm_f32x4_extract_lane(b_.wasm_v128, 0); + #elif defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = a_.f32[0] < b_.f32[0]; + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = a_.f32[0] < b_.f32[0]; + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_ucomilt_ss(a, b) simde_mm_ucomilt_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_ucomineq_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_ucomineq_ss(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + int r; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x4_t a_not_nan = vceqq_f32(a_.neon_f32, a_.neon_f32); + uint32x4_t b_not_nan = vceqq_f32(b_.neon_f32, b_.neon_f32); + uint32x4_t a_and_b_not_nan = vandq_u32(a_not_nan, b_not_nan); + uint32x4_t a_neq_b = vmvnq_u32(vceqq_f32(a_.neon_f32, b_.neon_f32)); + r = !!(vgetq_lane_u32(vandq_u32(a_and_b_not_nan, a_neq_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r = wasm_f32x4_extract_lane(a_.wasm_v128, 0) != wasm_f32x4_extract_lane(b_.wasm_v128, 0); + #elif defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = a_.f32[0] != b_.f32[0]; + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = a_.f32[0] != b_.f32[0]; + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_ucomineq_ss(a, b) simde_mm_ucomineq_ss((a), (b)) +#endif + +#if defined(SIMDE_X86_SSE_NATIVE) +# if defined(__has_builtin) +# if __has_builtin(__builtin_ia32_undef128) +# define SIMDE_HAVE_UNDEFINED128 +# endif +# elif !defined(__PGI) && !defined(SIMDE_BUG_GCC_REV_208793) && !defined(_MSC_VER) +# define SIMDE_HAVE_UNDEFINED128 +# endif +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_unpackhi_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_unpackhi_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f32 = vzip2q_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32x2_t a1 = vget_high_f32(a_.neon_f32); + float32x2_t b1 = vget_high_f32(b_.neon_f32); + float32x2x2_t result = vzip_f32(a1, b1); + r_.neon_f32 = vcombine_f32(result.val[0], result.val[1]); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vilvh_w(b_.lsx_i64, a_.lsx_i64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_shuffle(a_.wasm_v128, b_.wasm_v128, 2, 6, 3, 7); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.f32, b_.f32, 2, 6, 3, 7); + #else + r_.f32[0] = a_.f32[2]; + r_.f32[1] = b_.f32[2]; + r_.f32[2] = a_.f32[3]; + r_.f32[3] = b_.f32[3]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_unpackhi_ps(a, b) simde_mm_unpackhi_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_unpacklo_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE_NATIVE) + return _mm_unpacklo_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f32 = vzip1q_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_f32 = vec_mergeh(a_.altivec_f32, b_.altivec_f32); + #elif defined(SIMDE_LOONGARCH_LSX_NATIVE) + r_.lsx_i64 = __lsx_vilvl_w(b_.lsx_i64, a_.lsx_i64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_shuffle(a_.wasm_v128, b_.wasm_v128, 0, 4, 1, 5); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32x2_t a1 = vget_low_f32(a_.neon_f32); + float32x2_t b1 = vget_low_f32(b_.neon_f32); + float32x2x2_t result = vzip_f32(a1, b1); + r_.neon_f32 = vcombine_f32(result.val[0], result.val[1]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.f32, b_.f32, 0, 4, 1, 5); + #else + r_.f32[0] = a_.f32[0]; + r_.f32[1] = b_.f32[0]; + r_.f32[2] = a_.f32[1]; + r_.f32[3] = b_.f32[1]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_unpacklo_ps(a, b) simde_mm_unpacklo_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_stream_pi (simde__m64* mem_addr, simde__m64 a) { + #if defined(SIMDE_X86_SSE_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + _mm_stream_pi(HEDLEY_REINTERPRET_CAST(__m64*, mem_addr), a); + #elif HEDLEY_HAS_BUILTIN(__builtin_nontemporal_store) && ( \ + defined(SIMDE_ARM_NEON_A32V7_NATIVE) || defined(SIMDE_MIPS_LOONGSON_MMI_NATIVE) || \ + defined(SIMDE_VECTOR_SUBSCRIPT)) + __builtin_nontemporal_store(a, mem_addr); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + simde__m64_private a_ = simde__m64_to_private(a); + vst1_s64(HEDLEY_REINTERPRET_CAST(int64_t *, mem_addr), a_.neon_i64); + #else + simde__m64_private* + dest = HEDLEY_REINTERPRET_CAST(simde__m64_private*, mem_addr), + a_ = simde__m64_to_private(a); + + dest->i64[0] = a_.i64[0]; + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_stream_pi(mem_addr, a) simde_mm_stream_pi(mem_addr, (a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_stream_ps (simde_float32 mem_addr[4], simde__m128 a) { + #if defined(SIMDE_X86_SSE_NATIVE) + _mm_stream_ps(mem_addr, a); + #elif HEDLEY_HAS_BUILTIN(__builtin_nontemporal_store) && ( \ + defined(SIMDE_ARM_NEON_A32V7_NATIVE) || defined(SIMDE_VECTOR_SUBSCRIPT) || \ + defined(SIMDE_WASM_SIMD128_NATIVE) || defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || \ + defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) || defined(SIMDE_LOONGARCH_LSX_NATIVE)) + __builtin_nontemporal_store(a, SIMDE_ALIGN_ASSUME_CAST(__typeof__(a)*, mem_addr)); + #else + simde_mm_store_ps(mem_addr, a); + #endif +} +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _mm_stream_ps(mem_addr, a) simde_mm_stream_ps(SIMDE_CHECKED_REINTERPRET_CAST(float*, simde_float32*, mem_addr), (a)) +#endif + +#if defined(SIMDE_ARM_NEON_A32V7_NATIVE) && !defined(SIMDE_ARM_NEON_A64V8_NATIVE) + #define SIMDE_MM_TRANSPOSE4_PS(row0, row1, row2, row3) \ + do { \ + float32x4x2_t SIMDE_MM_TRANSPOSE4_PS_ROW01 = vtrnq_f32(row0, row1); \ + float32x4x2_t SIMDE_MM_TRANSPOSE4_PS_ROW23 = vtrnq_f32(row2, row3); \ + row0 = vcombine_f32(vget_low_f32(SIMDE_MM_TRANSPOSE4_PS_ROW01.val[0]), \ + vget_low_f32(SIMDE_MM_TRANSPOSE4_PS_ROW23.val[0])); \ + row1 = vcombine_f32(vget_low_f32(SIMDE_MM_TRANSPOSE4_PS_ROW01.val[1]), \ + vget_low_f32(SIMDE_MM_TRANSPOSE4_PS_ROW23.val[1])); \ + row2 = vcombine_f32(vget_high_f32(SIMDE_MM_TRANSPOSE4_PS_ROW01.val[0]), \ + vget_high_f32(SIMDE_MM_TRANSPOSE4_PS_ROW23.val[0])); \ + row3 = vcombine_f32(vget_high_f32(SIMDE_MM_TRANSPOSE4_PS_ROW01.val[1]), \ + vget_high_f32(SIMDE_MM_TRANSPOSE4_PS_ROW23.val[1])); \ + } while (0) +#else + #define SIMDE_MM_TRANSPOSE4_PS(row0, row1, row2, row3) \ + do { \ + simde__m128 SIMDE_MM_TRANSPOSE4_PS_tmp3, SIMDE_MM_TRANSPOSE4_PS_tmp2, SIMDE_MM_TRANSPOSE4_PS_tmp1, SIMDE_MM_TRANSPOSE4_PS_tmp0; \ + SIMDE_MM_TRANSPOSE4_PS_tmp0 = simde_mm_unpacklo_ps((row0), (row1)); \ + SIMDE_MM_TRANSPOSE4_PS_tmp2 = simde_mm_unpacklo_ps((row2), (row3)); \ + SIMDE_MM_TRANSPOSE4_PS_tmp1 = simde_mm_unpackhi_ps((row0), (row1)); \ + SIMDE_MM_TRANSPOSE4_PS_tmp3 = simde_mm_unpackhi_ps((row2), (row3)); \ + row0 = simde_mm_movelh_ps(SIMDE_MM_TRANSPOSE4_PS_tmp0, SIMDE_MM_TRANSPOSE4_PS_tmp2); \ + row1 = simde_mm_movehl_ps(SIMDE_MM_TRANSPOSE4_PS_tmp2, SIMDE_MM_TRANSPOSE4_PS_tmp0); \ + row2 = simde_mm_movelh_ps(SIMDE_MM_TRANSPOSE4_PS_tmp1, SIMDE_MM_TRANSPOSE4_PS_tmp3); \ + row3 = simde_mm_movehl_ps(SIMDE_MM_TRANSPOSE4_PS_tmp3, SIMDE_MM_TRANSPOSE4_PS_tmp1); \ + } while (0) +#endif +#if defined(SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES) +# define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) SIMDE_MM_TRANSPOSE4_PS(row0, row1, row2, row3) +#endif + +SIMDE_END_DECLS_ + +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_SSE_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse2.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse2.h new file mode 100644 index 00000000..be85177d --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse2.h @@ -0,0 +1,7741 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2017-2020 Evan Nemerson + * 2015-2017 John W. Ratcliff + * 2015 Brandon Rowlett + * 2015 Ken Fast + * 2017 Hasindu Gamaarachchi + * 2018 Jeff Daily + */ + +#if !defined(SIMDE_X86_SSE2_H) +#define SIMDE_X86_SSE2_H + +#include "sse.h" +#include "../simde-f16.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +typedef union { + #if defined(SIMDE_VECTOR_SUBSCRIPT) + SIMDE_ALIGN_TO_16 int8_t i8 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int16_t i16 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int32_t i32 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int64_t i64 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint8_t u8 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint16_t u16 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint32_t u32 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint64_t u64 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_ALIGN_TO_16 simde_int128 i128 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 simde_uint128 u128 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + #endif + #if defined(SIMDE_FLOAT16_VECTOR) + SIMDE_ALIGN_TO_16 simde_float16 f16 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + #else + SIMDE_ALIGN_TO_16 simde_float16 f16[8]; + #endif + SIMDE_ALIGN_TO_16 simde_float32 f32 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 simde_float64 f64 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + + SIMDE_ALIGN_TO_16 int_fast32_t i32f SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint_fast32_t u32f SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + #else + SIMDE_ALIGN_TO_16 int8_t i8[16]; + SIMDE_ALIGN_TO_16 int16_t i16[8]; + SIMDE_ALIGN_TO_16 int32_t i32[4]; + SIMDE_ALIGN_TO_16 int64_t i64[2]; + SIMDE_ALIGN_TO_16 uint8_t u8[16]; + SIMDE_ALIGN_TO_16 uint16_t u16[8]; + SIMDE_ALIGN_TO_16 uint32_t u32[4]; + SIMDE_ALIGN_TO_16 uint64_t u64[2]; + #if defined(SIMDE_HAVE_INT128_) + SIMDE_ALIGN_TO_16 simde_int128 i128[1]; + SIMDE_ALIGN_TO_16 simde_uint128 u128[1]; + #endif + SIMDE_ALIGN_TO_16 simde_float16 f16[8]; + SIMDE_ALIGN_TO_16 simde_float32 f32[4]; + SIMDE_ALIGN_TO_16 simde_float64 f64[2]; + + SIMDE_ALIGN_TO_16 int_fast32_t i32f[16 / sizeof(int_fast32_t)]; + SIMDE_ALIGN_TO_16 uint_fast32_t u32f[16 / sizeof(uint_fast32_t)]; + #endif + + SIMDE_ALIGN_TO_16 simde__m64_private m64_private[2]; + SIMDE_ALIGN_TO_16 simde__m64 m64[2]; + + #if defined(SIMDE_X86_SSE2_NATIVE) + SIMDE_ALIGN_TO_16 __m128i n; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_ALIGN_TO_16 int8x16_t neon_i8; + SIMDE_ALIGN_TO_16 int16x8_t neon_i16; + SIMDE_ALIGN_TO_16 int32x4_t neon_i32; + SIMDE_ALIGN_TO_16 int64x2_t neon_i64; + SIMDE_ALIGN_TO_16 uint8x16_t neon_u8; + SIMDE_ALIGN_TO_16 uint16x8_t neon_u16; + SIMDE_ALIGN_TO_16 uint32x4_t neon_u32; + SIMDE_ALIGN_TO_16 uint64x2_t neon_u64; + #if defined(__ARM_FP16_FORMAT_IEEE) + SIMDE_ALIGN_TO_16 float16x8_t neon_f16; + #endif + SIMDE_ALIGN_TO_16 float32x4_t neon_f32; + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + SIMDE_ALIGN_TO_16 float64x2_t neon_f64; + #endif + #elif defined(SIMDE_MIPS_MSA_NATIVE) + v16i8 msa_i8; + v8i16 msa_i16; + v4i32 msa_i32; + v2i64 msa_i64; + v16u8 msa_u8; + v8u16 msa_u16; + v4u32 msa_u32; + v2u64 msa_u64; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + SIMDE_ALIGN_TO_16 v128_t wasm_v128; + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed char) altivec_i8; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed short) altivec_i16; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed int) altivec_i32; + #if defined(__UINT_FAST32_TYPE__) && (defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE)) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(__INT_FAST32_TYPE__) altivec_i32f; + #else + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed int) altivec_i32f; + #endif + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) altivec_u8; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) altivec_u16; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32; + #if defined(__UINT_FAST32_TYPE__) && (defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE)) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(__UINT_FAST32_TYPE__) altivec_u32f; + #else + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32f; + #endif + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(float) altivec_f32; + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed long long) altivec_i64; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long) altivec_u64; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(double) altivec_f64; + #endif + #endif +} simde__m128i_private; + +typedef union { + #if defined(SIMDE_VECTOR_SUBSCRIPT) + SIMDE_ALIGN_TO_16 int8_t i8 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int16_t i16 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int32_t i32 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int64_t i64 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint8_t u8 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint16_t u16 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint32_t u32 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint64_t u64 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 simde_float32 f32 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 simde_float64 f64 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 int_fast32_t i32f SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + SIMDE_ALIGN_TO_16 uint_fast32_t u32f SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + #else + SIMDE_ALIGN_TO_16 int8_t i8[16]; + SIMDE_ALIGN_TO_16 int16_t i16[8]; + SIMDE_ALIGN_TO_16 int32_t i32[4]; + SIMDE_ALIGN_TO_16 int64_t i64[2]; + SIMDE_ALIGN_TO_16 uint8_t u8[16]; + SIMDE_ALIGN_TO_16 uint16_t u16[8]; + SIMDE_ALIGN_TO_16 uint32_t u32[4]; + SIMDE_ALIGN_TO_16 uint64_t u64[2]; + SIMDE_ALIGN_TO_16 simde_float32 f32[4]; + SIMDE_ALIGN_TO_16 simde_float64 f64[2]; + SIMDE_ALIGN_TO_16 int_fast32_t i32f[16 / sizeof(int_fast32_t)]; + SIMDE_ALIGN_TO_16 uint_fast32_t u32f[16 / sizeof(uint_fast32_t)]; + #endif + + SIMDE_ALIGN_TO_16 simde__m64_private m64_private[2]; + SIMDE_ALIGN_TO_16 simde__m64 m64[2]; + + #if defined(SIMDE_X86_SSE2_NATIVE) || defined(SIMDE_X86_SVML_NATIVE) + SIMDE_ALIGN_TO_16 __m128d n; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_ALIGN_TO_16 int8x16_t neon_i8; + SIMDE_ALIGN_TO_16 int16x8_t neon_i16; + SIMDE_ALIGN_TO_16 int32x4_t neon_i32; + SIMDE_ALIGN_TO_16 int64x2_t neon_i64; + SIMDE_ALIGN_TO_16 uint8x16_t neon_u8; + SIMDE_ALIGN_TO_16 uint16x8_t neon_u16; + SIMDE_ALIGN_TO_16 uint32x4_t neon_u32; + SIMDE_ALIGN_TO_16 uint64x2_t neon_u64; + SIMDE_ALIGN_TO_16 float32x4_t neon_f32; + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + SIMDE_ALIGN_TO_16 float64x2_t neon_f64; + #endif + #elif defined(SIMDE_MIPS_MSA_NATIVE) + v16i8 msa_i8; + v8i16 msa_i16; + v4i32 msa_i32; + v2i64 msa_i64; + v16u8 msa_u8; + v8u16 msa_u16; + v4u32 msa_u32; + v2u64 msa_u64; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + SIMDE_ALIGN_TO_16 v128_t wasm_v128; + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed char) altivec_i8; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed short) altivec_i16; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed int) altivec_i32; + #if defined(__INT_FAST32_TYPE__) && (defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE)) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(__INT_FAST32_TYPE__) altivec_i32f; + #else + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed int) altivec_i32f; + #endif + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) altivec_u8; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) altivec_u16; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32; + #if defined(__UINT_FAST32_TYPE__) && (defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE)) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(__UINT_FAST32_TYPE__) altivec_u32f; + #else + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned int) altivec_u32f; + #endif + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(float) altivec_f32; + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(signed long long) altivec_i64; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long) altivec_u64; + SIMDE_ALIGN_TO_16 SIMDE_POWER_ALTIVEC_VECTOR(double) altivec_f64; + #endif + #endif +} simde__m128d_private; + +#if defined(SIMDE_X86_SSE2_NATIVE) || defined(SIMDE_X86_SVML_NATIVE) + typedef __m128i simde__m128i; + typedef __m128d simde__m128d; +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + typedef int64x2_t simde__m128i; +# if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + typedef float64x2_t simde__m128d; +# elif defined(SIMDE_VECTOR_SUBSCRIPT) + typedef simde_float64 simde__m128d SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; +# else + typedef simde__m128d_private simde__m128d; +# endif +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + typedef v128_t simde__m128i; + typedef v128_t simde__m128d; +#elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + typedef SIMDE_POWER_ALTIVEC_VECTOR(float) simde__m128i; + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + typedef SIMDE_POWER_ALTIVEC_VECTOR(double) simde__m128d; + #else + typedef simde__m128d_private simde__m128d; + #endif +#elif defined(SIMDE_VECTOR_SUBSCRIPT) + typedef int64_t simde__m128i SIMDE_ALIGN_TO_16 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; + typedef simde_float64 simde__m128d SIMDE_ALIGN_TO_16 SIMDE_VECTOR(16) SIMDE_MAY_ALIAS; +#else + typedef simde__m128i_private simde__m128i; + typedef simde__m128d_private simde__m128d; +#endif + +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + typedef simde__m128i __m128i; + typedef simde__m128d __m128d; +#endif + +HEDLEY_STATIC_ASSERT(16 == sizeof(simde__m128i), "simde__m128i size incorrect"); +HEDLEY_STATIC_ASSERT(16 == sizeof(simde__m128i_private), "simde__m128i_private size incorrect"); +HEDLEY_STATIC_ASSERT(16 == sizeof(simde__m128d), "simde__m128d size incorrect"); +HEDLEY_STATIC_ASSERT(16 == sizeof(simde__m128d_private), "simde__m128d_private size incorrect"); +#if defined(SIMDE_CHECK_ALIGNMENT) && defined(SIMDE_ALIGN_OF) +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m128i) == 16, "simde__m128i is not 16-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m128i_private) == 16, "simde__m128i_private is not 16-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m128d) == 16, "simde__m128d is not 16-byte aligned"); +HEDLEY_STATIC_ASSERT(SIMDE_ALIGN_OF(simde__m128d_private) == 16, "simde__m128d_private is not 16-byte aligned"); +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde__m128i_from_private(simde__m128i_private v) { + simde__m128i r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i_private +simde__m128i_to_private(simde__m128i v) { + simde__m128i_private r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde__m128d_from_private(simde__m128d_private v) { + simde__m128d r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d_private +simde__m128d_to_private(simde__m128d v) { + simde__m128d_private r; + simde_memcpy(&r, &v, sizeof(r)); + return r; +} + +#if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, int8x16_t, neon, i8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, int16x8_t, neon, i16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, int32x4_t, neon, i32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, int64x2_t, neon, i64) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, uint8x16_t, neon, u8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, uint16x8_t, neon, u16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, uint32x4_t, neon, u32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, uint64x2_t, neon, u64) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, float32x4_t, neon, f32) + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, float64x2_t, neon, f64) + #endif +#elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, SIMDE_POWER_ALTIVEC_VECTOR(signed char), altivec, i8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, SIMDE_POWER_ALTIVEC_VECTOR(signed short), altivec, i16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, SIMDE_POWER_ALTIVEC_VECTOR(signed int), altivec, i32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), altivec, u8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, SIMDE_POWER_ALTIVEC_VECTOR(unsigned short), altivec, u16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, SIMDE_POWER_ALTIVEC_VECTOR(unsigned int), altivec, u32) + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long), altivec, u64) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, SIMDE_POWER_ALTIVEC_VECTOR(signed long long), altivec, i64) + #endif +#endif /* defined(SIMDE_ARM_NEON_A32V7_NATIVE) */ + +#if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, int8x16_t, neon, i8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, int16x8_t, neon, i16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, int32x4_t, neon, i32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, int64x2_t, neon, i64) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, uint8x16_t, neon, u8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, uint16x8_t, neon, u16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, uint32x4_t, neon, u32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, uint64x2_t, neon, u64) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, float32x4_t, neon, f32) + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, float64x2_t, neon, f64) + #endif +#elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, SIMDE_POWER_ALTIVEC_VECTOR(signed char), altivec, i8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, SIMDE_POWER_ALTIVEC_VECTOR(signed short), altivec, i16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, SIMDE_POWER_ALTIVEC_VECTOR(signed int), altivec, i32) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), altivec, u8) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, SIMDE_POWER_ALTIVEC_VECTOR(unsigned short), altivec, u16) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, SIMDE_POWER_ALTIVEC_VECTOR(unsigned int), altivec, u32) + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long), altivec, u64) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, SIMDE_POWER_ALTIVEC_VECTOR(signed long long), altivec, i64) + #if defined(SIMDE_BUG_GCC_95782) + SIMDE_FUNCTION_ATTRIBUTES + SIMDE_POWER_ALTIVEC_VECTOR(double) + simde__m128d_to_altivec_f64(simde__m128d value) { + simde__m128d_private r_ = simde__m128d_to_private(value); + return r_.altivec_f64; + } + + SIMDE_FUNCTION_ATTRIBUTES + simde__m128d + simde__m128d_from_altivec_f64(SIMDE_POWER_ALTIVEC_VECTOR(double) value) { + simde__m128d_private r_; + r_.altivec_f64 = value; + return simde__m128d_from_private(r_); + } + #else + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, SIMDE_POWER_ALTIVEC_VECTOR(double), altivec, f64) + #endif + #endif +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128d, v128_t, wasm, v128); + SIMDE_X86_GENERATE_CONVERSION_FUNCTION(m128i, v128_t, wasm, v128); +#endif /* defined(SIMDE_ARM_NEON_A32V7_NATIVE) */ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_set_pd (simde_float64 e1, simde_float64 e0) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_set_pd(e1, e0); + #else + simde__m128d_private r_; + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_make(e0, e1); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + SIMDE_ALIGN_TO_16 simde_float64 data[2] = { e0, e1 }; + r_.neon_f64 = vld1q_f64(data); + #else + r_.f64[0] = e0; + r_.f64[1] = e1; + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_set_pd(e1, e0) simde_mm_set_pd(e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_set1_pd (simde_float64 a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_set1_pd(a); + #else + simde__m128d_private r_; + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_splat(a); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vdupq_n_f64(a); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = vec_splats(HEDLEY_STATIC_CAST(double, a)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.f64[i] = a; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#define simde_mm_set_pd1(a) simde_mm_set1_pd(a) +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_set1_pd(a) simde_mm_set1_pd(a) + #define _mm_set_pd1(a) simde_mm_set1_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_x_mm_abs_pd(simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + simde_float64 mask_; + uint64_t u64_ = UINT64_C(0x7FFFFFFFFFFFFFFF); + simde_memcpy(&mask_, &u64_, sizeof(u64_)); + return _mm_and_pd(_mm_set1_pd(mask_), a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vabsq_f64(a_.neon_f64); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = vec_abs(a_.altivec_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_abs(a_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_fabs(a_.f64[i]); + } + #endif + + return simde__m128d_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_x_mm_not_pd(simde__m128d a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + __m128i ai = _mm_castpd_si128(a); + return _mm_castsi128_pd(_mm_ternarylogic_epi64(ai, ai, ai, 0x55)); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vmvnq_s32(a_.neon_i32); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + r_.altivec_f64 = vec_nor(a_.altivec_f64, a_.altivec_f64); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i32 = vec_nor(a_.altivec_i32, a_.altivec_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_not(a_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = ~a_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = ~(a_.i32f[i]); + } + #endif + + return simde__m128d_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_x_mm_select_pd(simde__m128d a, simde__m128d b, simde__m128d mask) { + /* This function is for when you want to blend two elements together + * according to a mask. It is similar to _mm_blendv_pd, except that + * it is undefined whether the blend is based on the highest bit in + * each lane (like blendv) or just bitwise operations. This allows + * us to implement the function efficiently everywhere. + * + * Basically, you promise that all the lanes in mask are either 0 or + * ~0. */ + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_blendv_pd(a, b, mask); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b), + mask_ = simde__m128d_to_private(mask); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 ^ ((a_.i64 ^ b_.i64) & mask_.i64); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vbslq_s64(mask_.neon_u64, b_.neon_i64, a_.neon_i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] ^ ((a_.i64[i] ^ b_.i64[i]) & mask_.i64[i]); + } + #endif + + return simde__m128d_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_add_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_add_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vaddq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i8 = vec_add(a_.altivec_i8, b_.altivec_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_add(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = a_.i8 + b_.i8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] + b_.i8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_add_epi8(a, b) simde_mm_add_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_add_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_add_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vaddq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i16 = vec_add(a_.altivec_i16, b_.altivec_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_add(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = a_.i16 + b_.i16; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] + b_.i16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_add_epi16(a, b) simde_mm_add_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_add_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_add_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vaddq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i32 = vec_add(a_.altivec_i32, b_.altivec_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_add(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 + b_.i32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] + b_.i32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_add_epi32(a, b) simde_mm_add_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_add_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_add_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vaddq_s64(a_.neon_i64, b_.neon_i64); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + r_.altivec_i64 = vec_add(a_.altivec_i64, b_.altivec_i64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_add(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 + b_.i64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] + b_.i64[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_add_epi64(a, b) simde_mm_add_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_add_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_add_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vaddq_f64(a_.neon_f64, b_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_add(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + r_.altivec_f64 = vec_add(a_.altivec_f64, b_.altivec_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_add(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = a_.f64 + b_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[i] + b_.f64[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_add_pd(a, b) simde_mm_add_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_move_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_move_sd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vsetq_lane_f64(vgetq_lane_f64(b_.neon_f64, 0), a_.neon_f64, 0); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + #if defined(HEDLEY_IBM_VERSION) + r_.altivec_f64 = vec_xxpermdi(a_.altivec_f64, b_.altivec_f64, 1); + #else + r_.altivec_f64 = vec_xxpermdi(b_.altivec_f64, a_.altivec_f64, 1); + #endif + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_shuffle(a_.wasm_v128, b_.wasm_v128, 2, 1); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.f64, b_.f64, 2, 1); + #else + r_.f64[0] = b_.f64[0]; + r_.f64[1] = a_.f64[1]; + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_move_sd(a, b) simde_mm_move_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_x_mm_broadcastlow_pd(simde__m128d a) { + /* This function broadcasts the first element in the input vector to + * all lanes. It is used to avoid generating spurious exceptions in + * *_sd functions since there may be garbage in the upper lanes. */ + + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_castsi128_pd(_mm_shuffle_epi32(_mm_castpd_si128(a), 0x44)); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vdupq_laneq_f64(a_.neon_f64, 0); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + r_.altivec_f64 = vec_splat(a_.altivec_f64, 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_splat(a_.f64[0]); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.f64, a_.f64, 0, 0); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[0]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_add_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_add_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_add_pd(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_add_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + r_.f64[0] = a_.f64[0] + b_.f64[0]; + r_.f64[1] = a_.f64[1]; + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_add_sd(a, b) simde_mm_add_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_add_si64 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_add_si64(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vadd_s64(a_.neon_i64, b_.neon_i64); + #else + r_.i64[0] = a_.i64[0] + b_.i64[0]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_add_si64(a, b) simde_mm_add_si64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_adds_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_adds_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vqaddq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_add_sat(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i8 = vec_adds(a_.altivec_i8, b_.altivec_i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = simde_math_adds_i8(a_.i8[i], b_.i8[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_adds_epi8(a, b) simde_mm_adds_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_adds_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_adds_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vqaddq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_add_sat(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i16 = vec_adds(a_.altivec_i16, b_.altivec_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = simde_math_adds_i16(a_.i16[i], b_.i16[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_adds_epi16(a, b) simde_mm_adds_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_adds_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_adds_epu8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vqaddq_u8(a_.neon_u8, b_.neon_u8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u8x16_add_sat(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + r_.altivec_u8 = vec_adds(a_.altivec_u8, b_.altivec_u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = simde_math_adds_u8(a_.u8[i], b_.u8[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_adds_epu8(a, b) simde_mm_adds_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_adds_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_adds_epu16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vqaddq_u16(a_.neon_u16, b_.neon_u16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u16x8_add_sat(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_u16 = vec_adds(a_.altivec_u16, b_.altivec_u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = simde_math_adds_u16(a_.u16[i], b_.u16[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_adds_epu16(a, b) simde_mm_adds_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_and_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_and_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vandq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_and(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + r_.altivec_f64 = vec_and(a_.altivec_f64, b_.altivec_f64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f & b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] & b_.i32f[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_and_pd(a, b) simde_mm_and_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_and_si128 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_and_si128(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vandq_s32(b_.neon_i32, a_.neon_i32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_u32f = vec_and(a_.altivec_u32f, b_.altivec_u32f); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_and(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f & b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] & b_.i32f[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_and_si128(a, b) simde_mm_and_si128(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_andnot_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_andnot_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vbicq_s32(b_.neon_i32, a_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_andnot(b_.wasm_v128, a_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = vec_andc(b_.altivec_f64, a_.altivec_f64); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i32f = vec_andc(b_.altivec_i32f, a_.altivec_i32f); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = ~a_.i32f & b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = ~a_.u64[i] & b_.u64[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_andnot_pd(a, b) simde_mm_andnot_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_andnot_si128 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_andnot_si128(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vbicq_s32(b_.neon_i32, a_.neon_i32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i32 = vec_andc(b_.altivec_i32, a_.altivec_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_andnot(b_.wasm_v128, a_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = ~a_.i32f & b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = ~(a_.i32f[i]) & b_.i32f[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_andnot_si128(a, b) simde_mm_andnot_si128(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_xor_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_xor_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f ^ b_.i32f; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_xor(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = veorq_s64(a_.neon_i64, b_.neon_i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] ^ b_.i32f[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_xor_pd(a, b) simde_mm_xor_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_avg_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_avg_epu8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vrhaddq_u8(b_.neon_u8, a_.neon_u8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u8x16_avgr(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u8 = vec_avg(a_.altivec_u8, b_.altivec_u8); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && defined(SIMDE_CONVERT_VECTOR_) + uint16_t wa SIMDE_VECTOR(32); + uint16_t wb SIMDE_VECTOR(32); + uint16_t wr SIMDE_VECTOR(32); + SIMDE_CONVERT_VECTOR_(wa, a_.u8); + SIMDE_CONVERT_VECTOR_(wb, b_.u8); + wr = (wa + wb + 1) >> 1; + SIMDE_CONVERT_VECTOR_(r_.u8, wr); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] + b_.u8[i] + 1) >> 1; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_avg_epu8(a, b) simde_mm_avg_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_avg_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_avg_epu16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vrhaddq_u16(b_.neon_u16, a_.neon_u16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u16x8_avgr(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u16 = vec_avg(a_.altivec_u16, b_.altivec_u16); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && defined(SIMDE_CONVERT_VECTOR_) + uint32_t wa SIMDE_VECTOR(32); + uint32_t wb SIMDE_VECTOR(32); + uint32_t wr SIMDE_VECTOR(32); + SIMDE_CONVERT_VECTOR_(wa, a_.u16); + SIMDE_CONVERT_VECTOR_(wb, b_.u16); + wr = (wa + wb + 1) >> 1; + SIMDE_CONVERT_VECTOR_(r_.u16, wr); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] + b_.u16[i] + 1) >> 1; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_avg_epu16(a, b) simde_mm_avg_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_setzero_si128 (void) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_setzero_si128(); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vdupq_n_s32(0); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i32 = vec_splats(HEDLEY_STATIC_CAST(signed int, 0)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_splat(INT32_C(0)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT) + r_.i32 = __extension__ (__typeof__(r_.i32)) { 0, 0, 0, 0 }; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = 0; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_setzero_si128() (simde_mm_setzero_si128()) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_bslli_si128 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + if (HEDLEY_UNLIKELY((imm8 & ~15))) { + return simde_mm_setzero_si128(); + } + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) && defined(SIMDE_ENDIAN_ORDER) + r_.altivec_i8 = + #if (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + vec_slo + #else /* SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_BIG */ + vec_sro + #endif + (a_.altivec_i8, vec_splats(HEDLEY_STATIC_CAST(unsigned char, imm8 * 8))); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i8 = vec_srb(a_.altivec_i8, vec_splats(HEDLEY_STATIC_CAST(unsigned char, (imm8 & 15) << 3))); + #elif defined(SIMDE_HAVE_INT128_) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + r_.u128[0] = a_.u128[0] << (imm8 * 8); + #else + r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + for (int i = imm8 ; i < HEDLEY_STATIC_CAST(int, sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i - imm8]; + } + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) && !defined(__PGI) + #define simde_mm_bslli_si128(a, imm8) _mm_slli_si128(a, imm8) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) && !defined(__clang__) + #define simde_mm_bslli_si128(a, imm8) \ + simde__m128i_from_neon_i8(((imm8) <= 0) ? simde__m128i_to_neon_i8(a) : (((imm8) > 15) ? (vdupq_n_s8(0)) : (vextq_s8(vdupq_n_s8(0), simde__m128i_to_neon_i8(a), 16 - (imm8))))) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_bslli_si128(a, imm8) __extension__ ({ \ + simde__m128i_from_wasm_v128( \ + wasm_i8x16_shuffle(wasm_i32x4_splat(INT32_C(0)), \ + simde__m128i_to_wasm_v128((a)), \ + ((imm8)&0xF0) ? 0 : 16 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 17 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 18 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 19 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 20 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 21 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 22 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 23 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 24 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 25 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 26 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 27 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 28 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 29 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 30 - ((imm8)&0xF), \ + ((imm8)&0xF0) ? 0 : 31 - ((imm8)&0xF))); }) +#elif defined(SIMDE_SHUFFLE_VECTOR_) && !defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) && !defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + #define simde_mm_bslli_si128(a, imm8) (__extension__ ({ \ + const simde__m128i_private simde_tmp_a_ = simde__m128i_to_private(a); \ + const simde__m128i_private simde_tmp_z_ = simde__m128i_to_private(simde_mm_setzero_si128()); \ + simde__m128i_private simde_tmp_r_; \ + if (HEDLEY_UNLIKELY(imm8 > 15)) { \ + simde_tmp_r_ = simde__m128i_to_private(simde_mm_setzero_si128()); \ + } else { \ + simde_tmp_r_.i8 = \ + SIMDE_SHUFFLE_VECTOR_(8, 16, \ + simde_tmp_z_.i8, \ + (simde_tmp_a_).i8, \ + HEDLEY_STATIC_CAST(int8_t, (16 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (17 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (18 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (19 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (20 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (21 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (22 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (23 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (24 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (25 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (26 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (27 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (28 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (29 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (30 - imm8) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (31 - imm8) & 31)); \ + } \ + simde__m128i_from_private(simde_tmp_r_); })) +#endif +#define simde_mm_slli_si128(a, imm8) simde_mm_bslli_si128(a, imm8) +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_bslli_si128(a, imm8) simde_mm_bslli_si128(a, imm8) + #define _mm_slli_si128(a, imm8) simde_mm_bslli_si128(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_bsrli_si128 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + if (HEDLEY_UNLIKELY((imm8 & ~15))) { + return simde_mm_setzero_si128(); + } + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) && defined(SIMDE_ENDIAN_ORDER) + r_.altivec_i8 = + #if (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + vec_sro + #else /* SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_BIG */ + vec_slo + #endif + (a_.altivec_i8, vec_splats(HEDLEY_STATIC_CAST(unsigned char, imm8 * 8))); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i8 = vec_slb(a_.altivec_i8, vec_splats(HEDLEY_STATIC_CAST(unsigned char, (imm8 & 15) << 3))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + const int e = HEDLEY_STATIC_CAST(int, i) + imm8; + r_.i8[i] = (e < 16) ? a_.i8[e] : 0; + } + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) && !defined(__PGI) + #define simde_mm_bsrli_si128(a, imm8) _mm_srli_si128(a, imm8) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) && !defined(__clang__) + #define simde_mm_bsrli_si128(a, imm8) \ + simde__m128i_from_neon_i8(((imm8 < 0) || (imm8 > 15)) ? vdupq_n_s8(0) : (vextq_s8(simde__m128i_to_private(a).neon_i8, vdupq_n_s8(0), ((imm8 & 15) != 0) ? imm8 : (imm8 & 15)))) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_bsrli_si128(a, imm8) (__extension__ ({ \ + const simde__m128i_private simde_tmp_a_ = simde__m128i_to_private(a); \ + const simde__m128i_private simde_tmp_z_ = simde__m128i_to_private(simde_mm_setzero_si128()); \ + simde__m128i_private simde_tmp_r_ = simde__m128i_to_private(a); \ + if (HEDLEY_UNLIKELY(imm8 > 15)) { \ + simde_tmp_r_ = simde__m128i_to_private(simde_mm_setzero_si128()); \ + } else { \ + simde_tmp_r_.wasm_v128 = \ + wasm_i8x16_shuffle( \ + simde_tmp_z_.wasm_v128, \ + simde_tmp_a_.wasm_v128, \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 16) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 17) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 18) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 19) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 20) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 21) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 22) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 23) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 24) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 25) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 26) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 27) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 28) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 29) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 30) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 31) & 31)); \ + } \ + simde__m128i_from_private(simde_tmp_r_); })) +#elif defined(SIMDE_SHUFFLE_VECTOR_) && !defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) && !defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + #define simde_mm_bsrli_si128(a, imm8) (__extension__ ({ \ + const simde__m128i_private simde_tmp_a_ = simde__m128i_to_private(a); \ + const simde__m128i_private simde_tmp_z_ = simde__m128i_to_private(simde_mm_setzero_si128()); \ + simde__m128i_private simde_tmp_r_ = simde__m128i_to_private(a); \ + if (HEDLEY_UNLIKELY(imm8 > 15)) { \ + simde_tmp_r_ = simde__m128i_to_private(simde_mm_setzero_si128()); \ + } else { \ + simde_tmp_r_.i8 = \ + SIMDE_SHUFFLE_VECTOR_(8, 16, \ + simde_tmp_z_.i8, \ + (simde_tmp_a_).i8, \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 16) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 17) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 18) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 19) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 20) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 21) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 22) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 23) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 24) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 25) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 26) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 27) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 28) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 29) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 30) & 31), \ + HEDLEY_STATIC_CAST(int8_t, (imm8 + 31) & 31)); \ + } \ + simde__m128i_from_private(simde_tmp_r_); })) +#endif +#define simde_mm_srli_si128(a, imm8) simde_mm_bsrli_si128((a), (imm8)) +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_bsrli_si128(a, imm8) simde_mm_bsrli_si128((a), (imm8)) + #define _mm_srli_si128(a, imm8) simde_mm_bsrli_si128((a), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_clflush (void const* p) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_clflush(p); + #else + (void) p; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_clflush(p) simde_mm_clflush(p) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_comieq_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_comieq_sd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return !!vgetq_lane_u64(vceqq_f64(a_.neon_f64, b_.neon_f64), 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f64x2_extract_lane(a_.wasm_v128, 0) == wasm_f64x2_extract_lane(b_.wasm_v128, 0); + #else + return a_.f64[0] == b_.f64[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_comieq_sd(a, b) simde_mm_comieq_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_comige_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_comige_sd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return !!vgetq_lane_u64(vcgeq_f64(a_.neon_f64, b_.neon_f64), 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f64x2_extract_lane(a_.wasm_v128, 0) >= wasm_f64x2_extract_lane(b_.wasm_v128, 0); + #else + return a_.f64[0] >= b_.f64[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_comige_sd(a, b) simde_mm_comige_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_comigt_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_comigt_sd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return !!vgetq_lane_u64(vcgtq_f64(a_.neon_f64, b_.neon_f64), 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f64x2_extract_lane(a_.wasm_v128, 0) > wasm_f64x2_extract_lane(b_.wasm_v128, 0); + #else + return a_.f64[0] > b_.f64[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_comigt_sd(a, b) simde_mm_comigt_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_comile_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_comile_sd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return !!vgetq_lane_u64(vcleq_f64(a_.neon_f64, b_.neon_f64), 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f64x2_extract_lane(a_.wasm_v128, 0) <= wasm_f64x2_extract_lane(b_.wasm_v128, 0); + #else + return a_.f64[0] <= b_.f64[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_comile_sd(a, b) simde_mm_comile_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_comilt_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_comilt_sd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return !!vgetq_lane_u64(vcltq_f64(a_.neon_f64, b_.neon_f64), 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f64x2_extract_lane(a_.wasm_v128, 0) < wasm_f64x2_extract_lane(b_.wasm_v128, 0); + #else + return a_.f64[0] < b_.f64[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_comilt_sd(a, b) simde_mm_comilt_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_comineq_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_comineq_sd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return !vgetq_lane_u64(vceqq_f64(a_.neon_f64, b_.neon_f64), 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f64x2_extract_lane(a_.wasm_v128, 0) != wasm_f64x2_extract_lane(b_.wasm_v128, 0); + #else + return a_.f64[0] != b_.f64[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_comineq_sd(a, b) simde_mm_comineq_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_x_mm_copysign_pd(simde__m128d dest, simde__m128d src) { + simde__m128d_private + r_, + dest_ = simde__m128d_to_private(dest), + src_ = simde__m128d_to_private(src); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint64x2_t sign_pos = vreinterpretq_u64_f64(vdupq_n_f64(-SIMDE_FLOAT64_C(0.0))); + #else + simde_float64 dbl_nz = -SIMDE_FLOAT64_C(0.0); + uint64_t u64_nz; + simde_memcpy(&u64_nz, &dbl_nz, sizeof(u64_nz)); + uint64x2_t sign_pos = vdupq_n_u64(u64_nz); + #endif + r_.neon_u64 = vbslq_u64(sign_pos, src_.neon_u64, dest_.neon_u64); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + #if defined(SIMDE_BUG_VEC_CPSGN_REVERSED_ARGS) + r_.altivec_f64 = vec_cpsgn(dest_.altivec_f64, src_.altivec_f64); + #else + r_.altivec_f64 = vec_cpsgn(src_.altivec_f64, dest_.altivec_f64); + #endif + #elif defined(simde_math_copysign) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_copysign(dest_.f64[i], src_.f64[i]); + } + #else + simde__m128d sgnbit = simde_mm_set1_pd(-SIMDE_FLOAT64_C(0.0)); + return simde_mm_xor_pd(simde_mm_and_pd(sgnbit, src), simde_mm_andnot_pd(sgnbit, dest)); + #endif + + return simde__m128d_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_x_mm_xorsign_pd(simde__m128d dest, simde__m128d src) { + return simde_mm_xor_pd(simde_mm_and_pd(simde_mm_set1_pd(-0.0), src), dest); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_castpd_ps (simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_castpd_ps(a); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return vreinterpretq_f32_f64(a); + #else + simde__m128 r; + simde_memcpy(&r, &a, sizeof(a)); + return r; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_castpd_ps(a) simde_mm_castpd_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_castpd_si128 (simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_castpd_si128(a); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return vreinterpretq_s64_f64(a); + #else + simde__m128i r; + simde_memcpy(&r, &a, sizeof(a)); + return r; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_castpd_si128(a) simde_mm_castpd_si128(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_castps_pd (simde__m128 a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_castps_pd(a); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return vreinterpretq_f64_f32(a); + #else + simde__m128d r; + simde_memcpy(&r, &a, sizeof(a)); + return r; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_castps_pd(a) simde_mm_castps_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_castps_si128 (simde__m128 a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_castps_si128(a); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + return simde__m128i_from_neon_i32(simde__m128_to_private(a).neon_i32); + #else + simde__m128i r; + simde_memcpy(&r, &a, sizeof(a)); + return r; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_castps_si128(a) simde_mm_castps_si128(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_castsi128_pd (simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_castsi128_pd(a); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return vreinterpretq_f64_s64(a); + #else + simde__m128d r; + simde_memcpy(&r, &a, sizeof(a)); + return r; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_castsi128_pd(a) simde_mm_castsi128_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_castsi128_ps (simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_castsi128_ps(a); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + return HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(float), a); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + return simde__m128_from_neon_i32(simde__m128i_to_private(a).neon_i32); + #else + simde__m128 r; + simde_memcpy(&r, &a, sizeof(a)); + return r; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_castsi128_ps(a) simde_mm_castsi128_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cmpeq_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpeq_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vceqq_s8(b_.neon_i8, a_.neon_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_eq(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i8 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed char), vec_cmpeq(a_.altivec_i8, b_.altivec_i8)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), (a_.i8 == b_.i8)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] == b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpeq_epi8(a, b) simde_mm_cmpeq_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cmpeq_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpeq_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vceqq_s16(b_.neon_i16, a_.neon_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_eq(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i16 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed short), vec_cmpeq(a_.altivec_i16, b_.altivec_i16)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = (a_.i16 == b_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] == b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpeq_epi16(a, b) simde_mm_cmpeq_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cmpeq_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpeq_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vceqq_s32(b_.neon_i32, a_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_eq(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed int), vec_cmpeq(a_.altivec_i32, b_.altivec_i32)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.i32 == b_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] == b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpeq_epi32(a, b) simde_mm_cmpeq_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpeq_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpeq_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vceqq_f64(b_.neon_f64, a_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_eq(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_cmpeq(a_.altivec_f64, b_.altivec_f64)); + #elif defined(SIMDE_MIPS_MSA_NATIVE) + r_.msa_i32 = __msa_addv_w(a_.msa_i32, b_.msa_i32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 == b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.u64[i] = (a_.f64[i] == b_.f64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpeq_pd(a, b) simde_mm_cmpeq_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpeq_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpeq_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_cmpeq_pd(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_cmpeq_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + r_.u64[0] = (a_.u64[0] == b_.u64[0]) ? ~UINT64_C(0) : 0; + r_.u64[1] = a_.u64[1]; + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpeq_sd(a, b) simde_mm_cmpeq_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpneq_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpneq_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u32 = vmvnq_u32(vreinterpretq_u32_u64(vceqq_f64(b_.neon_f64, a_.neon_f64))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_ne(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 != b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.u64[i] = (a_.f64[i] != b_.f64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpneq_pd(a, b) simde_mm_cmpneq_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpneq_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpneq_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_cmpneq_pd(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_cmpneq_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + r_.u64[0] = (a_.f64[0] != b_.f64[0]) ? ~UINT64_C(0) : UINT64_C(0); + r_.u64[1] = a_.u64[1]; + + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpneq_sd(a, b) simde_mm_cmpneq_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cmplt_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmplt_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcltq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i8 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed char),vec_cmplt(a_.altivec_i8, b_.altivec_i8)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_lt(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), (a_.i8 < b_.i8)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] < b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmplt_epi8(a, b) simde_mm_cmplt_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cmplt_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmplt_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vcltq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i16 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed short), vec_cmplt(a_.altivec_i16, b_.altivec_i16)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_lt(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.i16 < b_.i16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] < b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmplt_epi16(a, b) simde_mm_cmplt_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cmplt_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmplt_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcltq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed int), vec_cmplt(a_.altivec_i32, b_.altivec_i32)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_lt(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.i32 < b_.i32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] < b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmplt_epi32(a, b) simde_mm_cmplt_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmplt_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmplt_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcltq_f64(a_.neon_f64, b_.neon_f64); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_cmplt(a_.altivec_f64, b_.altivec_f64)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_lt(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 < b_.f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.u64[i] = (a_.f64[i] < b_.f64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmplt_pd(a, b) simde_mm_cmplt_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmplt_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmplt_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_cmplt_pd(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_cmplt_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + r_.u64[0] = (a_.f64[0] < b_.f64[0]) ? ~UINT64_C(0) : UINT64_C(0); + r_.u64[1] = a_.u64[1]; + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmplt_sd(a, b) simde_mm_cmplt_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmple_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmple_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 <= b_.f64)); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcleq_f64(a_.neon_f64, b_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_le(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_cmple(a_.altivec_f64, b_.altivec_f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.u64[i] = (a_.f64[i] <= b_.f64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmple_pd(a, b) simde_mm_cmple_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmple_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmple_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_cmple_pd(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_cmple_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + r_.u64[0] = (a_.f64[0] <= b_.f64[0]) ? ~UINT64_C(0) : UINT64_C(0); + r_.u64[1] = a_.u64[1]; + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmple_sd(a, b) simde_mm_cmple_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cmpgt_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpgt_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcgtq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_gt(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i8 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed char), vec_cmpgt(a_.altivec_i8, b_.altivec_i8)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), (a_.i8 > b_.i8)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] > b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpgt_epi8(a, b) simde_mm_cmpgt_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cmpgt_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpgt_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vcgtq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_gt(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i16 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed short), vec_cmpgt(a_.altivec_i16, b_.altivec_i16)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), (a_.i16 > b_.i16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] > b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpgt_epi16(a, b) simde_mm_cmpgt_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cmpgt_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpgt_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcgtq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_gt(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i32 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed int), vec_cmpgt(a_.altivec_i32, b_.altivec_i32)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), (a_.i32 > b_.i32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] > b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpgt_epi32(a, b) simde_mm_cmpgt_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpgt_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpgt_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 > b_.f64)); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcgtq_f64(a_.neon_f64, b_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_gt(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_cmpgt(a_.altivec_f64, b_.altivec_f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.u64[i] = (a_.f64[i] > b_.f64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpgt_pd(a, b) simde_mm_cmpgt_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpgt_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) && !defined(__PGI) + return _mm_cmpgt_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_cmpgt_pd(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_cmpgt_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + r_.u64[0] = (a_.f64[0] > b_.f64[0]) ? ~UINT64_C(0) : UINT64_C(0); + r_.u64[1] = a_.u64[1]; + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpgt_sd(a, b) simde_mm_cmpgt_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpge_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpge_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), (a_.f64 >= b_.f64)); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcgeq_f64(a_.neon_f64, b_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_ge(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_cmpge(a_.altivec_f64, b_.altivec_f64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.u64[i] = (a_.f64[i] >= b_.f64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpge_pd(a, b) simde_mm_cmpge_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpge_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) && !defined(__PGI) + return _mm_cmpge_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_cmpge_pd(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_cmpge_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + r_.u64[0] = (a_.f64[0] >= b_.f64[0]) ? ~UINT64_C(0) : UINT64_C(0); + r_.u64[1] = a_.u64[1]; + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpge_sd(a, b) simde_mm_cmpge_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpngt_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpngt_pd(a, b); + #else + return simde_mm_cmple_pd(a, b); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpngt_pd(a, b) simde_mm_cmpngt_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpngt_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) && !defined(__PGI) + return _mm_cmpngt_sd(a, b); + #else + return simde_mm_cmple_sd(a, b); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpngt_sd(a, b) simde_mm_cmpngt_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpnge_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpnge_pd(a, b); + #else + return simde_mm_cmplt_pd(a, b); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpnge_pd(a, b) simde_mm_cmpnge_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpnge_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) && !defined(__PGI) + return _mm_cmpnge_sd(a, b); + #else + return simde_mm_cmplt_sd(a, b); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpnge_sd(a, b) simde_mm_cmpnge_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpnlt_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpnlt_pd(a, b); + #else + return simde_mm_cmpge_pd(a, b); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpnlt_pd(a, b) simde_mm_cmpnlt_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpnlt_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpnlt_sd(a, b); + #else + return simde_mm_cmpge_sd(a, b); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpnlt_sd(a, b) simde_mm_cmpnlt_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpnle_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpnle_pd(a, b); + #else + return simde_mm_cmpgt_pd(a, b); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpnle_pd(a, b) simde_mm_cmpnle_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpnle_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpnle_sd(a, b); + #else + return simde_mm_cmpgt_sd(a, b); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpnle_sd(a, b) simde_mm_cmpnle_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpord_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpord_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + /* Note: NEON does not have ordered compare builtin + Need to compare a eq a and b eq b to check for NaN + Do AND of results to get final */ + uint64x2_t ceqaa = vceqq_f64(a_.neon_f64, a_.neon_f64); + uint64x2_t ceqbb = vceqq_f64(b_.neon_f64, b_.neon_f64); + r_.neon_u64 = vandq_u64(ceqaa, ceqbb); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_and(wasm_f64x2_eq(a_.wasm_v128, a_.wasm_v128), + wasm_f64x2_eq(b_.wasm_v128, b_.wasm_v128)); + #elif defined(simde_math_isnan) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.u64[i] = (!simde_math_isnan(a_.f64[i]) && !simde_math_isnan(b_.f64[i])) ? ~UINT64_C(0) : UINT64_C(0); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpord_pd(a, b) simde_mm_cmpord_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde_float64 +simde_mm_cvtsd_f64 (simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && !defined(__PGI) + return _mm_cvtsd_f64(a); + #else + simde__m128d_private a_ = simde__m128d_to_private(a); + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return HEDLEY_STATIC_CAST(simde_float64, vgetq_lane_f64(a_.neon_f64, 0)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return HEDLEY_STATIC_CAST(simde_float64, wasm_f64x2_extract_lane(a_.wasm_v128, 0)); + #else + return a_.f64[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtsd_f64(a) simde_mm_cvtsd_f64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpord_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpord_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_cmpord_pd(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_cmpord_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(simde_math_isnan) + r_.u64[0] = (!simde_math_isnan(a_.f64[0]) && !simde_math_isnan(b_.f64[0])) ? ~UINT64_C(0) : UINT64_C(0); + r_.u64[1] = a_.u64[1]; + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpord_sd(a, b) simde_mm_cmpord_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpunord_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpunord_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint64x2_t ceqaa = vceqq_f64(a_.neon_f64, a_.neon_f64); + uint64x2_t ceqbb = vceqq_f64(b_.neon_f64, b_.neon_f64); + r_.neon_u64 = vreinterpretq_u64_u32(vmvnq_u32(vreinterpretq_u32_u64(vandq_u64(ceqaa, ceqbb)))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_or(wasm_f64x2_ne(a_.wasm_v128, a_.wasm_v128), + wasm_f64x2_ne(b_.wasm_v128, b_.wasm_v128)); + #elif defined(simde_math_isnan) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.u64[i] = (simde_math_isnan(a_.f64[i]) || simde_math_isnan(b_.f64[i])) ? ~UINT64_C(0) : UINT64_C(0); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpunord_pd(a, b) simde_mm_cmpunord_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cmpunord_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cmpunord_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_cmpunord_pd(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_cmpunord_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(simde_math_isnan) + r_.u64[0] = (simde_math_isnan(a_.f64[0]) || simde_math_isnan(b_.f64[0])) ? ~UINT64_C(0) : UINT64_C(0); + r_.u64[1] = a_.u64[1]; + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cmpunord_sd(a, b) simde_mm_cmpunord_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cvtepi32_pd (simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cvtepi32_pd(a); + #else + simde__m128d_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_convert_low_i32x4(a_.wasm_v128); + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.f64, a_.m64_private[0].i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = (simde_float64) a_.i32[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtepi32_pd(a) simde_mm_cvtepi32_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cvtepi32_ps (simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cvtepi32_ps(a); + #else + simde__m128_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vcvtq_f32_s32(a_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_convert_i32x4(a_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + HEDLEY_DIAGNOSTIC_PUSH + #if HEDLEY_HAS_WARNING("-Wc11-extensions") + #pragma clang diagnostic ignored "-Wc11-extensions" + #endif + r_.altivec_f32 = vec_ctf(a_.altivec_i32, 0); + HEDLEY_DIAGNOSTIC_POP + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.f32, a_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = (simde_float32) a_.i32[i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtepi32_ps(a) simde_mm_cvtepi32_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cvtpd_pi32 (simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvtpd_pi32(a); + #else + simde__m64_private r_; + simde__m128d_private a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + simde_float64 v = simde_math_round(a_.f64[i]); + #if defined(SIMDE_FAST_CONVERSION_RANGE) + r_.i32[i] = SIMDE_CONVERT_FTOI(int32_t, v); + #else + r_.i32[i] = ((v > HEDLEY_STATIC_CAST(simde_float64, INT32_MIN)) && (v < HEDLEY_STATIC_CAST(simde_float64, INT32_MAX))) ? + SIMDE_CONVERT_FTOI(int32_t, v) : INT32_MIN; + #endif + } + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtpd_pi32(a) simde_mm_cvtpd_pi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtpd_epi32 (simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && !defined(SIMDE_BUG_PGI_30107) + return _mm_cvtpd_epi32(a); + #else + simde__m128i_private r_; + + r_.m64[0] = simde_mm_cvtpd_pi32(a); + r_.m64[1] = simde_mm_setzero_si64(); + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtpd_epi32(a) simde_mm_cvtpd_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cvtpd_ps (simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cvtpd_ps(a); + #else + simde__m128_private r_; + simde__m128d_private a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f32 = vcombine_f32(vcvt_f32_f64(a_.neon_f64), vdup_n_f32(0.0f)); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + r_.altivec_f32 = vec_float2(a_.altivec_f64, vec_splats(0.0)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f32x4_demote_f64x2_zero(a_.wasm_v128); + #elif HEDLEY_HAS_BUILTIN(__builtin_shufflevector) && HEDLEY_HAS_BUILTIN(__builtin_convertvector) + float __attribute__((__vector_size__(8))) z = { 0.0f, 0.0f }; + r_.f32 = + __builtin_shufflevector( + __builtin_convertvector(__builtin_shufflevector(a_.f64, a_.f64, 0, 1), __typeof__(z)), z, + 0, 1, 2, 3 + ); + #else + r_.f32[0] = HEDLEY_STATIC_CAST(simde_float32, a_.f64[0]); + r_.f32[1] = HEDLEY_STATIC_CAST(simde_float32, a_.f64[1]); + r_.f32[2] = SIMDE_FLOAT32_C(0.0); + r_.f32[3] = SIMDE_FLOAT32_C(0.0); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtpd_ps(a) simde_mm_cvtpd_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cvtpi32_pd (simde__m64 a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvtpi32_pd(a); + #else + simde__m128d_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.f64, a_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = (simde_float64) a_.i32[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtpi32_pd(a) simde_mm_cvtpi32_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtps_epi32 (simde__m128 a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cvtps_epi32(a); + #else + simde__m128i_private r_; + simde__m128_private a_; + + #if defined(SIMDE_ARM_NEON_A32V8_NATIVE) && defined(SIMDE_FAST_CONVERSION_RANGE) && defined(SIMDE_FAST_ROUND_TIES) && !defined(SIMDE_BUG_GCC_95399) + a_ = simde__m128_to_private(a); + r_.neon_i32 = vcvtnq_s32_f32(a_.neon_f32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) && defined(SIMDE_FAST_CONVERSION_RANGE) && defined(SIMDE_FAST_ROUND_TIES) + a_ = simde__m128_to_private(a); + HEDLEY_DIAGNOSTIC_PUSH + SIMDE_DIAGNOSTIC_DISABLE_C11_EXTENSIONS_ + SIMDE_DIAGNOSTIC_DISABLE_VECTOR_CONVERSION_ + r_.altivec_i32 = vec_cts(a_.altivec_f32, 1); + HEDLEY_DIAGNOSTIC_POP + #elif defined(SIMDE_WASM_SIMD128_NATIVE) && defined(SIMDE_FAST_CONVERSION_RANGE) && defined(SIMDE_FAST_ROUND_TIES) + a_ = simde__m128_to_private(a); + r_.wasm_v128 = wasm_i32x4_trunc_sat_f32x4(a_.wasm_v128); + #else + a_ = simde__m128_to_private(simde_x_mm_round_ps(a, SIMDE_MM_FROUND_TO_NEAREST_INT, 1)); + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + simde_float32 v = simde_math_roundf(a_.f32[i]); + #if defined(SIMDE_FAST_CONVERSION_RANGE) + r_.i32[i] = SIMDE_CONVERT_FTOI(int32_t, v); + #else + r_.i32[i] = ((v > HEDLEY_STATIC_CAST(simde_float32, INT32_MIN)) && (v < HEDLEY_STATIC_CAST(simde_float32, INT32_MAX))) ? + SIMDE_CONVERT_FTOI(int32_t, v) : INT32_MIN; + #endif + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtps_epi32(a) simde_mm_cvtps_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cvtps_pd (simde__m128 a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cvtps_pd(a); + #else + simde__m128d_private r_; + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_promote_low_f32x4(a_.wasm_v128); + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.f64, a_.m64_private[0].f32); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vcvt_f64_f32(vget_low_f32(a_.neon_f32)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f32[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtps_pd(a) simde_mm_cvtps_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm_cvtsd_si32 (simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cvtsd_si32(a); + #else + simde__m128d_private a_ = simde__m128d_to_private(a); + + simde_float64 v = simde_math_round(a_.f64[0]); + #if defined(SIMDE_FAST_CONVERSION_RANGE) + return SIMDE_CONVERT_FTOI(int32_t, v); + #else + return ((v > HEDLEY_STATIC_CAST(simde_float64, INT32_MIN)) && (v < HEDLEY_STATIC_CAST(simde_float64, INT32_MAX))) ? + SIMDE_CONVERT_FTOI(int32_t, v) : INT32_MIN; + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtsd_si32(a) simde_mm_cvtsd_si32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int64_t +simde_mm_cvtsd_si64 (simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_ARCH_AMD64) + #if defined(__PGI) + return _mm_cvtsd_si64x(a); + #else + return _mm_cvtsd_si64(a); + #endif + #else + simde__m128d_private a_ = simde__m128d_to_private(a); + return SIMDE_CONVERT_FTOI(int64_t, simde_math_round(a_.f64[0])); + #endif +} +#define simde_mm_cvtsd_si64x(a) simde_mm_cvtsd_si64(a) +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) + #define _mm_cvtsd_si64(a) simde_mm_cvtsd_si64(a) + #define _mm_cvtsd_si64x(a) simde_mm_cvtsd_si64x(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cvtsd_ss (simde__m128 a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cvtsd_ss(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + simde__m128d_private b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f32 = vsetq_lane_f32(vcvtxd_f32_f64(vgetq_lane_f64(b_.neon_f64, 0)), a_.neon_f32, 0); + #else + r_.f32[0] = HEDLEY_STATIC_CAST(simde_float32, b_.f64[0]); + + SIMDE_VECTORIZE + for (size_t i = 1 ; i < (sizeof(r_) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i]; + } + #endif + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtsd_ss(a, b) simde_mm_cvtsd_ss(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int16_t +simde_x_mm_cvtsi128_si16 (simde__m128i a) { + simde__m128i_private + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + return vgetq_lane_s16(a_.neon_i16, 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return HEDLEY_STATIC_CAST(int16_t, wasm_i16x8_extract_lane(a_.wasm_v128, 0)); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + #if defined(SIMDE_BUG_GCC_95227) + (void) a_; + #endif + return vec_extract(a_.altivec_i16, 0); + #else + return a_.i16[0]; + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm_cvtsi128_si32 (simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cvtsi128_si32(a); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + return vgetq_lane_s32(a_.neon_i32, 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return HEDLEY_STATIC_CAST(int32_t, wasm_i32x4_extract_lane(a_.wasm_v128, 0)); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + #if defined(SIMDE_BUG_GCC_95227) + (void) a_; + #endif + return vec_extract(a_.altivec_i32, 0); + #else + return a_.i32[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtsi128_si32(a) simde_mm_cvtsi128_si32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int64_t +simde_mm_cvtsi128_si64 (simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_ARCH_AMD64) + #if defined(__PGI) + return _mm_cvtsi128_si64x(a); + #else + return _mm_cvtsi128_si64(a); + #endif + #else + simde__m128i_private a_ = simde__m128i_to_private(a); + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) && !defined(HEDLEY_IBM_VERSION) + return vec_extract(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed long long), a_.i64), 0); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + return vgetq_lane_s64(a_.neon_i64, 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return HEDLEY_STATIC_CAST(int64_t, wasm_i64x2_extract_lane(a_.wasm_v128, 0)); + #endif + return a_.i64[0]; + #endif +} +#define simde_mm_cvtsi128_si64x(a) simde_mm_cvtsi128_si64(a) +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) + #define _mm_cvtsi128_si64(a) simde_mm_cvtsi128_si64(a) + #define _mm_cvtsi128_si64x(a) simde_mm_cvtsi128_si64x(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cvtsi32_sd (simde__m128d a, int32_t b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cvtsi32_sd(a, b); + #else + simde__m128d_private r_; + simde__m128d_private a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vsetq_lane_f64(HEDLEY_STATIC_CAST(float64_t, b), a_.neon_f64, 0); + #else + r_.f64[0] = HEDLEY_STATIC_CAST(simde_float64, b); + r_.i64[1] = a_.i64[1]; + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtsi32_sd(a, b) simde_mm_cvtsi32_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_cvtsi16_si128 (int16_t a) { + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vsetq_lane_s16(a, vdupq_n_s16(0), 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_make(a, 0, 0, 0, 0, 0, 0, 0); + #else + r_.i16[0] = a; + r_.i16[1] = 0; + r_.i16[2] = 0; + r_.i16[3] = 0; + r_.i16[4] = 0; + r_.i16[5] = 0; + r_.i16[6] = 0; + r_.i16[7] = 0; + #endif + + return simde__m128i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtsi32_si128 (int32_t a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cvtsi32_si128(a); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vsetq_lane_s32(a, vdupq_n_s32(0), 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_make(a, 0, 0, 0); + #else + r_.i32[0] = a; + r_.i32[1] = 0; + r_.i32[2] = 0; + r_.i32[3] = 0; + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtsi32_si128(a) simde_mm_cvtsi32_si128(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cvtsi64_sd (simde__m128d a, int64_t b) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_ARCH_AMD64) + #if !defined(__PGI) + return _mm_cvtsi64_sd(a, b); + #else + return _mm_cvtsi64x_sd(a, b); + #endif + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vsetq_lane_f64(HEDLEY_STATIC_CAST(float64_t, b), a_.neon_f64, 0); + #else + r_.f64[0] = HEDLEY_STATIC_CAST(simde_float64, b); + r_.f64[1] = a_.f64[1]; + #endif + + return simde__m128d_from_private(r_); + #endif +} +#define simde_mm_cvtsi64x_sd(a, b) simde_mm_cvtsi64_sd(a, b) +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) + #define _mm_cvtsi64_sd(a, b) simde_mm_cvtsi64_sd(a, b) + #define _mm_cvtsi64x_sd(a, b) simde_mm_cvtsi64x_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtsi64_si128 (int64_t a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_ARCH_AMD64) + #if !defined(__PGI) + return _mm_cvtsi64_si128(a); + #else + return _mm_cvtsi64x_si128(a); + #endif + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vsetq_lane_s64(a, vdupq_n_s64(0), 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_make(a, 0); + #else + r_.i64[0] = a; + r_.i64[1] = 0; + #endif + + return simde__m128i_from_private(r_); + #endif +} +#define simde_mm_cvtsi64x_si128(a) simde_mm_cvtsi64_si128(a) +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) + #define _mm_cvtsi64_si128(a) simde_mm_cvtsi64_si128(a) + #define _mm_cvtsi64x_si128(a) simde_mm_cvtsi64x_si128(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cvtss_sd (simde__m128d a, simde__m128 b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cvtss_sd(a, b); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + float64x2_t temp = vcvt_f64_f32(vset_lane_f32(vgetq_lane_f32(simde__m128_to_private(b).neon_f32, 0), vdup_n_f32(0), 0)); + return vsetq_lane_f64(vgetq_lane_f64(simde__m128d_to_private(a).neon_f64, 1), temp, 1); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a); + simde__m128_private b_ = simde__m128_to_private(b); + + a_.f64[0] = HEDLEY_STATIC_CAST(simde_float64, b_.f32[0]); + + return simde__m128d_from_private(a_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvtss_sd(a, b) simde_mm_cvtss_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_cvttpd_pi32 (simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_cvttpd_pi32(a); + #else + simde__m64_private r_; + simde__m128d_private a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_CONVERT_VECTOR_) && defined(SIMDE_FAST_CONVERSION_RANGE) + SIMDE_CONVERT_VECTOR_(r_.i32, a_.f64); + #else + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + simde_float64 v = a_.f64[i]; + #if defined(SIMDE_FAST_CONVERSION_RANGE) + r_.i32[i] = SIMDE_CONVERT_FTOI(int32_t, v); + #else + r_.i32[i] = ((v > HEDLEY_STATIC_CAST(simde_float64, INT32_MIN)) && (v < HEDLEY_STATIC_CAST(simde_float64, INT32_MAX))) ? + SIMDE_CONVERT_FTOI(int32_t, v) : INT32_MIN; + #endif + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvttpd_pi32(a) simde_mm_cvttpd_pi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvttpd_epi32 (simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cvttpd_epi32(a); + #else + simde__m128i_private r_; + + r_.m64[0] = simde_mm_cvttpd_pi32(a); + r_.m64[1] = simde_mm_setzero_si64(); + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvttpd_epi32(a) simde_mm_cvttpd_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvttps_epi32 (simde__m128 a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cvttps_epi32(a); + #else + simde__m128i_private r_; + simde__m128_private a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vcvtq_s32_f32(a_.neon_f32); + + #if !defined(SIMDE_FAST_CONVERSION_RANGE) || !defined(SIMDE_FAST_NANS) + /* Values below INT32_MIN saturate anyways, so we don't need to + * test for that. */ + #if !defined(SIMDE_FAST_CONVERSION_RANGE) && !defined(SIMDE_FAST_NANS) + uint32x4_t valid_input = + vandq_u32( + vcltq_f32(a_.neon_f32, vdupq_n_f32(SIMDE_FLOAT32_C(2147483648.0))), + vceqq_f32(a_.neon_f32, a_.neon_f32) + ); + #elif !defined(SIMDE_FAST_CONVERSION_RANGE) + uint32x4_t valid_input = vcltq_f32(a_.neon_f32, vdupq_n_f32(SIMDE_FLOAT32_C(2147483648.0))); + #elif !defined(SIMDE_FAST_NANS) + uint32x4_t valid_input = vceqq_f32(a_.neon_f32, a_.neon_f32); + #endif + + r_.neon_i32 = vbslq_s32(valid_input, r_.neon_i32, vdupq_n_s32(INT32_MIN)); + #endif + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_trunc_sat_f32x4(a_.wasm_v128); + + #if !defined(SIMDE_FAST_CONVERSION_RANGE) || !defined(SIMDE_FAST_NANS) + #if !defined(SIMDE_FAST_CONVERSION_RANGE) && !defined(SIMDE_FAST_NANS) + v128_t valid_input = + wasm_v128_and( + wasm_f32x4_lt(a_.wasm_v128, wasm_f32x4_splat(SIMDE_FLOAT32_C(2147483648.0))), + wasm_f32x4_eq(a_.wasm_v128, a_.wasm_v128) + ); + #elif !defined(SIMDE_FAST_CONVERSION_RANGE) + v128_t valid_input = wasm_f32x4_lt(a_.wasm_v128, wasm_f32x4_splat(SIMDE_FLOAT32_C(2147483648.0))); + #elif !defined(SIMDE_FAST_NANS) + v128_t valid_input = wasm_f32x4_eq(a_.wasm_v128, a_.wasm_v128); + #endif + + r_.wasm_v128 = wasm_v128_bitselect(r_.wasm_v128, wasm_i32x4_splat(INT32_MIN), valid_input); + #endif + #elif defined(SIMDE_CONVERT_VECTOR_) && !defined(SIMDE_ARCH_POWER) + SIMDE_CONVERT_VECTOR_(r_.i32, a_.f32); + + #if !defined(SIMDE_FAST_CONVERSION_RANGE) || !defined(SIMDE_FAST_NANS) + #if !defined(SIMDE_FAST_CONVERSION_RANGE) + static const simde_float32 SIMDE_VECTOR(16) first_too_high = { SIMDE_FLOAT32_C(2147483648.0), SIMDE_FLOAT32_C(2147483648.0), SIMDE_FLOAT32_C(2147483648.0), SIMDE_FLOAT32_C(2147483648.0) }; + + __typeof__(r_.i32) valid_input = + HEDLEY_REINTERPRET_CAST( + __typeof__(r_.i32), + (a_.f32 < first_too_high) & (a_.f32 >= -first_too_high) + ); + #elif !defined(SIMDE_FAST_NANS) + __typeof__(r_.i32) valid_input = HEDLEY_REINTERPRET_CAST( __typeof__(valid_input), a_.f32 == a_.f32); + #endif + + __typeof__(r_.i32) invalid_output = { INT32_MIN, INT32_MIN, INT32_MIN, INT32_MIN }; + r_.i32 = (r_.i32 & valid_input) | (invalid_output & ~valid_input); + #endif + #else + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + simde_float32 v = a_.f32[i]; + #if defined(SIMDE_FAST_CONVERSION_RANGE) && defined(SIMDE_FAST_NANS) + r_.i32[i] = SIMDE_CONVERT_FTOI(int32_t, v); + #else + r_.i32[i] = ((v > HEDLEY_STATIC_CAST(simde_float32, INT32_MIN)) && (v < HEDLEY_STATIC_CAST(simde_float32, INT32_MAX))) ? + SIMDE_CONVERT_FTOI(int32_t, v) : INT32_MIN; + #endif + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvttps_epi32(a) simde_mm_cvttps_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm_cvttsd_si32 (simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_cvttsd_si32(a); + #else + simde__m128d_private a_ = simde__m128d_to_private(a); + simde_float64 v = a_.f64[0]; + #if defined(SIMDE_FAST_CONVERSION_RANGE) + return SIMDE_CONVERT_FTOI(int32_t, v); + #else + return ((v > HEDLEY_STATIC_CAST(simde_float64, INT32_MIN)) && (v < HEDLEY_STATIC_CAST(simde_float64, INT32_MAX))) ? + SIMDE_CONVERT_FTOI(int32_t, v) : INT32_MIN; + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_cvttsd_si32(a) simde_mm_cvttsd_si32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int64_t +simde_mm_cvttsd_si64 (simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_ARCH_AMD64) + #if !defined(__PGI) + return _mm_cvttsd_si64(a); + #else + return _mm_cvttsd_si64x(a); + #endif + #else + simde__m128d_private a_ = simde__m128d_to_private(a); + return SIMDE_CONVERT_FTOI(int64_t, a_.f64[0]); + #endif +} +#define simde_mm_cvttsd_si64x(a) simde_mm_cvttsd_si64(a) +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) + #define _mm_cvttsd_si64(a) simde_mm_cvttsd_si64(a) + #define _mm_cvttsd_si64x(a) simde_mm_cvttsd_si64x(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_div_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_div_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = a_.f64 / b_.f64; + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vdivq_f64(a_.neon_f64, b_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_div(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[i] / b_.f64[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_div_pd(a, b) simde_mm_div_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_div_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_div_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_div_pd(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_div_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + float64x2_t temp = vdivq_f64(a_.neon_f64, b_.neon_f64); + r_.neon_f64 = vsetq_lane_f64(vgetq_lane(a_.neon_f64, 1), temp, 1); + #else + r_.f64[0] = a_.f64[0] / b_.f64[0]; + r_.f64[1] = a_.f64[1]; + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_div_sd(a, b) simde_mm_div_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm_extract_epi16 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + uint16_t r; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + #if defined(SIMDE_BUG_GCC_95227) + (void) a_; + (void) imm8; + #endif + r = HEDLEY_STATIC_CAST(uint16_t, vec_extract(a_.altivec_i16, imm8)); + #else + r = a_.u16[imm8 & 7]; + #endif + + return HEDLEY_STATIC_CAST(int32_t, r); +} +#if defined(SIMDE_X86_SSE2_NATIVE) && (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(4,6,0)) + #define simde_mm_extract_epi16(a, imm8) _mm_extract_epi16(a, imm8) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #define simde_mm_extract_epi16(a, imm8) (HEDLEY_STATIC_CAST(int32_t, vgetq_lane_s16(simde__m128i_to_private(a).neon_i16, (imm8))) & (INT32_C(0x0000ffff))) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_extract_epi16(a, imm8) HEDLEY_STATIC_CAST(int32_t, wasm_u16x8_extract_lane(simde__m128i_to_wasm_v128((a)), (imm8) & 7)) +#endif +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_extract_epi16(a, imm8) simde_mm_extract_epi16(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_insert_epi16 (simde__m128i a, int16_t i, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + simde__m128i_private a_ = simde__m128i_to_private(a); + a_.i16[imm8 & 7] = i; + return simde__m128i_from_private(a_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) && !defined(__PGI) + #define simde_mm_insert_epi16(a, i, imm8) _mm_insert_epi16((a), (i), (imm8)) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #define simde_mm_insert_epi16(a, i, imm8) simde__m128i_from_neon_i16(vsetq_lane_s16((i), simde__m128i_to_neon_i16(a), (imm8))) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_insert_epi16(a, i, imm8) wasm_i16x8_replace_lane(simde__m128i_to_wasm_v128((a)), (imm8) & 7, (i)) +#endif +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_insert_epi16(a, i, imm8) simde_mm_insert_epi16(a, i, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_load_pd (simde_float64 const mem_addr[HEDLEY_ARRAY_PARAM(2)]) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_load_pd(mem_addr); + #else + simde__m128d_private r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vld1q_f64(mem_addr); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vld1q_u32(HEDLEY_REINTERPRET_CAST(uint32_t const*, mem_addr)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_load(mem_addr); + #else + simde_memcpy(&r_, SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m128d), sizeof(r_)); + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_load_pd(mem_addr) simde_mm_load_pd(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_load1_pd (simde_float64 const* mem_addr) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_load1_pd(mem_addr); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return simde__m128d_from_neon_f64(vld1q_dup_f64(mem_addr)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return simde__m128d_from_wasm_v128(wasm_v128_load64_splat(mem_addr)); + #else + return simde_mm_set1_pd(*mem_addr); + #endif +} +#define simde_mm_load_pd1(mem_addr) simde_mm_load1_pd(mem_addr) +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_load_pd1(mem_addr) simde_mm_load1_pd(mem_addr) + #define _mm_load1_pd(mem_addr) simde_mm_load1_pd(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_load_sd (simde_float64 const* mem_addr) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_load_sd(mem_addr); + #else + simde__m128d_private r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vsetq_lane_f64(*mem_addr, vdupq_n_f64(0), 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_load64_zero(HEDLEY_REINTERPRET_CAST(const void*, mem_addr)); + #else + r_.f64[0] = *mem_addr; + r_.u64[1] = UINT64_C(0); + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_load_sd(mem_addr) simde_mm_load_sd(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_load_si128 (simde__m128i const* mem_addr) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_load_si128(HEDLEY_REINTERPRET_CAST(__m128i const*, mem_addr)); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + return vld1q_s64(HEDLEY_REINTERPRET_CAST(int64_t const*, mem_addr)); + #else + simde__m128i_private r_; + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i32 = vec_ld(0, HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(int) const*, mem_addr)); + #else + simde_memcpy(&r_, SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m128i), sizeof(simde__m128i)); + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_load_si128(mem_addr) simde_mm_load_si128(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_loadh_pd (simde__m128d a, simde_float64 const* mem_addr) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_loadh_pd(a, mem_addr); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vcombine_f64(vget_low_f64(a_.neon_f64), vld1_f64(HEDLEY_REINTERPRET_CAST(const float64_t*, mem_addr))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_load64_lane(HEDLEY_REINTERPRET_CAST(const void*, mem_addr), a_.wasm_v128, 1); + #else + simde_float64 t; + + simde_memcpy(&t, mem_addr, sizeof(t)); + r_.f64[0] = a_.f64[0]; + r_.f64[1] = t; + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_loadh_pd(a, mem_addr) simde_mm_loadh_pd(a, mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_loadl_epi64 (simde__m128i const* mem_addr) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_loadl_epi64(mem_addr); + #else + simde__m128i_private r_; + + int64_t value; + simde_memcpy(&value, mem_addr, sizeof(value)); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vcombine_s64(vld1_s64(HEDLEY_REINTERPRET_CAST(int64_t const *, mem_addr)), vdup_n_s64(0)); + #else + r_.i64[0] = value; + r_.i64[1] = 0; + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_loadl_epi64(mem_addr) simde_mm_loadl_epi64(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_loadl_pd (simde__m128d a, simde_float64 const* mem_addr) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_loadl_pd(a, mem_addr); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vcombine_f64(vld1_f64( + HEDLEY_REINTERPRET_CAST(const float64_t*, mem_addr)), vget_high_f64(a_.neon_f64)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_load64_lane(HEDLEY_REINTERPRET_CAST(const void*, mem_addr), a_.wasm_v128, 0); + #else + r_.f64[0] = *mem_addr; + r_.u64[1] = a_.u64[1]; + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_loadl_pd(a, mem_addr) simde_mm_loadl_pd(a, mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_loadr_pd (simde_float64 const mem_addr[HEDLEY_ARRAY_PARAM(2)]) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_loadr_pd(mem_addr); + #else + simde__m128d_private + r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vld1q_f64(mem_addr); + r_.neon_f64 = vextq_f64(r_.neon_f64, r_.neon_f64, 1); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vld1q_s64(HEDLEY_REINTERPRET_CAST(int64_t const *, mem_addr)); + r_.neon_i64 = vextq_s64(r_.neon_i64, r_.neon_i64, 1); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t tmp = wasm_v128_load(mem_addr); + r_.wasm_v128 = wasm_i64x2_shuffle(tmp, tmp, 1, 0); + #else + r_.f64[0] = mem_addr[1]; + r_.f64[1] = mem_addr[0]; + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_loadr_pd(mem_addr) simde_mm_loadr_pd(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_loadu_pd (simde_float64 const mem_addr[HEDLEY_ARRAY_PARAM(2)]) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_loadu_pd(mem_addr); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return vld1q_f64(mem_addr); + #else + simde__m128d_private r_; + + simde_memcpy(&r_, mem_addr, sizeof(r_)); + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_loadu_pd(mem_addr) simde_mm_loadu_pd(mem_addr) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) \ + && !defined(SIMDE_BUG_GCC_95483) && !defined(SIMDE_BUG_CLANG_REV_344862) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define simde_mm_loadu_epi8(mem_addr) _mm_loadu_epi8(mem_addr) +#else +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_loadu_epi8(void const * mem_addr) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_loadu_si128(SIMDE_ALIGN_CAST(__m128i const *, mem_addr)); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vld1q_s8(HEDLEY_REINTERPRET_CAST(int8_t const*, mem_addr)); + #else + simde_memcpy(&r_, mem_addr, sizeof(r_)); + #endif + + return simde__m128i_from_private(r_); + #endif +} +#endif +#define simde_x_mm_loadu_epi8(mem_addr) simde_mm_loadu_epi8(mem_addr) +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && (defined(SIMDE_BUG_GCC_95483) || defined(SIMDE_BUG_CLANG_REV_344862))) + #undef _mm_loadu_epi8 + #define _mm_loadu_epi8(a) simde_mm_loadu_epi8(a) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && defined(SIMDE_X86_AVX512BW_NATIVE) \ + && !defined(SIMDE_BUG_GCC_95483) && !defined(SIMDE_BUG_CLANG_REV_344862) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define simde_mm_loadu_epi16(mem_addr) _mm_loadu_epi16(mem_addr) +#else +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_loadu_epi16(void const * mem_addr) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_loadu_si128(SIMDE_ALIGN_CAST(__m128i const *, mem_addr)); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vreinterpretq_s16_s8(vld1q_s8(HEDLEY_REINTERPRET_CAST(int8_t const*, mem_addr))); + #else + simde_memcpy(&r_, mem_addr, sizeof(r_)); + #endif + + return simde__m128i_from_private(r_); + #endif +} +#endif +#define simde_x_mm_loadu_epi16(mem_addr) simde_mm_loadu_epi16(mem_addr) +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || defined(SIMDE_X86_AVX512BW_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && (defined(SIMDE_BUG_GCC_95483) || defined(SIMDE_BUG_CLANG_REV_344862))) + #undef _mm_loadu_epi16 + #define _mm_loadu_epi16(a) simde_mm_loadu_epi16(a) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && !defined(SIMDE_BUG_GCC_95483) \ + && !defined(SIMDE_BUG_CLANG_REV_344862) && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define simde_mm_loadu_epi32(mem_addr) _mm_loadu_epi32(mem_addr) +#else +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_loadu_epi32(void const * mem_addr) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_loadu_si128(SIMDE_ALIGN_CAST(__m128i const *, mem_addr)); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vreinterpretq_s32_s8(vld1q_s8(HEDLEY_REINTERPRET_CAST(int8_t const*, mem_addr))); + #else + simde_memcpy(&r_, mem_addr, sizeof(r_)); + #endif + + return simde__m128i_from_private(r_); + #endif +} +#endif +#define simde_x_mm_loadu_epi32(mem_addr) simde_mm_loadu_epi32(mem_addr) +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && (defined(SIMDE_BUG_GCC_95483) || defined(SIMDE_BUG_CLANG_REV_344862))) + #undef _mm_loadu_epi32 + #define _mm_loadu_epi32(a) simde_mm_loadu_epi32(a) +#endif + +#if defined(SIMDE_X86_AVX512VL_NATIVE) && !defined(SIMDE_BUG_GCC_95483) \ + && !defined(SIMDE_BUG_CLANG_REV_344862) \ + && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define simde_mm_loadu_epi64(mem_addr) _mm_loadu_epi64(mem_addr) +#else +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_loadu_epi64(void const * mem_addr) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_loadu_si128(SIMDE_ALIGN_CAST(__m128i const *, mem_addr)); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vreinterpretq_s64_s8(vld1q_s8(HEDLEY_REINTERPRET_CAST(int8_t const*, mem_addr))); + #else + simde_memcpy(&r_, mem_addr, sizeof(r_)); + #endif + + return simde__m128i_from_private(r_); + #endif +} +#endif +#define simde_x_mm_loadu_epi64(mem_addr) simde_mm_loadu_epi64(mem_addr) +#if defined(SIMDE_X86_AVX512VL_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && (defined(SIMDE_BUG_GCC_95483) || defined(SIMDE_BUG_CLANG_REV_344862))) + #undef _mm_loadu_epi64 + #define _mm_loadu_epi64(a) simde_mm_loadu_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_loadu_si128 (void const* mem_addr) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_loadu_si128(HEDLEY_STATIC_CAST(__m128i const*, mem_addr)); + #else + simde__m128i_private r_; + + #if HEDLEY_GNUC_HAS_ATTRIBUTE(may_alias,3,3,0) + HEDLEY_DIAGNOSTIC_PUSH + SIMDE_DIAGNOSTIC_DISABLE_PACKED_ + struct simde_mm_loadu_si128_s { + __typeof__(r_) v; + } __attribute__((__packed__, __may_alias__)); + r_ = HEDLEY_REINTERPRET_CAST(const struct simde_mm_loadu_si128_s *, mem_addr)->v; + HEDLEY_DIAGNOSTIC_POP + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vld1q_s8(HEDLEY_REINTERPRET_CAST(int8_t const*, mem_addr)); + #else + simde_memcpy(&r_, mem_addr, sizeof(r_)); + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_loadu_si128(mem_addr) simde_mm_loadu_si128(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_madd_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_madd_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + int32x4_t pl = vmull_s16(vget_low_s16(a_.neon_i16), vget_low_s16(b_.neon_i16)); + int32x4_t ph = vmull_high_s16(a_.neon_i16, b_.neon_i16); + r_.neon_i32 = vpaddq_s32(pl, ph); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int32x4_t pl = vmull_s16(vget_low_s16(a_.neon_i16), vget_low_s16(b_.neon_i16)); + int32x4_t ph = vmull_s16(vget_high_s16(a_.neon_i16), vget_high_s16(b_.neon_i16)); + int32x2_t rl = vpadd_s32(vget_low_s32(pl), vget_high_s32(pl)); + int32x2_t rh = vpadd_s32(vget_low_s32(ph), vget_high_s32(ph)); + r_.neon_i32 = vcombine_s32(rl, rh); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i32 = vec_msum(a_.altivec_i16, b_.altivec_i16, vec_splats(0)); + #elif defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i32 = vec_mule(a_.altivec_i16, b_.altivec_i16) + vec_mulo(a_.altivec_i16, b_.altivec_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_dot_i16x8(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_CONVERT_VECTOR_) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) + int32_t SIMDE_VECTOR(32) a32, b32, p32; + SIMDE_CONVERT_VECTOR_(a32, a_.i16); + SIMDE_CONVERT_VECTOR_(b32, b_.i16); + p32 = a32 * b32; + r_.i32 = + __builtin_shufflevector(p32, p32, 0, 2, 4, 6) + + __builtin_shufflevector(p32, p32, 1, 3, 5, 7); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_) / sizeof(r_.i16[0])) ; i += 2) { + r_.i32[i / 2] = (a_.i16[i] * b_.i16[i]) + (a_.i16[i + 1] * b_.i16[i + 1]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_madd_epi16(a, b) simde_mm_madd_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_maskmoveu_si128 (simde__m128i a, simde__m128i mask, int8_t mem_addr[HEDLEY_ARRAY_PARAM(16)]) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_maskmoveu_si128(a, mask, HEDLEY_REINTERPRET_CAST(char*, mem_addr)); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + mask_ = simde__m128i_to_private(mask); + + for (size_t i = 0 ; i < (sizeof(a_.i8) / sizeof(a_.i8[0])) ; i++) { + if (mask_.u8[i] & 0x80) { + mem_addr[i] = a_.i8[i]; + } + } + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_maskmoveu_si128(a, mask, mem_addr) simde_mm_maskmoveu_si128((a), (mask), SIMDE_CHECKED_REINTERPRET_CAST(int8_t*, char*, (mem_addr))) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm_movemask_epi8 (simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && !defined(__INTEL_COMPILER) + /* ICC has trouble with _mm_movemask_epi8 at -O2 and above: */ + return _mm_movemask_epi8(a); + #else + int32_t r = 0; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + /* https://github.com/WebAssembly/simd/pull/201#issue-380682845 */ + static const uint8_t md[16] = { + 1 << 0, 1 << 1, 1 << 2, 1 << 3, + 1 << 4, 1 << 5, 1 << 6, 1 << 7, + 1 << 0, 1 << 1, 1 << 2, 1 << 3, + 1 << 4, 1 << 5, 1 << 6, 1 << 7, + }; + + /* Extend sign bit over entire lane */ + uint8x16_t extended = vreinterpretq_u8_s8(vshrq_n_s8(a_.neon_i8, 7)); + /* Clear all but the bit we're interested in. */ + uint8x16_t masked = vandq_u8(vld1q_u8(md), extended); + /* Alternate bytes from low half and high half */ + uint8x8x2_t tmp = vzip_u8(vget_low_u8(masked), vget_high_u8(masked)); + uint16x8_t x = vreinterpretq_u16_u8(vcombine_u8(tmp.val[0], tmp.val[1])); + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r = vaddvq_u16(x); + #else + uint64x2_t t64 = vpaddlq_u32(vpaddlq_u16(x)); + r = + HEDLEY_STATIC_CAST(int32_t, vgetq_lane_u64(t64, 0)) + + HEDLEY_STATIC_CAST(int32_t, vgetq_lane_u64(t64, 1)); + #endif + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) && !defined(HEDLEY_IBM_VERSION) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + static const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) perm = { 120, 112, 104, 96, 88, 80, 72, 64, 56, 48, 40, 32, 24, 16, 8, 0 }; + r = HEDLEY_STATIC_CAST(int32_t, vec_extract(vec_vbpermq(a_.altivec_u8, perm), 1)); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) && !defined(HEDLEY_IBM_VERSION) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_BIG) + static const SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) perm = { 120, 112, 104, 96, 88, 80, 72, 64, 56, 48, 40, 32, 24, 16, 8, 0 }; + r = HEDLEY_STATIC_CAST(int32_t, vec_extract(vec_vbpermq(a_.altivec_u8, perm), 14)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r = HEDLEY_STATIC_CAST(int32_t, wasm_i8x16_bitmask(a_.wasm_v128)); + #else + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u8) / sizeof(a_.u8[0])) ; i++) { + r |= (a_.u8[15 - i] >> 7) << (15 - i); + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_movemask_epi8(a) simde_mm_movemask_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm_movemask_pd (simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_movemask_pd(a); + #else + int32_t r = 0; + simde__m128d_private a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + HEDLEY_DIAGNOSTIC_PUSH + SIMDE_DIAGNOSTIC_DISABLE_VECTOR_CONVERSION_ + uint64x2_t shifted = vshrq_n_u64(a_.neon_u64, 63); + r = + HEDLEY_STATIC_CAST(int32_t, vgetq_lane_u64(shifted, 0)) + + (HEDLEY_STATIC_CAST(int32_t, vgetq_lane_u64(shifted, 1)) << 1); + HEDLEY_DIAGNOSTIC_POP + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) && defined(SIMDE_BUG_CLANG_50932) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) idx = { 64, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }; + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) res = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), vec_bperm(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned __int128), a_.altivec_u64), idx)); + r = HEDLEY_STATIC_CAST(int32_t, vec_extract(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed int), res), 2)); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) idx = { 64, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }; + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) res = vec_bperm(a_.altivec_u8, idx); + r = HEDLEY_STATIC_CAST(int32_t, vec_extract(HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed int), res), 2)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r = HEDLEY_STATIC_CAST(int32_t, wasm_i64x2_bitmask(a_.wasm_v128)); + #else + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + r |= (a_.u64[i] >> 63) << i; + } + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_movemask_pd(a) simde_mm_movemask_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_movepi64_pi64 (simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_movepi64_pi64(a); + #else + simde__m64_private r_; + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i64 = vget_low_s64(a_.neon_i64); + #else + r_.i64[0] = a_.i64[0]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_movepi64_pi64(a) simde_mm_movepi64_pi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_movpi64_epi64 (simde__m64 a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_movpi64_epi64(a); + #else + simde__m128i_private r_; + simde__m64_private a_ = simde__m64_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vcombine_s64(a_.neon_i64, vdup_n_s64(0)); + #else + r_.i64[0] = a_.i64[0]; + r_.i64[1] = 0; + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_movpi64_epi64(a) simde_mm_movpi64_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_min_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_min_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vminq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_min(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i16 = vec_min(a_.altivec_i16, b_.altivec_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] < b_.i16[i]) ? a_.i16[i] : b_.i16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_min_epi16(a, b) simde_mm_min_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_min_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_min_epu8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vminq_u8(a_.neon_u8, b_.neon_u8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u8x16_min(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u8 = vec_min(a_.altivec_u8, b_.altivec_u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] < b_.u8[i]) ? a_.u8[i] : b_.u8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_min_epu8(a, b) simde_mm_min_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_min_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_min_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = vec_min(a_.altivec_f64, b_.altivec_f64); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vminq_f64(a_.neon_f64, b_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_min(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = (a_.f64[i] < b_.f64[i]) ? a_.f64[i] : b_.f64[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_min_pd(a, b) simde_mm_min_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_min_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_min_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_min_pd(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_min_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + float64x2_t temp = vminq_f64(a_.neon_f64, b_.neon_f64); + r_.neon_f64 = vsetq_lane_f64(vgetq_lane(a_.neon_f64, 1), temp, 1); + #else + r_.f64[0] = (a_.f64[0] < b_.f64[0]) ? a_.f64[0] : b_.f64[0]; + r_.f64[1] = a_.f64[1]; + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_min_sd(a, b) simde_mm_min_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_max_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_max_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vmaxq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_max(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i16 = vec_max(a_.altivec_i16, b_.altivec_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] > b_.i16[i]) ? a_.i16[i] : b_.i16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_max_epi16(a, b) simde_mm_max_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_max_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_max_epu8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vmaxq_u8(a_.neon_u8, b_.neon_u8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u8x16_max(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u8 = vec_max(a_.altivec_u8, b_.altivec_u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] > b_.u8[i]) ? a_.u8[i] : b_.u8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_max_epu8(a, b) simde_mm_max_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_max_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_max_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = vec_max(a_.altivec_f64, b_.altivec_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_max(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vmaxq_f64(a_.neon_f64, b_.neon_f64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = (a_.f64[i] > b_.f64[i]) ? a_.f64[i] : b_.f64[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_max_pd(a, b) simde_mm_max_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_max_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_max_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_max_pd(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_max_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + float64x2_t temp = vmaxq_f64(a_.neon_f64, b_.neon_f64); + r_.neon_f64 = vsetq_lane_f64(vgetq_lane(a_.neon_f64, 1), temp, 1); + #else + r_.f64[0] = (a_.f64[0] > b_.f64[0]) ? a_.f64[0] : b_.f64[0]; + r_.f64[1] = a_.f64[1]; + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_max_sd(a, b) simde_mm_max_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_move_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_move_epi64(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vsetq_lane_s64(0, a_.neon_i64, 1); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_shuffle(a_.wasm_v128, wasm_i64x2_const(0, 0), 0, 2); + #else + r_.i64[0] = a_.i64[0]; + r_.i64[1] = 0; + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_move_epi64(a) simde_mm_move_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mul_epu32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_mul_epu32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x2_t a_lo = vmovn_u64(a_.neon_u64); + uint32x2_t b_lo = vmovn_u64(b_.neon_u64); + r_.neon_u64 = vmull_u32(a_lo, b_lo); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u64x2_extmul_low_u32x4( + wasm_i32x4_shuffle(a_.wasm_v128, a_.wasm_v128, 0, 2, 0, 2), + wasm_i32x4_shuffle(b_.wasm_v128, b_.wasm_v128, 0, 2, 0, 2)); + #elif defined(SIMDE_SHUFFLE_VECTOR_) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + __typeof__(a_.u32) z = { 0, }; + a_.u32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.u32, z, 0, 4, 2, 6); + b_.u32 = SIMDE_SHUFFLE_VECTOR_(32, 16, b_.u32, z, 0, 4, 2, 6); + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), a_.u32) * + HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = HEDLEY_STATIC_CAST(uint64_t, a_.u32[i * 2]) * HEDLEY_STATIC_CAST(uint64_t, b_.u32[i * 2]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_mul_epu32(a, b) simde_mm_mul_epu32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_mul_epi64 (simde__m128i a, simde__m128i b) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_mul(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 * b_.i64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] * b_.i64[i]; + } + #endif + + return simde__m128i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_mod_epi64 (simde__m128i a, simde__m128i b) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.i64 = a_.i64 % b_.i64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] % b_.i64[i]; + } + #endif + + return simde__m128i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mul_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_mul_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = a_.f64 * b_.f64; + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vmulq_f64(a_.neon_f64, b_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_mul(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[i] * b_.f64[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_mul_pd(a, b) simde_mm_mul_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_mul_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_mul_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_mul_pd(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_mul_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + float64x2_t temp = vmulq_f64(a_.neon_f64, b_.neon_f64); + r_.neon_f64 = vsetq_lane_f64(vgetq_lane(a_.neon_f64, 1), temp, 1); + #else + r_.f64[0] = a_.f64[0] * b_.f64[0]; + r_.f64[1] = a_.f64[1]; + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_mul_sd(a, b) simde_mm_mul_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_mul_su32 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) && !defined(__PGI) + return _mm_mul_su32(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.u64[0] = vget_lane_u64(vget_low_u64(vmull_u32(vreinterpret_u32_s64(a_.neon_i64), vreinterpret_u32_s64(b_.neon_i64))), 0); + #else + r_.u64[0] = HEDLEY_STATIC_CAST(uint64_t, a_.u32[0]) * HEDLEY_STATIC_CAST(uint64_t, b_.u32[0]); + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_mul_su32(a, b) simde_mm_mul_su32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mulhi_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_mulhi_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int16x4_t a3210 = vget_low_s16(a_.neon_i16); + int16x4_t b3210 = vget_low_s16(b_.neon_i16); + int32x4_t ab3210 = vmull_s16(a3210, b3210); /* 3333222211110000 */ + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + int32x4_t ab7654 = vmull_high_s16(a_.neon_i16, b_.neon_i16); + r_.neon_i16 = vuzp2q_s16(vreinterpretq_s16_s32(ab3210), vreinterpretq_s16_s32(ab7654)); + #else + int16x4_t a7654 = vget_high_s16(a_.neon_i16); + int16x4_t b7654 = vget_high_s16(b_.neon_i16); + int32x4_t ab7654 = vmull_s16(a7654, b7654); /* 7777666655554444 */ + uint16x8x2_t rv = vuzpq_u16(vreinterpretq_u16_s32(ab3210), vreinterpretq_u16_s32(ab7654)); + r_.neon_u16 = rv.val[1]; + #endif + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + const v128_t lo = wasm_i32x4_extmul_low_i16x8(a_.wasm_v128, b_.wasm_v128); + const v128_t hi = wasm_i32x4_extmul_high_i16x8(a_.wasm_v128, b_.wasm_v128); + r_.wasm_v128 = wasm_i16x8_shuffle(lo, hi, 1, 3, 5, 7, 9, 11, 13, 15); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, (HEDLEY_STATIC_CAST(uint32_t, HEDLEY_STATIC_CAST(int32_t, a_.i16[i]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[i])) >> 16)); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_mulhi_epi16(a, b) simde_mm_mulhi_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mulhi_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) && !defined(__PGI) + return _mm_mulhi_epu16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint16x4_t a3210 = vget_low_u16(a_.neon_u16); + uint16x4_t b3210 = vget_low_u16(b_.neon_u16); + uint32x4_t ab3210 = vmull_u16(a3210, b3210); /* 3333222211110000 */ + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint32x4_t ab7654 = vmull_high_u16(a_.neon_u16, b_.neon_u16); + r_.neon_u16 = vuzp2q_u16(vreinterpretq_u16_u32(ab3210), vreinterpretq_u16_u32(ab7654)); + #else + uint16x4_t a7654 = vget_high_u16(a_.neon_u16); + uint16x4_t b7654 = vget_high_u16(b_.neon_u16); + uint32x4_t ab7654 = vmull_u16(a7654, b7654); /* 7777666655554444 */ + uint16x8x2_t neon_r = vuzpq_u16(vreinterpretq_u16_u32(ab3210), vreinterpretq_u16_u32(ab7654)); + r_.neon_u16 = neon_r.val[1]; + #endif + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + const v128_t lo = wasm_u32x4_extmul_low_u16x8(a_.wasm_v128, b_.wasm_v128); + const v128_t hi = wasm_u32x4_extmul_high_u16x8(a_.wasm_v128, b_.wasm_v128); + r_.wasm_v128 = wasm_i16x8_shuffle(lo, hi, 1, 3, 5, 7, 9, 11, 13, 15); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, HEDLEY_STATIC_CAST(uint32_t, a_.u16[i]) * HEDLEY_STATIC_CAST(uint32_t, b_.u16[i]) >> 16); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_mulhi_epu16(a, b) simde_mm_mulhi_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mullo_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_mullo_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vmulq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + (void) a_; + (void) b_; + r_.altivec_i16 = vec_mul(a_.altivec_i16, b_.altivec_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_mul(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, HEDLEY_STATIC_CAST(uint32_t, a_.u16[i]) * HEDLEY_STATIC_CAST(uint32_t, b_.u16[i])); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_mullo_epi16(a, b) simde_mm_mullo_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_or_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_or_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f | b_.i32f; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_or(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vorrq_s64(a_.neon_i64, b_.neon_i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] | b_.i32f[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_or_pd(a, b) simde_mm_or_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_or_si128 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_or_si128(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vorrq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i32 = vec_or(a_.altivec_i32, b_.altivec_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_or(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f | b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] | b_.i32f[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_or_si128(a, b) simde_mm_or_si128(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_packs_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_packs_epi16(a, b); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i8 = vqmovn_high_s16(vqmovn_s16(a_.neon_i16), b_.neon_i16); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vcombine_s8(vqmovn_s16(a_.neon_i16), vqmovn_s16(b_.neon_i16)); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i8 = vec_packs(a_.altivec_i16, b_.altivec_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_narrow_i16x8(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_CONVERT_VECTOR_) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) + int16_t SIMDE_VECTOR(32) v = SIMDE_SHUFFLE_VECTOR_(16, 32, a_.i16, b_.i16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + const int16_t SIMDE_VECTOR(32) min = { INT8_MIN, INT8_MIN, INT8_MIN, INT8_MIN, INT8_MIN, INT8_MIN, INT8_MIN, INT8_MIN, INT8_MIN, INT8_MIN, INT8_MIN, INT8_MIN, INT8_MIN, INT8_MIN, INT8_MIN, INT8_MIN }; + const int16_t SIMDE_VECTOR(32) max = { INT8_MAX, INT8_MAX, INT8_MAX, INT8_MAX, INT8_MAX, INT8_MAX, INT8_MAX, INT8_MAX, INT8_MAX, INT8_MAX, INT8_MAX, INT8_MAX, INT8_MAX, INT8_MAX, INT8_MAX, INT8_MAX }; + + int16_t m SIMDE_VECTOR(32); + m = HEDLEY_REINTERPRET_CAST(__typeof__(m), v < min); + v = (v & ~m) | (min & m); + + m = v > max; + v = (v & ~m) | (max & m); + + SIMDE_CONVERT_VECTOR_(r_.i8, v); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + int16_t v = (i < (sizeof(a_.i16) / sizeof(a_.i16[0]))) ? a_.i16[i] : b_.i16[i & 7]; + r_.i8[i] = (v < INT8_MIN) ? INT8_MIN : ((v > INT8_MAX) ? INT8_MAX : HEDLEY_STATIC_CAST(int8_t, v)); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_packs_epi16(a, b) simde_mm_packs_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_packs_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_packs_epi32(a, b); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i16 = vqmovn_high_s32(vqmovn_s32(a_.neon_i32), b_.neon_i32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vcombine_s16(vqmovn_s32(a_.neon_i32), vqmovn_s32(b_.neon_i32)); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i16 = vec_packs(a_.altivec_i32, b_.altivec_i32); + #elif defined(SIMDE_X86_SSE2_NATIVE) + r_.sse_m128i = _mm_packs_epi32(a_.sse_m128i, b_.sse_m128i); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_narrow_i32x4(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_CONVERT_VECTOR_) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) + int32_t SIMDE_VECTOR(32) v = SIMDE_SHUFFLE_VECTOR_(32, 32, a_.i32, b_.i32, 0, 1, 2, 3, 4, 5, 6, 7); + const int32_t SIMDE_VECTOR(32) min = { INT16_MIN, INT16_MIN, INT16_MIN, INT16_MIN, INT16_MIN, INT16_MIN, INT16_MIN, INT16_MIN }; + const int32_t SIMDE_VECTOR(32) max = { INT16_MAX, INT16_MAX, INT16_MAX, INT16_MAX, INT16_MAX, INT16_MAX, INT16_MAX, INT16_MAX }; + + int32_t m SIMDE_VECTOR(32); + m = HEDLEY_REINTERPRET_CAST(__typeof__(m), v < min); + v = (v & ~m) | (min & m); + + m = HEDLEY_REINTERPRET_CAST(__typeof__(m), v > max); + v = (v & ~m) | (max & m); + + SIMDE_CONVERT_VECTOR_(r_.i16, v); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + int32_t v = (i < (sizeof(a_.i32) / sizeof(a_.i32[0]))) ? a_.i32[i] : b_.i32[i & 3]; + r_.i16[i] = (v < INT16_MIN) ? INT16_MIN : ((v > INT16_MAX) ? INT16_MAX : HEDLEY_STATIC_CAST(int16_t, v)); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_packs_epi32(a, b) simde_mm_packs_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_packus_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_packus_epi16(a, b); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + #if defined(SIMDE_BUG_CLANG_46840) + r_.neon_u8 = vqmovun_high_s16(vreinterpret_s8_u8(vqmovun_s16(a_.neon_i16)), b_.neon_i16); + #else + r_.neon_u8 = vqmovun_high_s16(vqmovun_s16(a_.neon_i16), b_.neon_i16); + #endif + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = + vcombine_u8( + vqmovun_s16(a_.neon_i16), + vqmovun_s16(b_.neon_i16) + ); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_u8 = vec_packsu(a_.altivec_i16, b_.altivec_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u8x16_narrow_i16x8(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_CONVERT_VECTOR_) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + int16_t v SIMDE_VECTOR(32) = SIMDE_SHUFFLE_VECTOR_(16, 32, a_.i16, b_.i16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + + v &= ~(v >> 15); + v |= HEDLEY_REINTERPRET_CAST(__typeof__(v), v > UINT8_MAX); + + SIMDE_CONVERT_VECTOR_(r_.i8, v); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + int16_t v = (i < (sizeof(a_.i16) / sizeof(a_.i16[0]))) ? a_.i16[i] : b_.i16[i & 7]; + r_.u8[i] = (v < 0) ? UINT8_C(0) : ((v > UINT8_MAX) ? UINT8_MAX : HEDLEY_STATIC_CAST(uint8_t, v)); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_packus_epi16(a, b) simde_mm_packus_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_pause (void) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_pause(); + #elif defined(SIMDE_ARCH_X86) + #if defined(_MSC_VER) + __asm pause; + #else + __asm__ __volatile__("pause"); + #endif + #elif defined(SIMDE_ARCH_ARM_NEON) + #if defined(_MSC_VER) + __isb(_ARM64_BARRIER_SY); + #else + __asm__ __volatile__("isb\n"); + #endif + #elif defined(SIMDE_ARCH_POWER) + __asm__ __volatile__ ("or 27,27,27" ::: "memory"); + #elif defined(SIMDE_ARCH_WASM) + __asm__ __volatile__ ("nop"); + #elif defined(HEDLEY_GCC_VERSION) + #if defined(SIMDE_ARCH_RISCV) + __builtin_riscv_pause(); + #else + __asm__ __volatile__ ("nop" ::: "memory"); + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_pause() (simde_mm_pause()) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sad_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_sad_epu8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const uint16x8_t t = vpaddlq_u8(vabdq_u8(a_.neon_u8, b_.neon_u8)); + r_.neon_u64 = vcombine_u64( + vpaddl_u32(vpaddl_u16(vget_low_u16(t))), + vpaddl_u32(vpaddl_u16(vget_high_u16(t)))); + #else + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + uint16_t tmp = 0; + SIMDE_VECTORIZE_REDUCTION(+:tmp) + for (size_t j = 0 ; j < ((sizeof(r_.u8) / sizeof(r_.u8[0])) / 2) ; j++) { + const size_t e = j + (i * 8); + tmp += (a_.u8[e] > b_.u8[e]) ? (a_.u8[e] - b_.u8[e]) : (b_.u8[e] - a_.u8[e]); + } + r_.i64[i] = tmp; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sad_epu8(a, b) simde_mm_sad_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_set_epi8 (int8_t e15, int8_t e14, int8_t e13, int8_t e12, + int8_t e11, int8_t e10, int8_t e9, int8_t e8, + int8_t e7, int8_t e6, int8_t e5, int8_t e4, + int8_t e3, int8_t e2, int8_t e1, int8_t e0) { + + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_set_epi8( + e15, e14, e13, e12, e11, e10, e9, e8, + e7, e6, e5, e4, e3, e2, e1, e0); + #else + simde__m128i_private r_; + + #if defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_make( + e0, e1, e2, e3, e4, e5, e6, e7, + e8, e9, e10, e11, e12, e13, e14, e15); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_ALIGN_LIKE_16(int8x16_t) int8_t data[16] = { + e0, e1, e2, e3, + e4, e5, e6, e7, + e8, e9, e10, e11, + e12, e13, e14, e15}; + r_.neon_i8 = vld1q_s8(data); + #else + r_.i8[ 0] = e0; + r_.i8[ 1] = e1; + r_.i8[ 2] = e2; + r_.i8[ 3] = e3; + r_.i8[ 4] = e4; + r_.i8[ 5] = e5; + r_.i8[ 6] = e6; + r_.i8[ 7] = e7; + r_.i8[ 8] = e8; + r_.i8[ 9] = e9; + r_.i8[10] = e10; + r_.i8[11] = e11; + r_.i8[12] = e12; + r_.i8[13] = e13; + r_.i8[14] = e14; + r_.i8[15] = e15; + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_set_epi8(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) simde_mm_set_epi8(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_set_epi16 (int16_t e7, int16_t e6, int16_t e5, int16_t e4, + int16_t e3, int16_t e2, int16_t e1, int16_t e0) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_set_epi16(e7, e6, e5, e4, e3, e2, e1, e0); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_ALIGN_LIKE_16(int16x8_t) int16_t data[8] = { e0, e1, e2, e3, e4, e5, e6, e7 }; + r_.neon_i16 = vld1q_s16(data); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_make(e0, e1, e2, e3, e4, e5, e6, e7); + #else + r_.i16[0] = e0; + r_.i16[1] = e1; + r_.i16[2] = e2; + r_.i16[3] = e3; + r_.i16[4] = e4; + r_.i16[5] = e5; + r_.i16[6] = e6; + r_.i16[7] = e7; + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_set_epi16(e7, e6, e5, e4, e3, e2, e1, e0) simde_mm_set_epi16(e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_loadu_si16 (void const* mem_addr) { + #if defined(SIMDE_X86_SSE2_NATIVE) && ( \ + SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0) || \ + HEDLEY_INTEL_VERSION_CHECK(20,21,1) || \ + HEDLEY_GCC_VERSION_CHECK(12,1,0)) + return _mm_loadu_si16(mem_addr); + #else + int16_t val; + simde_memcpy(&val, mem_addr, sizeof(val)); + return simde_x_mm_cvtsi16_si128(val); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_loadu_si16(mem_addr) simde_mm_loadu_si16(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_set_epi32 (int32_t e3, int32_t e2, int32_t e1, int32_t e0) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_set_epi32(e3, e2, e1, e0); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_ALIGN_LIKE_16(int32x4_t) int32_t data[4] = { e0, e1, e2, e3 }; + r_.neon_i32 = vld1q_s32(data); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_make(e0, e1, e2, e3); + #else + r_.i32[0] = e0; + r_.i32[1] = e1; + r_.i32[2] = e2; + r_.i32[3] = e3; + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_set_epi32(e3, e2, e1, e0) simde_mm_set_epi32(e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_loadu_si32 (void const* mem_addr) { + #if defined(SIMDE_X86_SSE2_NATIVE) && ( \ + SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0) || \ + HEDLEY_INTEL_VERSION_CHECK(20,21,1) || \ + HEDLEY_GCC_VERSION_CHECK(12,1,0)) + return _mm_loadu_si32(mem_addr); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return simde__m128i_from_wasm_v128(wasm_v128_load32_zero(mem_addr)); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + simde__m128i_private r_; + r_.neon_i32 = vsetq_lane_s32(* HEDLEY_REINTERPRET_CAST(const int32_t *, mem_addr), vdupq_n_s32(0), 0); + return simde__m128i_from_private(r_); + #else + int32_t val; + simde_memcpy(&val, mem_addr, sizeof(val)); + return simde_mm_cvtsi32_si128(val); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_loadu_si32(mem_addr) simde_mm_loadu_si32(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_set_epi64 (simde__m64 e1, simde__m64 e0) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_set_epi64(e1, e0); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vcombine_s64(simde__m64_to_neon_i64(e0), simde__m64_to_neon_i64(e1)); + #else + r_.m64[0] = e0; + r_.m64[1] = e1; + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_set_epi64(e1, e0) (simde_mm_set_epi64((e1), (e0))) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_set_epi64x (int64_t e1, int64_t e0) { + #if defined(SIMDE_X86_SSE2_NATIVE) && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,0,0)) + return _mm_set_epi64x(e1, e0); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_ALIGN_LIKE_16(int64x2_t) int64_t data[2] = {e0, e1}; + r_.neon_i64 = vld1q_s64(data); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_make(e0, e1); + #else + r_.i64[0] = e0; + r_.i64[1] = e1; + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_set_epi64x(e1, e0) simde_mm_set_epi64x(e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_loadu_si64 (void const* mem_addr) { + #if defined(SIMDE_X86_SSE2_NATIVE) && ( \ + SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0) || \ + HEDLEY_GCC_VERSION_CHECK(11,0,0) || \ + HEDLEY_INTEL_VERSION_CHECK(20,21,1)) + return _mm_loadu_si64(mem_addr); + #else + int64_t val; + simde_memcpy(&val, mem_addr, sizeof(val)); + return simde_mm_cvtsi64_si128(val); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_loadu_si64(mem_addr) simde_mm_loadu_si64(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_set_epu8 (uint8_t e15, uint8_t e14, uint8_t e13, uint8_t e12, + uint8_t e11, uint8_t e10, uint8_t e9, uint8_t e8, + uint8_t e7, uint8_t e6, uint8_t e5, uint8_t e4, + uint8_t e3, uint8_t e2, uint8_t e1, uint8_t e0) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_set_epi8( + HEDLEY_STATIC_CAST(char, e15), HEDLEY_STATIC_CAST(char, e14), HEDLEY_STATIC_CAST(char, e13), HEDLEY_STATIC_CAST(char, e12), + HEDLEY_STATIC_CAST(char, e11), HEDLEY_STATIC_CAST(char, e10), HEDLEY_STATIC_CAST(char, e9), HEDLEY_STATIC_CAST(char, e8), + HEDLEY_STATIC_CAST(char, e7), HEDLEY_STATIC_CAST(char, e6), HEDLEY_STATIC_CAST(char, e5), HEDLEY_STATIC_CAST(char, e4), + HEDLEY_STATIC_CAST(char, e3), HEDLEY_STATIC_CAST(char, e2), HEDLEY_STATIC_CAST(char, e1), HEDLEY_STATIC_CAST(char, e0)); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_ALIGN_LIKE_16(uint8x16_t) uint8_t data[16] = { + e0, e1, e2, e3, + e4, e5, e6, e7, + e8, e9, e10, e11, + e12, e13, e14, e15}; + r_.neon_u8 = vld1q_u8(data); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u8x16_make(e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15); + #else + r_.u8[ 0] = e0; r_.u8[ 1] = e1; r_.u8[ 2] = e2; r_.u8[ 3] = e3; + r_.u8[ 4] = e4; r_.u8[ 5] = e5; r_.u8[ 6] = e6; r_.u8[ 7] = e7; + r_.u8[ 8] = e8; r_.u8[ 9] = e9; r_.u8[10] = e10; r_.u8[11] = e11; + r_.u8[12] = e12; r_.u8[13] = e13; r_.u8[14] = e14; r_.u8[15] = e15; + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_set_epu16 (uint16_t e7, uint16_t e6, uint16_t e5, uint16_t e4, + uint16_t e3, uint16_t e2, uint16_t e1, uint16_t e0) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_set_epi16( + HEDLEY_STATIC_CAST(short, e7), HEDLEY_STATIC_CAST(short, e6), HEDLEY_STATIC_CAST(short, e5), HEDLEY_STATIC_CAST(short, e4), + HEDLEY_STATIC_CAST(short, e3), HEDLEY_STATIC_CAST(short, e2), HEDLEY_STATIC_CAST(short, e1), HEDLEY_STATIC_CAST(short, e0)); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_ALIGN_LIKE_16(uint16x8_t) uint16_t data[8] = { e0, e1, e2, e3, e4, e5, e6, e7 }; + r_.neon_u16 = vld1q_u16(data); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u16x8_make(e0, e1, e2, e3, e4, e5, e6, e7); + #else + r_.u16[0] = e0; r_.u16[1] = e1; r_.u16[2] = e2; r_.u16[3] = e3; + r_.u16[4] = e4; r_.u16[5] = e5; r_.u16[6] = e6; r_.u16[7] = e7; + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_set_epu32 (uint32_t e3, uint32_t e2, uint32_t e1, uint32_t e0) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_set_epi32( + HEDLEY_STATIC_CAST(int, e3), HEDLEY_STATIC_CAST(int, e2), HEDLEY_STATIC_CAST(int, e1), HEDLEY_STATIC_CAST(int, e0)); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_ALIGN_LIKE_16(uint32x4_t) uint32_t data[4] = { e0, e1, e2, e3 }; + r_.neon_u32 = vld1q_u32(data); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u32x4_make(e0, e1, e2, e3); + #else + r_.u32[0] = e0; + r_.u32[1] = e1; + r_.u32[2] = e2; + r_.u32[3] = e3; + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_set_epu64x (uint64_t e1, uint64_t e0) { + #if defined(SIMDE_X86_SSE2_NATIVE) && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,0,0)) + return _mm_set_epi64x(HEDLEY_STATIC_CAST(int64_t, e1), HEDLEY_STATIC_CAST(int64_t, e0)); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + SIMDE_ALIGN_LIKE_16(uint64x2_t) uint64_t data[2] = {e0, e1}; + r_.neon_u64 = vld1q_u64(data); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u64x2_make(e0, e1); + #else + r_.u64[0] = e0; + r_.u64[1] = e1; + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_set_sd (simde_float64 a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_set_sd(a); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return vsetq_lane_f64(a, vdupq_n_f64(SIMDE_FLOAT64_C(0.0)), 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return simde__m128d_from_wasm_v128(wasm_f64x2_make(a, 0)); + #else + return simde_mm_set_pd(SIMDE_FLOAT64_C(0.0), a); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_set_sd(a) simde_mm_set_sd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_set1_epi8 (int8_t a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_set1_epi8(a); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vdupq_n_s8(a); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_splat(a); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i8 = vec_splats(HEDLEY_STATIC_CAST(signed char, a)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_set1_epi8(a) simde_mm_set1_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_set1_epi16 (int16_t a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_set1_epi16(a); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vdupq_n_s16(a); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_splat(a); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i16 = vec_splats(HEDLEY_STATIC_CAST(signed short, a)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_set1_epi16(a) simde_mm_set1_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_set1_epi32 (int32_t a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_set1_epi32(a); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vdupq_n_s32(a); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_splat(a); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i32 = vec_splats(HEDLEY_STATIC_CAST(signed int, a)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_set1_epi32(a) simde_mm_set1_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_set1_epi64x (int64_t a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && (!defined(HEDLEY_MSVC_VERSION) || HEDLEY_MSVC_VERSION_CHECK(19,0,0)) + return _mm_set1_epi64x(a); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vdupq_n_s64(a); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_splat(a); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i64 = vec_splats(HEDLEY_STATIC_CAST(signed long long, a)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_set1_epi64x(a) simde_mm_set1_epi64x(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_set1_epi64 (simde__m64 a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_set1_epi64(a); + #else + simde__m64_private a_ = simde__m64_to_private(a); + return simde_mm_set1_epi64x(a_.i64[0]); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_set1_epi64(a) simde_mm_set1_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_set1_epu8 (uint8_t value) { + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + return simde__m128i_from_altivec_u8(vec_splats(HEDLEY_STATIC_CAST(unsigned char, value))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return simde__m128i_from_wasm_v128(wasm_u8x16_splat(value)); + #else + return simde_mm_set1_epi8(HEDLEY_STATIC_CAST(int8_t, value)); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_set1_epu16 (uint16_t value) { + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + return simde__m128i_from_altivec_u16(vec_splats(HEDLEY_STATIC_CAST(unsigned short, value))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return simde__m128i_from_wasm_v128(wasm_u16x8_splat(value)); + #else + return simde_mm_set1_epi16(HEDLEY_STATIC_CAST(int16_t, value)); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_set1_epu32 (uint32_t value) { + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + return simde__m128i_from_altivec_u32(vec_splats(HEDLEY_STATIC_CAST(unsigned int, value))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return simde__m128i_from_wasm_v128(wasm_u32x4_splat(value)); + #else + return simde_mm_set1_epi32(HEDLEY_STATIC_CAST(int32_t, value)); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_set1_epu64 (uint64_t value) { + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + return simde__m128i_from_altivec_u64(vec_splats(HEDLEY_STATIC_CAST(unsigned long long, value))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return simde__m128i_from_wasm_v128(wasm_u64x2_splat(value)); + #else + return simde_mm_set1_epi64x(HEDLEY_STATIC_CAST(int64_t, value)); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_setr_epi8 (int8_t e15, int8_t e14, int8_t e13, int8_t e12, + int8_t e11, int8_t e10, int8_t e9, int8_t e8, + int8_t e7, int8_t e6, int8_t e5, int8_t e4, + int8_t e3, int8_t e2, int8_t e1, int8_t e0) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_setr_epi8( + e15, e14, e13, e12, e11, e10, e9, e8, + e7, e6, e5, e4, e3, e2, e1, e0); + #else + return simde_mm_set_epi8( + e0, e1, e2, e3, e4, e5, e6, e7, + e8, e9, e10, e11, e12, e13, e14, e15); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_setr_epi8(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) simde_mm_setr_epi8(e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_setr_epi16 (int16_t e7, int16_t e6, int16_t e5, int16_t e4, + int16_t e3, int16_t e2, int16_t e1, int16_t e0) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_setr_epi16(e7, e6, e5, e4, e3, e2, e1, e0); + #else + return simde_mm_set_epi16(e0, e1, e2, e3, e4, e5, e6, e7); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_setr_epi16(e7, e6, e5, e4, e3, e2, e1, e0) simde_mm_setr_epi16(e7, e6, e5, e4, e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_setr_epi32 (int32_t e3, int32_t e2, int32_t e1, int32_t e0) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_setr_epi32(e3, e2, e1, e0); + #else + return simde_mm_set_epi32(e0, e1, e2, e3); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_setr_epi32(e3, e2, e1, e0) simde_mm_setr_epi32(e3, e2, e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_setr_epi64 (simde__m64 e1, simde__m64 e0) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_setr_epi64(e1, e0); + #else + return simde_mm_set_epi64(e0, e1); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_setr_epi64(e1, e0) (simde_mm_setr_epi64((e1), (e0))) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_setr_pd (simde_float64 e1, simde_float64 e0) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_setr_pd(e1, e0); + #else + return simde_mm_set_pd(e0, e1); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_setr_pd(e1, e0) simde_mm_setr_pd(e1, e0) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_setzero_pd (void) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_setzero_pd(); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return simde__m128d_from_wasm_v128(wasm_f64x2_const(0.0, 0.0)); + #else + return simde_mm_castsi128_pd(simde_mm_setzero_si128()); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_setzero_pd() simde_mm_setzero_pd() +#endif + +#if defined(SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_) +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_ +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_undefined_pd (void) { + simde__m128d_private r_; + + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE__HAVE_UNDEFINED128) + r_.n = _mm_undefined_pd(); + #elif !defined(SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_) + r_ = simde__m128d_to_private(simde_mm_setzero_pd()); + #endif + + return simde__m128d_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_undefined_pd() simde_mm_undefined_pd() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_undefined_si128 (void) { + simde__m128i_private r_; + + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE__HAVE_UNDEFINED128) + r_.n = _mm_undefined_si128(); + #elif !defined(SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_) + r_ = simde__m128i_to_private(simde_mm_setzero_si128()); + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_undefined_si128() (simde_mm_undefined_si128()) +#endif + +#if defined(SIMDE_DIAGNOSTIC_DISABLE_UNINITIALIZED_) +HEDLEY_DIAGNOSTIC_POP +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_x_mm_setone_pd (void) { + return simde_mm_castps_pd(simde_x_mm_setone_ps()); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_setone_si128 (void) { + return simde_mm_castps_si128(simde_x_mm_setone_ps()); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_shuffle_epi32 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[(imm8 >> (i * 2)) & 3]; + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) + #define simde_mm_shuffle_epi32(a, imm8) _mm_shuffle_epi32((a), (imm8)) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_shuffle_epi32(a, imm8) (__extension__ ({ \ + const simde__m128i_private simde_tmp_a_ = simde__m128i_to_private(a); \ + simde__m128i_from_wasm_v128( \ + wasm_i32x4_shuffle( \ + (simde_tmp_a_).wasm_v128, \ + (simde_tmp_a_).wasm_v128, \ + ((imm8) ) & 3, \ + ((imm8) >> 2) & 3, \ + ((imm8) >> 4) & 3, \ + ((imm8) >> 6) & 3)); })) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm_shuffle_epi32(a, imm8) \ + (__extension__ ({ \ + const int32x4_t simde_mm_shuffle_epi32_a_ = simde__m128i_to_neon_i32(a); \ + int32x4_t simde_mm_shuffle_epi32_r_; \ + simde_mm_shuffle_epi32_r_ = vmovq_n_s32(vgetq_lane_s32(simde_mm_shuffle_epi32_a_, (imm8) & (0x3))); \ + simde_mm_shuffle_epi32_r_ = vsetq_lane_s32(vgetq_lane_s32(simde_mm_shuffle_epi32_a_, ((imm8) >> 2) & 0x3), simde_mm_shuffle_epi32_r_, 1); \ + simde_mm_shuffle_epi32_r_ = vsetq_lane_s32(vgetq_lane_s32(simde_mm_shuffle_epi32_a_, ((imm8) >> 4) & 0x3), simde_mm_shuffle_epi32_r_, 2); \ + simde_mm_shuffle_epi32_r_ = vsetq_lane_s32(vgetq_lane_s32(simde_mm_shuffle_epi32_a_, ((imm8) >> 6) & 0x3), simde_mm_shuffle_epi32_r_, 3); \ + vreinterpretq_s64_s32(simde_mm_shuffle_epi32_r_); \ + })) +#elif defined(SIMDE_SHUFFLE_VECTOR_) + #define simde_mm_shuffle_epi32(a, imm8) (__extension__ ({ \ + const simde__m128i_private simde_tmp_a_ = simde__m128i_to_private(a); \ + simde__m128i_from_private((simde__m128i_private) { .i32 = \ + SIMDE_SHUFFLE_VECTOR_(32, 16, \ + (simde_tmp_a_).i32, \ + (simde_tmp_a_).i32, \ + ((imm8) ) & 3, \ + ((imm8) >> 2) & 3, \ + ((imm8) >> 4) & 3, \ + ((imm8) >> 6) & 3) }); })) +#endif +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_shuffle_epi32(a, imm8) simde_mm_shuffle_epi32(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_shuffle_pd (simde__m128d a, simde__m128d b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + r_.f64[0] = ((imm8 & 1) == 0) ? a_.f64[0] : a_.f64[1]; + r_.f64[1] = ((imm8 & 2) == 0) ? b_.f64[0] : b_.f64[1]; + + return simde__m128d_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) && !defined(__PGI) + #define simde_mm_shuffle_pd(a, b, imm8) _mm_shuffle_pd((a), (b), (imm8)) +#elif defined(SIMDE_SHUFFLE_VECTOR_) + #define simde_mm_shuffle_pd(a, b, imm8) (__extension__ ({ \ + simde__m128d_from_private((simde__m128d_private) { .f64 = \ + SIMDE_SHUFFLE_VECTOR_(64, 16, \ + simde__m128d_to_private(a).f64, \ + simde__m128d_to_private(b).f64, \ + (((imm8) ) & 1), \ + (((imm8) >> 1) & 1) + 2) }); })) +#endif +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_shuffle_pd(a, b, imm8) simde_mm_shuffle_pd(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_shufflehi_epi16 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < ((sizeof(a_.i16) / sizeof(a_.i16[0])) / 2) ; i++) { + r_.i16[i] = a_.i16[i]; + } + for (size_t i = ((sizeof(a_.i16) / sizeof(a_.i16[0])) / 2) ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[((imm8 >> ((i - 4) * 2)) & 3) + 4]; + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) + #define simde_mm_shufflehi_epi16(a, imm8) _mm_shufflehi_epi16((a), (imm8)) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm_shufflehi_epi16(a, imm8) \ + (__extension__ ({ \ + int16x8_t simde_mm_shufflehi_epi16_a_ = simde__m128i_to_neon_i16(a); \ + int16x8_t simde_mm_shufflehi_epi16_r_ = simde_mm_shufflehi_epi16_a_; \ + simde_mm_shufflehi_epi16_r_ = vsetq_lane_s16(vgetq_lane_s16(simde_mm_shufflehi_epi16_a_, (((imm8) ) & 0x3) + 4), simde_mm_shufflehi_epi16_r_, 4); \ + simde_mm_shufflehi_epi16_r_ = vsetq_lane_s16(vgetq_lane_s16(simde_mm_shufflehi_epi16_a_, (((imm8) >> 2) & 0x3) + 4), simde_mm_shufflehi_epi16_r_, 5); \ + simde_mm_shufflehi_epi16_r_ = vsetq_lane_s16(vgetq_lane_s16(simde_mm_shufflehi_epi16_a_, (((imm8) >> 4) & 0x3) + 4), simde_mm_shufflehi_epi16_r_, 6); \ + simde_mm_shufflehi_epi16_r_ = vsetq_lane_s16(vgetq_lane_s16(simde_mm_shufflehi_epi16_a_, (((imm8) >> 6) & 0x3) + 4), simde_mm_shufflehi_epi16_r_, 7); \ + simde__m128i_from_neon_i16(simde_mm_shufflehi_epi16_r_); \ + })) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_shufflehi_epi16(a, imm8) (__extension__ ({ \ + const simde__m128i_private simde_tmp_a_ = simde__m128i_to_private(a); \ + simde__m128i_from_private((simde__m128i_private) { .wasm_v128 = \ + wasm_i16x8_shuffle( \ + (simde_tmp_a_).wasm_v128, \ + (simde_tmp_a_).wasm_v128, \ + 0, 1, 2, 3, \ + (((imm8) ) & 3) + 4, \ + (((imm8) >> 2) & 3) + 4, \ + (((imm8) >> 4) & 3) + 4, \ + (((imm8) >> 6) & 3) + 4) }); })) +#elif defined(SIMDE_SHUFFLE_VECTOR_) + #define simde_mm_shufflehi_epi16(a, imm8) (__extension__ ({ \ + const simde__m128i_private simde_tmp_a_ = simde__m128i_to_private(a); \ + simde__m128i_from_private((simde__m128i_private) { .i16 = \ + SIMDE_SHUFFLE_VECTOR_(16, 16, \ + (simde_tmp_a_).i16, \ + (simde_tmp_a_).i16, \ + 0, 1, 2, 3, \ + (((imm8) ) & 3) + 4, \ + (((imm8) >> 2) & 3) + 4, \ + (((imm8) >> 4) & 3) + 4, \ + (((imm8) >> 6) & 3) + 4) }); })) +#endif +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_shufflehi_epi16(a, imm8) simde_mm_shufflehi_epi16(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_shufflelo_epi16 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + for (size_t i = 0 ; i < ((sizeof(r_.i16) / sizeof(r_.i16[0])) / 2) ; i++) { + r_.i16[i] = a_.i16[((imm8 >> (i * 2)) & 3)]; + } + SIMDE_VECTORIZE + for (size_t i = ((sizeof(a_.i16) / sizeof(a_.i16[0])) / 2) ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i]; + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) + #define simde_mm_shufflelo_epi16(a, imm8) _mm_shufflelo_epi16((a), (imm8)) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_shufflelo_epi16(a, imm8) \ + simde__m128i_from_wasm_v128( \ + wasm_i16x8_shuffle( \ + simde__m128i_to_wasm_v128((a)), \ + wasm_i16x8_splat(0), \ + (((imm8) & 0x03) ), \ + (((imm8) & 0x0c) >> 2), \ + (((imm8) & 0x30) >> 4), \ + (((imm8) & 0xc0) >> 6), \ + 4, 5, 6, 7)) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) && defined(SIMDE_STATEMENT_EXPR_) + #define simde_mm_shufflelo_epi16(a, imm8) \ + (__extension__({ \ + int16x8_t simde_mm_shufflelo_epi16_a_ = simde__m128i_to_neon_i16(a); \ + int16x8_t simde_mm_shufflelo_epi16_r_ = simde_mm_shufflelo_epi16_a_; \ + simde_mm_shufflelo_epi16_r_ = vsetq_lane_s16(vgetq_lane_s16(simde_mm_shufflelo_epi16_a_, (((imm8) ) & 0x3)), simde_mm_shufflelo_epi16_r_, 0); \ + simde_mm_shufflelo_epi16_r_ = vsetq_lane_s16(vgetq_lane_s16(simde_mm_shufflelo_epi16_a_, (((imm8) >> 2) & 0x3)), simde_mm_shufflelo_epi16_r_, 1); \ + simde_mm_shufflelo_epi16_r_ = vsetq_lane_s16(vgetq_lane_s16(simde_mm_shufflelo_epi16_a_, (((imm8) >> 4) & 0x3)), simde_mm_shufflelo_epi16_r_, 2); \ + simde_mm_shufflelo_epi16_r_ = vsetq_lane_s16(vgetq_lane_s16(simde_mm_shufflelo_epi16_a_, (((imm8) >> 6) & 0x3)), simde_mm_shufflelo_epi16_r_, 3); \ + simde__m128i_from_neon_i16(simde_mm_shufflelo_epi16_r_); \ + })) +#elif defined(SIMDE_SHUFFLE_VECTOR_) + #define simde_mm_shufflelo_epi16(a, imm8) (__extension__ ({ \ + const simde__m128i_private simde_tmp_a_ = simde__m128i_to_private(a); \ + simde__m128i_from_private((simde__m128i_private) { .i16 = \ + SIMDE_SHUFFLE_VECTOR_(16, 16, \ + (simde_tmp_a_).i16, \ + (simde_tmp_a_).i16, \ + (((imm8) ) & 3), \ + (((imm8) >> 2) & 3), \ + (((imm8) >> 4) & 3), \ + (((imm8) >> 6) & 3), \ + 4, 5, 6, 7) }); })) +#endif +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_shufflelo_epi16(a, imm8) simde_mm_shufflelo_epi16(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sll_epi16 (simde__m128i a, simde__m128i count) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_sll_epi16(a, count); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + count_ = simde__m128i_to_private(count); + + if (count_.u64[0] > 15) + return simde_mm_setzero_si128(); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u16 = (a_.u16 << count_.u64[0]); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vshlq_u16(a_.neon_u16, vdupq_n_s16(HEDLEY_STATIC_CAST(int16_t, count_.u64[0]))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = ((wasm_i64x2_extract_lane(count_.wasm_v128, 0) < 16) ? wasm_i16x8_shl(a_.wasm_v128, HEDLEY_STATIC_CAST(int32_t, wasm_i64x2_extract_lane(count_.wasm_v128, 0))) : wasm_i16x8_const(0,0,0,0,0,0,0,0)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, (a_.u16[i] << count_.u64[0])); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sll_epi16(a, count) simde_mm_sll_epi16((a), (count)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sll_epi32 (simde__m128i a, simde__m128i count) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_sll_epi32(a, count); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + count_ = simde__m128i_to_private(count); + + if (count_.u64[0] > 31) + return simde_mm_setzero_si128(); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = (a_.u32 << count_.u64[0]); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vshlq_u32(a_.neon_u32, vdupq_n_s32(HEDLEY_STATIC_CAST(int32_t, count_.u64[0]))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = ((wasm_i64x2_extract_lane(count_.wasm_v128, 0) < 32) ? wasm_i32x4_shl(a_.wasm_v128, HEDLEY_STATIC_CAST(int32_t, wasm_i64x2_extract_lane(count_.wasm_v128, 0))) : wasm_i32x4_const(0,0,0,0)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = HEDLEY_STATIC_CAST(uint32_t, (a_.u32[i] << count_.u64[0])); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sll_epi32(a, count) (simde_mm_sll_epi32(a, (count))) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sll_epi64 (simde__m128i a, simde__m128i count) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_sll_epi64(a, count); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + count_ = simde__m128i_to_private(count); + + if (count_.u64[0] > 63) + return simde_mm_setzero_si128(); + + const int_fast16_t s = HEDLEY_STATIC_CAST(int_fast16_t, count_.u64[0]); + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u64 = vshlq_u64(a_.neon_u64, vdupq_n_s64(HEDLEY_STATIC_CAST(int64_t, s))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = (s < 64) ? wasm_i64x2_shl(a_.wasm_v128, HEDLEY_STATIC_CAST(uint32_t, s)) : wasm_i64x2_const(0,0); + #else + #if !defined(SIMDE_BUG_GCC_94488) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = a_.u64[i] << s; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sll_epi64(a, count) (simde_mm_sll_epi64(a, (count))) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_sqrt_pd (simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_sqrt_pd(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vsqrtq_f64(a_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_sqrt(a_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = vec_sqrt(a_.altivec_f64); + #elif defined(simde_math_sqrt) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_sqrt(a_.f64[i]); + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sqrt_pd(a) simde_mm_sqrt_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_sqrt_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_sqrt_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_sqrt_pd(b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_sqrt_pd(simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(simde_math_sqrt) + r_.f64[0] = simde_math_sqrt(b_.f64[0]); + r_.f64[1] = a_.f64[1]; + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sqrt_sd(a, b) simde_mm_sqrt_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_srl_epi16 (simde__m128i a, simde__m128i count) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_srl_epi16(a, count); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + count_ = simde__m128i_to_private(count); + + const int cnt = HEDLEY_STATIC_CAST(int, (count_.i64[0] > 16 ? 16 : count_.i64[0])); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vshlq_u16(a_.neon_u16, vdupq_n_s16(HEDLEY_STATIC_CAST(int16_t, -cnt))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = a_.u16[i] >> cnt; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_srl_epi16(a, count) (simde_mm_srl_epi16(a, (count))) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_srl_epi32 (simde__m128i a, simde__m128i count) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_srl_epi32(a, count); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + count_ = simde__m128i_to_private(count); + + const int cnt = HEDLEY_STATIC_CAST(int, (count_.i64[0] > 32 ? 32 : count_.i64[0])); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vshlq_u32(a_.neon_u32, vdupq_n_s32(HEDLEY_STATIC_CAST(int32_t, -cnt))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u32x4_shr(a_.wasm_v128, HEDLEY_STATIC_CAST(uint32_t, cnt)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] >> cnt; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_srl_epi32(a, count) (simde_mm_srl_epi32(a, (count))) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_srl_epi64 (simde__m128i a, simde__m128i count) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_srl_epi64(a, count); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + count_ = simde__m128i_to_private(count); + + const int cnt = HEDLEY_STATIC_CAST(int, (count_.i64[0] > 64 ? 64 : count_.i64[0])); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u64 = vshlq_u64(a_.neon_u64, vdupq_n_s64(HEDLEY_STATIC_CAST(int64_t, -cnt))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u64x2_shr(a_.wasm_v128, HEDLEY_STATIC_CAST(uint32_t, cnt)); + #else + #if !defined(SIMDE_BUG_GCC_94488) + SIMDE_VECTORIZE + #endif + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = a_.u64[i] >> cnt; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_srl_epi64(a, count) (simde_mm_srl_epi64(a, (count))) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_srai_epi16 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + /* MSVC requires a range of (0, 255). */ + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + const int cnt = (imm8 & ~15) ? 15 : imm8; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vshlq_s16(a_.neon_i16, vdupq_n_s16(HEDLEY_STATIC_CAST(int16_t, -cnt))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_shr(a_.wasm_v128, HEDLEY_STATIC_CAST(uint32_t, cnt)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] >> cnt; + } + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) + #define simde_mm_srai_epi16(a, imm8) _mm_srai_epi16((a), (imm8)) +#endif +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_srai_epi16(a, imm8) simde_mm_srai_epi16(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_srai_epi32 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_RANGE(imm8, 0, 255) { + /* MSVC requires a range of (0, 255). */ + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + const int cnt = (imm8 & ~31) ? 31 : imm8; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vshlq_s32(a_.neon_i32, vdupq_n_s32(-cnt)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_shr(a_.wasm_v128, HEDLEY_STATIC_CAST(uint32_t, cnt)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] >> cnt; + } + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) + #define simde_mm_srai_epi32(a, imm8) _mm_srai_epi32((a), (imm8)) +#endif +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_srai_epi32(a, imm8) simde_mm_srai_epi32(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sra_epi16 (simde__m128i a, simde__m128i count) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_sra_epi16(a, count); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + count_ = simde__m128i_to_private(count); + + const int cnt = HEDLEY_STATIC_CAST(int, (count_.i64[0] > 15 ? 15 : count_.i64[0])); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vshlq_s16(a_.neon_i16, vdupq_n_s16(HEDLEY_STATIC_CAST(int16_t, -cnt))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_shr(a_.wasm_v128, HEDLEY_STATIC_CAST(uint32_t, cnt)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] >> cnt; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sra_epi16(a, count) (simde_mm_sra_epi16(a, count)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sra_epi32 (simde__m128i a, simde__m128i count) { + #if defined(SIMDE_X86_SSE2_NATIVE) && !defined(SIMDE_BUG_GCC_BAD_MM_SRA_EPI32) + return _mm_sra_epi32(a, count); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + count_ = simde__m128i_to_private(count); + + const int cnt = count_.u64[0] > 31 ? 31 : HEDLEY_STATIC_CAST(int, count_.u64[0]); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vshlq_s32(a_.neon_i32, vdupq_n_s32(HEDLEY_STATIC_CAST(int32_t, -cnt))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_shr(a_.wasm_v128, HEDLEY_STATIC_CAST(uint32_t, cnt)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] >> cnt; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sra_epi32(a, count) (simde_mm_sra_epi32(a, (count))) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_slli_epi16 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + if (HEDLEY_UNLIKELY((imm8 > 15))) { + return simde_mm_setzero_si128(); + } + + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i16 = a_.i16 << SIMDE_CAST_VECTOR_SHIFT_COUNT(8, imm8 & 0xff); + #else + const int s = (imm8 > HEDLEY_STATIC_CAST(int, sizeof(r_.i16[0]) * CHAR_BIT) - 1) ? 0 : imm8; + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, a_.i16[i] << s); + } + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) + #define simde_mm_slli_epi16(a, imm8) _mm_slli_epi16(a, imm8) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #define simde_mm_slli_epi16(a, imm8) \ + (((imm8) <= 0) ? \ + (a) : \ + simde__m128i_from_neon_i16( \ + ((imm8) > 15) ? \ + vandq_s16(simde__m128i_to_neon_i16(a), vdupq_n_s16(0)) : \ + vshlq_n_s16(simde__m128i_to_neon_i16(a), ((imm8) & 15)))) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_slli_epi16(a, imm8) \ + ((imm8 < 16) ? wasm_i16x8_shl(simde__m128i_to_private(a).wasm_v128, imm8) : wasm_i16x8_const(0,0,0,0,0,0,0,0)) +#elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + #define simde_mm_slli_epi16(a, imm8) \ + ((imm8 & ~15) ? simde_mm_setzero_si128() : simde__m128i_from_altivec_i16(vec_sl(simde__m128i_to_altivec_i16(a), vec_splat_u16(HEDLEY_STATIC_CAST(unsigned short, imm8))))) +#endif +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_slli_epi16(a, imm8) simde_mm_slli_epi16(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_slli_epi32 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + if (HEDLEY_UNLIKELY((imm8 > 31))) { + return simde_mm_setzero_si128(); + } + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i32 = a_.i32 << imm8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] << (imm8 & 0xff); + } + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) + #define simde_mm_slli_epi32(a, imm8) _mm_slli_epi32(a, imm8) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #define simde_mm_slli_epi32(a, imm8) \ + (((imm8) <= 0) ? \ + (a) : \ + simde__m128i_from_neon_i32( \ + ((imm8) > 31) ? \ + vandq_s32(simde__m128i_to_neon_i32(a), vdupq_n_s32(0)) : \ + vshlq_n_s32(simde__m128i_to_neon_i32(a), ((imm8) & 31)))) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_slli_epi32(a, imm8) \ + ((imm8 < 32) ? wasm_i32x4_shl(simde__m128i_to_private(a).wasm_v128, imm8) : wasm_i32x4_const(0,0,0,0)) +#elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + #define simde_mm_slli_epi32(a, imm8) \ + (__extension__ ({ \ + simde__m128i ret; \ + if ((imm8) <= 0) { \ + ret = a; \ + } else if ((imm8) > 31) { \ + ret = simde_mm_setzero_si128(); \ + } else { \ + ret = simde__m128i_from_altivec_i32( \ + vec_sl(simde__m128i_to_altivec_i32(a), \ + vec_splats(HEDLEY_STATIC_CAST(unsigned int, (imm8) & 31)))); \ + } \ + ret; \ + })) +#endif +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_slli_epi32(a, imm8) simde_mm_slli_epi32(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_slli_epi64 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + if (HEDLEY_UNLIKELY((imm8 > 63))) { + return simde_mm_setzero_si128(); + } + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i64 = a_.i64 << imm8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] << (imm8 & 0xff); + } + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) + #define simde_mm_slli_epi64(a, imm8) _mm_slli_epi64(a, imm8) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #define simde_mm_slli_epi64(a, imm8) \ + (((imm8) <= 0) ? \ + (a) : \ + simde__m128i_from_neon_i64( \ + ((imm8) > 63) ? \ + vandq_s64(simde__m128i_to_neon_i64(a), vdupq_n_s64(0)) : \ + vshlq_n_s64(simde__m128i_to_neon_i64(a), ((imm8) & 63)))) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_slli_epi64(a, imm8) \ + ((imm8 < 64) ? wasm_i64x2_shl(simde__m128i_to_private(a).wasm_v128, imm8) : wasm_i64x2_const(0,0)) +#endif +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_slli_epi64(a, imm8) simde_mm_slli_epi64(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_srli_epi16 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + if (HEDLEY_UNLIKELY((imm8 > 15))) { + return simde_mm_setzero_si128(); + } + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u16 = a_.u16 >> SIMDE_CAST_VECTOR_SHIFT_COUNT(8, imm8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.u16[i] = a_.u16[i] >> (imm8 & 0xff); + } + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) + #define simde_mm_srli_epi16(a, imm8) _mm_srli_epi16(a, imm8) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #define simde_mm_srli_epi16(a, imm8) \ + (((imm8) <= 0) ? \ + (a) : \ + simde__m128i_from_neon_u16( \ + ((imm8) > 15) ? \ + vandq_u16(simde__m128i_to_neon_u16(a), vdupq_n_u16(0)) : \ + vshrq_n_u16(simde__m128i_to_neon_u16(a), ((imm8) & 15) | (((imm8) & 15) == 0)))) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_srli_epi16(a, imm8) \ + ((imm8 < 16) ? wasm_u16x8_shr(simde__m128i_to_private(a).wasm_v128, imm8) : wasm_i16x8_const(0,0,0,0,0,0,0,0)) +#elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + #define simde_mm_srli_epi16(a, imm8) \ + ((imm8 & ~15) ? simde_mm_setzero_si128() : simde__m128i_from_altivec_i16(vec_sr(simde__m128i_to_altivec_i16(a), vec_splat_u16(HEDLEY_STATIC_CAST(unsigned short, imm8))))) +#endif +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_srli_epi16(a, imm8) simde_mm_srli_epi16(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_srli_epi32 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + if (HEDLEY_UNLIKELY((imm8 > 31))) { + return simde_mm_setzero_si128(); + } + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = a_.u32 >> SIMDE_CAST_VECTOR_SHIFT_COUNT(8, imm8 & 0xff); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.u32[i] = a_.u32[i] >> (imm8 & 0xff); + } + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) + #define simde_mm_srli_epi32(a, imm8) _mm_srli_epi32(a, imm8) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #define simde_mm_srli_epi32(a, imm8) \ + (((imm8) <= 0) ? \ + (a) : \ + simde__m128i_from_neon_u32( \ + ((imm8) > 31) ? \ + vandq_u32(simde__m128i_to_neon_u32(a), vdupq_n_u32(0)) : \ + vshrq_n_u32(simde__m128i_to_neon_u32(a), ((imm8) & 31) | (((imm8) & 31) == 0)))) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_srli_epi32(a, imm8) \ + ((imm8 < 32) ? wasm_u32x4_shr(simde__m128i_to_private(a).wasm_v128, imm8) : wasm_i32x4_const(0,0,0,0)) +#elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + #define simde_mm_srli_epi32(a, imm8) \ + (__extension__ ({ \ + simde__m128i ret; \ + if ((imm8) <= 0) { \ + ret = a; \ + } else if ((imm8) > 31) { \ + ret = simde_mm_setzero_si128(); \ + } else { \ + ret = simde__m128i_from_altivec_i32( \ + vec_sr(simde__m128i_to_altivec_i32(a), \ + vec_splats(HEDLEY_STATIC_CAST(unsigned int, (imm8) & 31)))); \ + } \ + ret; \ + })) +#endif +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_srli_epi32(a, imm8) simde_mm_srli_epi32(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_srli_epi64 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + if (HEDLEY_UNLIKELY((imm8 & 63) != imm8)) + return simde_mm_setzero_si128(); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u64 = vshlq_u64(a_.neon_u64, vdupq_n_s64(-imm8)); + #else + #if defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && !defined(SIMDE_BUG_GCC_94488) + r_.u64 = a_.u64 >> SIMDE_CAST_VECTOR_SHIFT_COUNT(8, imm8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.u64[i] = a_.u64[i] >> imm8; + } + #endif + #endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE2_NATIVE) + #define simde_mm_srli_epi64(a, imm8) _mm_srli_epi64(a, imm8) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #define simde_mm_srli_epi64(a, imm8) \ + (((imm8) <= 0) ? \ + (a) : \ + simde__m128i_from_neon_u64( \ + ((imm8) > 63) ? \ + vandq_u64(simde__m128i_to_neon_u64(a), vdupq_n_u64(0)) : \ + vshrq_n_u64(simde__m128i_to_neon_u64(a), ((imm8) & 63) | (((imm8) & 63) == 0)))) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_srli_epi64(a, imm8) \ + ((imm8 < 64) ? wasm_u64x2_shr(simde__m128i_to_private(a).wasm_v128, imm8) : wasm_i64x2_const(0,0)) +#endif +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_srli_epi64(a, imm8) simde_mm_srli_epi64(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_store_pd (simde_float64 mem_addr[HEDLEY_ARRAY_PARAM(2)], simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_store_pd(mem_addr, a); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + vst1q_f64(mem_addr, simde__m128d_to_private(a).neon_f64); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + vst1q_s64(HEDLEY_REINTERPRET_CAST(int64_t*, mem_addr), simde__m128d_to_private(a).neon_i64); + #else + simde_memcpy(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m128d), &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_store_pd(mem_addr, a) simde_mm_store_pd(HEDLEY_REINTERPRET_CAST(double*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_store1_pd (simde_float64 mem_addr[HEDLEY_ARRAY_PARAM(2)], simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_store1_pd(mem_addr, a); + #else + simde__m128d_private a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + vst1q_f64(mem_addr, vdupq_laneq_f64(a_.neon_f64, 0)); + #else + mem_addr[0] = a_.f64[0]; + mem_addr[1] = a_.f64[0]; + #endif + #endif +} +#define simde_mm_store_pd1(mem_addr, a) simde_mm_store1_pd(HEDLEY_REINTERPRET_CAST(double*, mem_addr), a) +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_store1_pd(mem_addr, a) simde_mm_store1_pd(HEDLEY_REINTERPRET_CAST(double*, mem_addr), a) + #define _mm_store_pd1(mem_addr, a) simde_mm_store_pd1(HEDLEY_REINTERPRET_CAST(double*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_store_sd (simde_float64* mem_addr, simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_store_sd(mem_addr, a); + #else + simde__m128d_private a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + const simde_float64 v = vgetq_lane_f64(a_.neon_f64, 0); + simde_memcpy(mem_addr, &v, sizeof(v)); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + const int64_t v = vgetq_lane_s64(a_.neon_i64, 0); + simde_memcpy(HEDLEY_REINTERPRET_CAST(int64_t*, mem_addr), &v, sizeof(v)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + wasm_v128_store64_lane(HEDLEY_REINTERPRET_CAST(void*, mem_addr), a_.wasm_v128, 0); + #else + simde_float64 v = a_.f64[0]; + simde_memcpy(mem_addr, &v, sizeof(simde_float64)); + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_store_sd(mem_addr, a) simde_mm_store_sd(HEDLEY_REINTERPRET_CAST(double*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_store_si128 (simde__m128i* mem_addr, simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_store_si128(HEDLEY_STATIC_CAST(__m128i*, mem_addr), a); + #else + simde__m128i_private a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + vst1q_s32(HEDLEY_REINTERPRET_CAST(int32_t*, mem_addr), a_.neon_i32); + #else + simde_memcpy(SIMDE_ALIGN_ASSUME_LIKE(mem_addr, simde__m128i), &a_, sizeof(a_)); + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_store_si128(mem_addr, a) simde_mm_store_si128(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void + simde_mm_storeh_pd (simde_float64* mem_addr, simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_storeh_pd(mem_addr, a); + #else + simde__m128d_private a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + *mem_addr = vgetq_lane_f64(a_.neon_f64, 1); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + wasm_v128_store64_lane(HEDLEY_REINTERPRET_CAST(void*, mem_addr), a_.wasm_v128, 1); + #else + *mem_addr = a_.f64[1]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_storeh_pd(mem_addr, a) simde_mm_storeh_pd(HEDLEY_REINTERPRET_CAST(double*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_storel_epi64 (simde__m128i* mem_addr, simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_storel_epi64(HEDLEY_STATIC_CAST(__m128i*, mem_addr), a); + #else + simde__m128i_private a_ = simde__m128i_to_private(a); + int64_t tmp; + + /* memcpy to prevent aliasing, tmp because we can't take the + * address of a vector element. */ + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + tmp = vgetq_lane_s64(a_.neon_i64, 0); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + #if defined(SIMDE_BUG_GCC_95227) + (void) a_; + #endif + tmp = vec_extract(a_.altivec_i64, 0); + #else + tmp = a_.i64[0]; + #endif + + simde_memcpy(mem_addr, &tmp, sizeof(tmp)); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_storel_epi64(mem_addr, a) simde_mm_storel_epi64(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_storel_pd (simde_float64* mem_addr, simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_storel_pd(mem_addr, a); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + wasm_v128_store64_lane(HEDLEY_REINTERPRET_CAST(void*, mem_addr), simde__m128d_to_wasm_v128(a), 0); + #else + simde__m128d_private a_ = simde__m128d_to_private(a); + + simde_float64 tmp; + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + tmp = vgetq_lane_f64(a_.neon_f64, 0); + #else + tmp = a_.f64[0]; + #endif + simde_memcpy(mem_addr, &tmp, sizeof(tmp)); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_storel_pd(mem_addr, a) simde_mm_storel_pd(HEDLEY_REINTERPRET_CAST(double*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_storer_pd (simde_float64 mem_addr[2], simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_storer_pd(mem_addr, a); + #else + simde__m128d_private a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + vst1q_s64(HEDLEY_REINTERPRET_CAST(int64_t*, mem_addr), vextq_s64(a_.neon_i64, a_.neon_i64, 1)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + a_.wasm_v128 = wasm_i64x2_shuffle(a_.wasm_v128, a_.wasm_v128, 1, 0); + simde_mm_store_pd(mem_addr, simde__m128d_from_private(a_)); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + a_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.f64, a_.f64, 1, 0); + simde_mm_store_pd(mem_addr, simde__m128d_from_private(a_)); + #else + mem_addr[0] = a_.f64[1]; + mem_addr[1] = a_.f64[0]; + #endif + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_storer_pd(mem_addr, a) simde_mm_storer_pd(HEDLEY_REINTERPRET_CAST(double*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_storeu_pd (simde_float64* mem_addr, simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_storeu_pd(mem_addr, a); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + vst1q_f64(mem_addr, simde__m128d_to_private(a).neon_f64); + #else + simde_memcpy(mem_addr, &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_storeu_pd(mem_addr, a) simde_mm_storeu_pd(HEDLEY_REINTERPRET_CAST(double*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_storeu_si128 (void* mem_addr, simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_storeu_si128(HEDLEY_STATIC_CAST(__m128i*, mem_addr), a); + #else + simde_memcpy(mem_addr, &a, sizeof(a)); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_storeu_si128(mem_addr, a) simde_mm_storeu_si128(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_storeu_si16 (void* mem_addr, simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && ( \ + SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0) || \ + HEDLEY_GCC_VERSION_CHECK(11,0,0) || \ + HEDLEY_INTEL_VERSION_CHECK(20,21,1)) + _mm_storeu_si16(mem_addr, a); + #else + int16_t val = simde_x_mm_cvtsi128_si16(a); + simde_memcpy(mem_addr, &val, sizeof(val)); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_storeu_si16(mem_addr, a) simde_mm_storeu_si16(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_storeu_si32 (void* mem_addr, simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && ( \ + SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0) || \ + HEDLEY_GCC_VERSION_CHECK(11,0,0) || \ + HEDLEY_INTEL_VERSION_CHECK(20,21,1)) + _mm_storeu_si32(mem_addr, a); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + wasm_v128_store32_lane(mem_addr, simde__m128i_to_wasm_v128(a), 0); + #else + int32_t val = simde_mm_cvtsi128_si32(a); + simde_memcpy(mem_addr, &val, sizeof(val)); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_storeu_si32(mem_addr, a) simde_mm_storeu_si32(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_storeu_si64 (void* mem_addr, simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && ( \ + SIMDE_DETECT_CLANG_VERSION_CHECK(8,0,0) || \ + HEDLEY_GCC_VERSION_CHECK(11,0,0) || \ + HEDLEY_INTEL_VERSION_CHECK(20,21,1)) + _mm_storeu_si64(mem_addr, a); + #else + int64_t val = simde_mm_cvtsi128_si64(a); + simde_memcpy(mem_addr, &val, sizeof(val)); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_storeu_si64(mem_addr, a) simde_mm_storeu_si64(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_stream_pd (simde_float64 mem_addr[HEDLEY_ARRAY_PARAM(2)], simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_stream_pd(mem_addr, a); + #elif HEDLEY_HAS_BUILTIN(__builtin_nontemporal_store) && ( \ + defined(SIMDE_VECTOR_SUBSCRIPT) || defined(SIMDE_ARM_NEON_A64V8_NATIVE) || \ + defined(SIMDE_WASM_SIMD128_NATIVE) || defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || \ + defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE)) + __builtin_nontemporal_store(a, SIMDE_ALIGN_CAST(__typeof__(a)*, mem_addr)); + #else + simde_mm_store_pd(mem_addr, a); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_stream_pd(mem_addr, a) simde_mm_stream_pd(HEDLEY_REINTERPRET_CAST(double*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_stream_si128 (simde__m128i* mem_addr, simde__m128i a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_ARCH_AMD64) + _mm_stream_si128(HEDLEY_STATIC_CAST(__m128i*, mem_addr), a); + #elif HEDLEY_HAS_BUILTIN(__builtin_nontemporal_store) && ( \ + defined(SIMDE_VECTOR_SUBSCRIPT) || defined(SIMDE_ARM_NEON_A32V7_NATIVE) || \ + defined(SIMDE_WASM_SIMD128_NATIVE) || defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || \ + defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE)) + __builtin_nontemporal_store(a, SIMDE_ALIGN_CAST(__typeof__(a)*, mem_addr)); + #else + simde_mm_store_si128(mem_addr, a); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_stream_si128(mem_addr, a) simde_mm_stream_si128(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_stream_si32 (int32_t* mem_addr, int32_t a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_stream_si32(mem_addr, a); + #elif HEDLEY_HAS_BUILTIN(__builtin_nontemporal_store) + __builtin_nontemporal_store(a, mem_addr); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + vst1q_lane_s32(mem_addr, vdupq_n_s32(a), 0); + #else + *mem_addr = a; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_stream_si32(mem_addr, a) simde_mm_stream_si32(mem_addr, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_stream_si64 (int64_t* mem_addr, int64_t a) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_ARCH_AMD64) && !defined(HEDLEY_MSVC_VERSION) + _mm_stream_si64(SIMDE_CHECKED_REINTERPRET_CAST(long long int*, int64_t*, mem_addr), a); + #elif HEDLEY_HAS_BUILTIN(__builtin_nontemporal_store) + __builtin_nontemporal_store(a, mem_addr); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + vst1_s64(mem_addr, vdup_n_s64(a)); + #else + *mem_addr = a; + #endif +} +#define simde_mm_stream_si64x(mem_addr, a) simde_mm_stream_si64(mem_addr, a) +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) + #define _mm_stream_si64(mem_addr, a) simde_mm_stream_si64(SIMDE_CHECKED_REINTERPRET_CAST(int64_t*, __int64*, mem_addr), a) + #define _mm_stream_si64x(mem_addr, a) simde_mm_stream_si64(SIMDE_CHECKED_REINTERPRET_CAST(int64_t*, __int64*, mem_addr), a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sub_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_sub_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vsubq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_sub(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = a_.i8 - b_.i8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] - b_.i8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sub_epi8(a, b) simde_mm_sub_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sub_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_sub_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vsubq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_sub(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = a_.i16 - b_.i16; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] - b_.i16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sub_epi16(a, b) simde_mm_sub_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sub_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_sub_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vsubq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_sub(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 - b_.i32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] - b_.i32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sub_epi32(a, b) simde_mm_sub_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sub_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_sub_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vsubq_s64(a_.neon_i64, b_.neon_i64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_sub(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 - b_.i64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] - b_.i64[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sub_epi64(a, b) simde_mm_sub_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_sub_epu32 (simde__m128i a, simde__m128i b) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = a_.u32 - b_.u32; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vsubq_u32(a_.neon_u32, b_.neon_u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] - b_.u32[i]; + } + #endif + + return simde__m128i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_sub_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_sub_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.f64 = a_.f64 - b_.f64; + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vsubq_f64(a_.neon_f64, b_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_sub(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = a_.f64[i] - b_.f64[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sub_pd(a, b) simde_mm_sub_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_sub_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_sub_sd(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_sd(a, simde_mm_sub_pd(a, b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_sd(a, simde_mm_sub_pd(simde_x_mm_broadcastlow_pd(a), simde_x_mm_broadcastlow_pd(b))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + r_.f64[0] = a_.f64[0] - b_.f64[0]; + r_.f64[1] = a_.f64[1]; + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sub_sd(a, b) simde_mm_sub_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_sub_si64 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSE2_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_sub_si64(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 - b_.i64; + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vsub_s64(a_.neon_i64, b_.neon_i64); + #else + r_.i64[0] = a_.i64[0] - b_.i64[0]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_sub_si64(a, b) simde_mm_sub_si64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_subs_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_subs_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vqsubq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_sub_sat(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = simde_math_subs_i8(a_.i8[i], b_.i8[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_subs_epi8(a, b) simde_mm_subs_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_subs_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_subs_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vqsubq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_sub_sat(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = simde_math_subs_i16(a_.i16[i], b_.i16[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_subs_epi16(a, b) simde_mm_subs_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_subs_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_subs_epu8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vqsubq_u8(a_.neon_u8, b_.neon_u8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u8x16_sub_sat(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_u8 = vec_subs(a_.altivec_u8, b_.altivec_u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = simde_math_subs_u8(a_.u8[i], b_.u8[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_subs_epu8(a, b) simde_mm_subs_epu8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_subs_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_subs_epu16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vqsubq_u16(a_.neon_u16, b_.neon_u16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u16x8_sub_sat(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_u16 = vec_subs(a_.altivec_u16, b_.altivec_u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = simde_math_subs_u16(a_.u16[i], b_.u16[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_subs_epu16(a, b) simde_mm_subs_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_ucomieq_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_ucomieq_sd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + int r; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint64x2_t a_not_nan = vceqq_f64(a_.neon_f64, a_.neon_f64); + uint64x2_t b_not_nan = vceqq_f64(b_.neon_f64, b_.neon_f64); + uint64x2_t a_or_b_nan = vreinterpretq_u64_u32(vmvnq_u32(vreinterpretq_u32_u64(vandq_u64(a_not_nan, b_not_nan)))); + uint64x2_t a_eq_b = vceqq_f64(a_.neon_f64, b_.neon_f64); + r = !!(vgetq_lane_u64(vorrq_u64(a_or_b_nan, a_eq_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f64x2_extract_lane(a_.wasm_v128, 0) == wasm_f64x2_extract_lane(b_.wasm_v128, 0); + #elif defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = a_.f64[0] == b_.f64[0]; + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = a_.f64[0] == b_.f64[0]; + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_ucomieq_sd(a, b) simde_mm_ucomieq_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_ucomige_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_ucomige_sd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + int r; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint64x2_t a_not_nan = vceqq_f64(a_.neon_f64, a_.neon_f64); + uint64x2_t b_not_nan = vceqq_f64(b_.neon_f64, b_.neon_f64); + uint64x2_t a_and_b_not_nan = vandq_u64(a_not_nan, b_not_nan); + uint64x2_t a_ge_b = vcgeq_f64(a_.neon_f64, b_.neon_f64); + r = !!(vgetq_lane_u64(vandq_u64(a_and_b_not_nan, a_ge_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f64x2_extract_lane(a_.wasm_v128, 0) >= wasm_f64x2_extract_lane(b_.wasm_v128, 0); + #elif defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = a_.f64[0] >= b_.f64[0]; + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = a_.f64[0] >= b_.f64[0]; + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_ucomige_sd(a, b) simde_mm_ucomige_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_ucomigt_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_ucomigt_sd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + int r; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint64x2_t a_not_nan = vceqq_f64(a_.neon_f64, a_.neon_f64); + uint64x2_t b_not_nan = vceqq_f64(b_.neon_f64, b_.neon_f64); + uint64x2_t a_and_b_not_nan = vandq_u64(a_not_nan, b_not_nan); + uint64x2_t a_gt_b = vcgtq_f64(a_.neon_f64, b_.neon_f64); + r = !!(vgetq_lane_u64(vandq_u64(a_and_b_not_nan, a_gt_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f64x2_extract_lane(a_.wasm_v128, 0) > wasm_f64x2_extract_lane(b_.wasm_v128, 0); + #elif defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = a_.f64[0] > b_.f64[0]; + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = a_.f64[0] > b_.f64[0]; + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_ucomigt_sd(a, b) simde_mm_ucomigt_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_ucomile_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_ucomile_sd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + int r; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint64x2_t a_not_nan = vceqq_f64(a_.neon_f64, a_.neon_f64); + uint64x2_t b_not_nan = vceqq_f64(b_.neon_f64, b_.neon_f64); + uint64x2_t a_or_b_nan = vreinterpretq_u64_u32(vmvnq_u32(vreinterpretq_u32_u64(vandq_u64(a_not_nan, b_not_nan)))); + uint64x2_t a_le_b = vcleq_f64(a_.neon_f64, b_.neon_f64); + r = !!(vgetq_lane_u64(vorrq_u64(a_or_b_nan, a_le_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f64x2_extract_lane(a_.wasm_v128, 0) <= wasm_f64x2_extract_lane(b_.wasm_v128, 0); + #elif defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = a_.f64[0] <= b_.f64[0]; + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = a_.f64[0] <= b_.f64[0]; + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_ucomile_sd(a, b) simde_mm_ucomile_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_ucomilt_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_ucomilt_sd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + int r; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint64x2_t a_not_nan = vceqq_f64(a_.neon_f64, a_.neon_f64); + uint64x2_t b_not_nan = vceqq_f64(b_.neon_f64, b_.neon_f64); + uint64x2_t a_or_b_nan = vreinterpretq_u64_u32(vmvnq_u32(vreinterpretq_u32_u64(vandq_u64(a_not_nan, b_not_nan)))); + uint64x2_t a_lt_b = vcltq_f64(a_.neon_f64, b_.neon_f64); + r = !!(vgetq_lane_u64(vorrq_u64(a_or_b_nan, a_lt_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f64x2_extract_lane(a_.wasm_v128, 0) < wasm_f64x2_extract_lane(b_.wasm_v128, 0); + #elif defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = a_.f64[0] < b_.f64[0]; + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = a_.f64[0] < b_.f64[0]; + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_ucomilt_sd(a, b) simde_mm_ucomilt_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_ucomineq_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_ucomineq_sd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + int r; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + uint64x2_t a_not_nan = vceqq_f64(a_.neon_f64, a_.neon_f64); + uint64x2_t b_not_nan = vceqq_f64(b_.neon_f64, b_.neon_f64); + uint64x2_t a_and_b_not_nan = vandq_u64(a_not_nan, b_not_nan); + uint64x2_t a_neq_b = vreinterpretq_u64_u32(vmvnq_u32(vreinterpretq_u32_u64(vceqq_f64(a_.neon_f64, b_.neon_f64)))); + r = !!(vgetq_lane_u64(vandq_u64(a_and_b_not_nan, a_neq_b), 0) != 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + return wasm_f64x2_extract_lane(a_.wasm_v128, 0) != wasm_f64x2_extract_lane(b_.wasm_v128, 0); + #elif defined(SIMDE_HAVE_FENV_H) + fenv_t envp; + int x = feholdexcept(&envp); + r = a_.f64[0] != b_.f64[0]; + if (HEDLEY_LIKELY(x == 0)) + fesetenv(&envp); + #else + r = a_.f64[0] != b_.f64[0]; + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_ucomineq_sd(a, b) simde_mm_ucomineq_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_lfence (void) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_lfence(); + #else + simde_mm_sfence(); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_lfence() simde_mm_lfence() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +void +simde_mm_mfence (void) { + #if defined(SIMDE_X86_SSE2_NATIVE) + _mm_mfence(); + #else + simde_mm_sfence(); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_mfence() simde_mm_mfence() +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_unpackhi_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_unpackhi_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i8 = vzip2q_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int8x8_t a1 = vreinterpret_s8_s16(vget_high_s16(a_.neon_i16)); + int8x8_t b1 = vreinterpret_s8_s16(vget_high_s16(b_.neon_i16)); + int8x8x2_t result = vzip_s8(a1, b1); + r_.neon_i8 = vcombine_s8(result.val[0], result.val[1]); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_shuffle(a_.wasm_v128, b_.wasm_v128, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i8 = SIMDE_SHUFFLE_VECTOR_(8, 16, a_.i8, b_.i8, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < ((sizeof(r_) / sizeof(r_.i8[0])) / 2) ; i++) { + r_.i8[(i * 2)] = a_.i8[i + ((sizeof(r_) / sizeof(r_.i8[0])) / 2)]; + r_.i8[(i * 2) + 1] = b_.i8[i + ((sizeof(r_) / sizeof(r_.i8[0])) / 2)]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_unpackhi_epi8(a, b) simde_mm_unpackhi_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_unpackhi_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_unpackhi_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i16 = vzip2q_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int16x4_t a1 = vget_high_s16(a_.neon_i16); + int16x4_t b1 = vget_high_s16(b_.neon_i16); + int16x4x2_t result = vzip_s16(a1, b1); + r_.neon_i16 = vcombine_s16(result.val[0], result.val[1]); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_shuffle(a_.wasm_v128, b_.wasm_v128, 4, 12, 5, 13, 6, 14, 7, 15); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i16 = SIMDE_SHUFFLE_VECTOR_(16, 16, a_.i16, b_.i16, 4, 12, 5, 13, 6, 14, 7, 15); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < ((sizeof(r_) / sizeof(r_.i16[0])) / 2) ; i++) { + r_.i16[(i * 2)] = a_.i16[i + ((sizeof(r_) / sizeof(r_.i16[0])) / 2)]; + r_.i16[(i * 2) + 1] = b_.i16[i + ((sizeof(r_) / sizeof(r_.i16[0])) / 2)]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_unpackhi_epi16(a, b) simde_mm_unpackhi_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_unpackhi_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_unpackhi_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i32 = vzip2q_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int32x2_t a1 = vget_high_s32(a_.neon_i32); + int32x2_t b1 = vget_high_s32(b_.neon_i32); + int32x2x2_t result = vzip_s32(a1, b1); + r_.neon_i32 = vcombine_s32(result.val[0], result.val[1]); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_shuffle(a_.wasm_v128, b_.wasm_v128, 2, 6, 3, 7); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.i32, b_.i32, 2, 6, 3, 7); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < ((sizeof(r_) / sizeof(r_.i32[0])) / 2) ; i++) { + r_.i32[(i * 2)] = a_.i32[i + ((sizeof(r_) / sizeof(r_.i32[0])) / 2)]; + r_.i32[(i * 2) + 1] = b_.i32[i + ((sizeof(r_) / sizeof(r_.i32[0])) / 2)]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_unpackhi_epi32(a, b) simde_mm_unpackhi_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_unpackhi_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_unpackhi_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int64x1_t a_h = vget_high_s64(a_.neon_i64); + int64x1_t b_h = vget_high_s64(b_.neon_i64); + r_.neon_i64 = vcombine_s64(a_h, b_h); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_shuffle(a_.wasm_v128, b_.wasm_v128, 1, 3); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.i64, b_.i64, 1, 3); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < ((sizeof(r_) / sizeof(r_.i64[0])) / 2) ; i++) { + r_.i64[(i * 2)] = a_.i64[i + ((sizeof(r_) / sizeof(r_.i64[0])) / 2)]; + r_.i64[(i * 2) + 1] = b_.i64[i + ((sizeof(r_) / sizeof(r_.i64[0])) / 2)]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_unpackhi_epi64(a, b) simde_mm_unpackhi_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_unpackhi_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_unpackhi_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vzip2q_f64(a_.neon_f64, b_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_shuffle(a_.wasm_v128, b_.wasm_v128, 1, 3); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.f64, b_.f64, 1, 3); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < ((sizeof(r_) / sizeof(r_.f64[0])) / 2) ; i++) { + r_.f64[(i * 2)] = a_.f64[i + ((sizeof(r_) / sizeof(r_.f64[0])) / 2)]; + r_.f64[(i * 2) + 1] = b_.f64[i + ((sizeof(r_) / sizeof(r_.f64[0])) / 2)]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_unpackhi_pd(a, b) simde_mm_unpackhi_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_unpacklo_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_unpacklo_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i8 = vzip1q_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int8x8_t a1 = vreinterpret_s8_s16(vget_low_s16(a_.neon_i16)); + int8x8_t b1 = vreinterpret_s8_s16(vget_low_s16(b_.neon_i16)); + int8x8x2_t result = vzip_s8(a1, b1); + r_.neon_i8 = vcombine_s8(result.val[0], result.val[1]); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_shuffle(a_.wasm_v128, b_.wasm_v128, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i8 = SIMDE_SHUFFLE_VECTOR_(8, 16, a_.i8, b_.i8, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < ((sizeof(r_) / sizeof(r_.i8[0])) / 2) ; i++) { + r_.i8[(i * 2)] = a_.i8[i]; + r_.i8[(i * 2) + 1] = b_.i8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_unpacklo_epi8(a, b) simde_mm_unpacklo_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_unpacklo_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_unpacklo_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i16 = vzip1q_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int16x4_t a1 = vget_low_s16(a_.neon_i16); + int16x4_t b1 = vget_low_s16(b_.neon_i16); + int16x4x2_t result = vzip_s16(a1, b1); + r_.neon_i16 = vcombine_s16(result.val[0], result.val[1]); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_shuffle(a_.wasm_v128, b_.wasm_v128, 0, 8, 1, 9, 2, 10, 3, 11); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i16 = SIMDE_SHUFFLE_VECTOR_(16, 16, a_.i16, b_.i16, 0, 8, 1, 9, 2, 10, 3, 11); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < ((sizeof(r_) / sizeof(r_.i16[0])) / 2) ; i++) { + r_.i16[(i * 2)] = a_.i16[i]; + r_.i16[(i * 2) + 1] = b_.i16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_unpacklo_epi16(a, b) simde_mm_unpacklo_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_unpacklo_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_unpacklo_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i32 = vzip1q_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int32x2_t a1 = vget_low_s32(a_.neon_i32); + int32x2_t b1 = vget_low_s32(b_.neon_i32); + int32x2x2_t result = vzip_s32(a1, b1); + r_.neon_i32 = vcombine_s32(result.val[0], result.val[1]); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_shuffle(a_.wasm_v128, b_.wasm_v128, 0, 4, 1, 5); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.i32, b_.i32, 0, 4, 1, 5); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < ((sizeof(r_) / sizeof(r_.i32[0])) / 2) ; i++) { + r_.i32[(i * 2)] = a_.i32[i]; + r_.i32[(i * 2) + 1] = b_.i32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_unpacklo_epi32(a, b) simde_mm_unpacklo_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_unpacklo_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_unpacklo_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int64x1_t a_l = vget_low_s64(a_.neon_i64); + int64x1_t b_l = vget_low_s64(b_.neon_i64); + r_.neon_i64 = vcombine_s64(a_l, b_l); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_shuffle(a_.wasm_v128, b_.wasm_v128, 0, 2); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.i64, b_.i64, 0, 2); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < ((sizeof(r_) / sizeof(r_.i64[0])) / 2) ; i++) { + r_.i64[(i * 2)] = a_.i64[i]; + r_.i64[(i * 2) + 1] = b_.i64[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_unpacklo_epi64(a, b) simde_mm_unpacklo_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_unpacklo_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_unpacklo_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vzip1q_f64(a_.neon_f64, b_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_shuffle(a_.wasm_v128, b_.wasm_v128, 0, 2); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.f64, b_.f64, 0, 2); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < ((sizeof(r_) / sizeof(r_.f64[0])) / 2) ; i++) { + r_.f64[(i * 2)] = a_.f64[i]; + r_.f64[(i * 2) + 1] = b_.f64[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_unpacklo_pd(a, b) simde_mm_unpacklo_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_x_mm_negate_pd(simde__m128d a) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return simde_mm_xor_pd(a, _mm_set1_pd(SIMDE_FLOAT64_C(-0.0))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) && \ + (!defined(HEDLEY_GCC_VERSION) || HEDLEY_GCC_VERSION_CHECK(8,1,0)) + r_.altivec_f64 = vec_neg(a_.altivec_f64); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vnegq_f64(a_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_neg(a_.wasm_v128); + #elif defined(SIMDE_VECTOR_NEGATE) + r_.f64 = -a_.f64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = -a_.f64[i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_xor_si128 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE2_NATIVE) + return _mm_xor_si128(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = veorq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i32 = vec_xor(a_.altivec_i32, b_.altivec_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_xor(b_.wasm_v128, a_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = a_.i32f ^ b_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = a_.i32f[i] ^ b_.i32f[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _mm_xor_si128(a, b) simde_mm_xor_si128(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_not_si128 (simde__m128i a) { + #if defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_ternarylogic_epi32(a, a, a, 0x55); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vmvnq_s32(a_.neon_i32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i32 = vec_nor(a_.altivec_i32, a_.altivec_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_not(a_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = ~a_.i32f; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = ~(a_.i32f[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +#define SIMDE_MM_SHUFFLE2(x, y) (((x) << 1) | (y)) +#if defined(SIMDE_X86_SSE2_ENABLE_NATIVE_ALIASES) + #define _MM_SHUFFLE2(x, y) SIMDE_MM_SHUFFLE2(x, y) +#endif + +SIMDE_END_DECLS_ + +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_SSE2_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse3.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse3.h new file mode 100644 index 00000000..db2683c3 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse3.h @@ -0,0 +1,515 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2017-2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_SSE3_H) +#define SIMDE_X86_SSE3_H + +#include "sse2.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_deinterleaveeven_epi16 (simde__m128i a, simde__m128i b) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i16 = vuzp1q_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int16x8x2_t t = vuzpq_s16(a_.neon_i16, b_.neon_i16); + r_.neon_i16 = t.val[0]; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_shuffle(a_.wasm_v128, b_.wasm_v128, 0, 2, 4, 6, 8, 10, 12, 14); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i16 = SIMDE_SHUFFLE_VECTOR_(16, 16, a_.i16, b_.i16, 0, 2, 4, 6, 8, 10, 12, 14); + #else + const size_t halfway_point = (sizeof(r_.i16) / sizeof(r_.i16[0])) / 2; + for(size_t i = 0 ; i < halfway_point ; i++) { + r_.i16[i] = a_.i16[2 * i]; + r_.i16[i + halfway_point] = b_.i16[2 * i]; + } + #endif + + return simde__m128i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_deinterleaveodd_epi16 (simde__m128i a, simde__m128i b) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i16 = vuzp2q_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int16x8x2_t t = vuzpq_s16(a_.neon_i16, b_.neon_i16); + r_.neon_i16 = t.val[1]; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_shuffle(a_.wasm_v128, b_.wasm_v128, 1, 3, 5, 7, 9, 11, 13, 15); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i16 = SIMDE_SHUFFLE_VECTOR_(16, 16, a_.i16, b_.i16, 1, 3, 5, 7, 9, 11, 13, 15); + #else + const size_t halfway_point = (sizeof(r_.i16) / sizeof(r_.i16[0])) / 2; + for(size_t i = 0 ; i < halfway_point ; i++) { + r_.i16[i] = a_.i16[2 * i + 1]; + r_.i16[i + halfway_point] = b_.i16[2 * i + 1]; + } + #endif + + return simde__m128i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_deinterleaveeven_epi32 (simde__m128i a, simde__m128i b) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i32 = vuzp1q_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int32x4x2_t t = vuzpq_s32(a_.neon_i32, b_.neon_i32); + r_.neon_i32 = t.val[0]; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_shuffle(a_.wasm_v128, b_.wasm_v128, 0, 2, 4, 6); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.i32, b_.i32, 0, 2, 4, 6); + #else + const size_t halfway_point = (sizeof(r_.i32) / sizeof(r_.i32[0])) / 2; + for(size_t i = 0 ; i < halfway_point ; i++) { + r_.i32[i] = a_.i32[2 * i]; + r_.i32[i + halfway_point] = b_.i32[2 * i]; + } + #endif + + return simde__m128i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_deinterleaveodd_epi32 (simde__m128i a, simde__m128i b) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i32 = vuzp2q_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int32x4x2_t t = vuzpq_s32(a_.neon_i32, b_.neon_i32); + r_.neon_i32 = t.val[1]; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_shuffle(a_.wasm_v128, b_.wasm_v128, 1, 3, 5, 7); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.i32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.i32, b_.i32, 1, 3, 5, 7); + #else + const size_t halfway_point = (sizeof(r_.i32) / sizeof(r_.i32[0])) / 2; + for(size_t i = 0 ; i < halfway_point ; i++) { + r_.i32[i] = a_.i32[2 * i + 1]; + r_.i32[i + halfway_point] = b_.i32[2 * i + 1]; + } + #endif + + return simde__m128i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_x_mm_deinterleaveeven_ps (simde__m128 a, simde__m128 b) { + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f32 = vuzp1q_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32x4x2_t t = vuzpq_f32(a_.neon_f32, b_.neon_f32); + r_.neon_f32 = t.val[0]; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_shuffle(a_.wasm_v128, b_.wasm_v128, 0, 2, 4, 6); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.f32, b_.f32, 0, 2, 4, 6); + #else + const size_t halfway_point = (sizeof(r_.f32) / sizeof(r_.f32[0])) / 2; + for(size_t i = 0 ; i < halfway_point ; i++) { + r_.f32[i] = a_.f32[2 * i]; + r_.f32[i + halfway_point] = b_.f32[2 * i]; + } + #endif + + return simde__m128_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_x_mm_deinterleaveodd_ps (simde__m128 a, simde__m128 b) { + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f32 = vuzp2q_f32(a_.neon_f32, b_.neon_f32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32x4x2_t t = vuzpq_f32(a_.neon_f32, b_.neon_f32); + r_.neon_f32 = t.val[1]; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_shuffle(a_.wasm_v128, b_.wasm_v128, 1, 3, 5, 7); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.f32, b_.f32, 1, 3, 5, 7); + #else + const size_t halfway_point = (sizeof(r_.f32) / sizeof(r_.f32[0])) / 2; + for(size_t i = 0 ; i < halfway_point ; i++) { + r_.f32[i] = a_.f32[2 * i + 1]; + r_.f32[i + halfway_point] = b_.f32[2 * i + 1]; + } + #endif + + return simde__m128_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_x_mm_deinterleaveeven_pd (simde__m128d a, simde__m128d b) { + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vuzp1q_f64(a_.neon_f64, b_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_shuffle(a_.wasm_v128, b_.wasm_v128, 0, 2); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.f64, b_.f64, 0, 2); + #else + const size_t halfway_point = (sizeof(r_.f64) / sizeof(r_.f64[0])) / 2; + for(size_t i = 0 ; i < halfway_point ; i++) { + r_.f64[i] = a_.f64[2 * i]; + r_.f64[i + halfway_point] = b_.f64[2 * i]; + } + #endif + + return simde__m128d_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_x_mm_deinterleaveodd_pd (simde__m128d a, simde__m128d b) { + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vuzp2q_f64(a_.neon_f64, b_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_shuffle(a_.wasm_v128, b_.wasm_v128, 1, 3); + #elif defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.f64, b_.f64, 1, 3); + #else + const size_t halfway_point = (sizeof(r_.f64) / sizeof(r_.f64[0])) / 2; + for(size_t i = 0 ; i < halfway_point ; i++) { + r_.f64[i] = a_.f64[2 * i + 1]; + r_.f64[i + halfway_point] = b_.f64[2 * i + 1]; + } + #endif + + return simde__m128d_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_addsub_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE3_NATIVE) + return _mm_addsub_pd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + float64x2_t rs = vsubq_f64(a_.neon_f64, b_.neon_f64); + float64x2_t ra = vaddq_f64(a_.neon_f64, b_.neon_f64); + return vcombine_f64(vget_low_f64(rs), vget_high_f64(ra)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.f64 - b_.f64, a_.f64 + b_.f64, 0, 3); + #else + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i += 2) { + r_.f64[ i ] = a_.f64[ i ] - b_.f64[ i ]; + r_.f64[1 + i] = a_.f64[1 + i] + b_.f64[1 + i]; + } + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE3_ENABLE_NATIVE_ALIASES) +# define _mm_addsub_pd(a, b) simde_mm_addsub_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_addsub_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE3_NATIVE) + return _mm_addsub_ps(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + float32x4_t rs = vsubq_f32(a_.neon_f32, b_.neon_f32); + float32x4_t ra = vaddq_f32(a_.neon_f32, b_.neon_f32); + return vtrn2q_f32(vreinterpretq_f32_s32(vrev64q_s32(vreinterpretq_s32_f32(rs))), ra); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.f32 - b_.f32, a_.f32 + b_.f32, 0, 5, 2, 7); + #else + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i += 2) { + r_.f32[ i ] = a_.f32[ i ] - b_.f32[ i ]; + r_.f32[1 + i] = a_.f32[1 + i] + b_.f32[1 + i]; + } + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE3_ENABLE_NATIVE_ALIASES) +# define _mm_addsub_ps(a, b) simde_mm_addsub_ps((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_hadd_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE3_NATIVE) + return _mm_hadd_pd(a, b); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return simde__m128d_from_neon_f64(vpaddq_f64(simde__m128d_to_neon_f64(a), simde__m128d_to_neon_f64(b))); + #else + return simde_mm_add_pd(simde_x_mm_deinterleaveeven_pd(a, b), simde_x_mm_deinterleaveodd_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_SSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hadd_pd(a, b) simde_mm_hadd_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_hadd_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE3_NATIVE) + return _mm_hadd_ps(a, b); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return simde__m128_from_neon_f32(vpaddq_f32(simde__m128_to_neon_f32(a), simde__m128_to_neon_f32(b))); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32x4x2_t t = vuzpq_f32(simde__m128_to_neon_f32(a), simde__m128_to_neon_f32(b)); + return simde__m128_from_neon_f32(vaddq_f32(t.val[0], t.val[1])); + #else + return simde_mm_add_ps(simde_x_mm_deinterleaveeven_ps(a, b), simde_x_mm_deinterleaveodd_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_SSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hadd_ps(a, b) simde_mm_hadd_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_hsub_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE3_NATIVE) + return _mm_hsub_pd(a, b); + #else + return simde_mm_sub_pd(simde_x_mm_deinterleaveeven_pd(a, b), simde_x_mm_deinterleaveodd_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_SSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hsub_pd(a, b) simde_mm_hsub_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_hsub_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE3_NATIVE) + return _mm_hsub_ps(a, b); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + float32x4x2_t t = vuzpq_f32(simde__m128_to_neon_f32(a), simde__m128_to_neon_f32(b)); + return simde__m128_from_neon_f32(vaddq_f32(t.val[0], vnegq_f32(t.val[1]))); + #else + return simde_mm_sub_ps(simde_x_mm_deinterleaveeven_ps(a, b), simde_x_mm_deinterleaveodd_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_SSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hsub_ps(a, b) simde_mm_hsub_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_lddqu_si128 (simde__m128i const* mem_addr) { + #if defined(SIMDE_X86_SSE3_NATIVE) + return _mm_lddqu_si128(mem_addr); + #else + simde__m128i_private r_; + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vld1q_s32(HEDLEY_REINTERPRET_CAST(int32_t const*, mem_addr)); + #else + simde_memcpy(&r_, mem_addr, sizeof(r_)); + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE3_ENABLE_NATIVE_ALIASES) +# define _mm_lddqu_si128(mem_addr) simde_mm_lddqu_si128(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_loaddup_pd (simde_float64 const* mem_addr) { + #if defined(SIMDE_X86_SSE3_NATIVE) + return _mm_loaddup_pd(mem_addr); + #else + simde__m128d_private r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vdupq_n_f64(*mem_addr); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vdupq_n_s64(*HEDLEY_REINTERPRET_CAST(int64_t const*, mem_addr)); + #else + r_.f64[0] = *mem_addr; + r_.f64[1] = *mem_addr; + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE3_ENABLE_NATIVE_ALIASES) +# define _mm_loaddup_pd(mem_addr) simde_mm_loaddup_pd(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_movedup_pd (simde__m128d a) { + #if defined(SIMDE_X86_SSE3_NATIVE) + return _mm_movedup_pd(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vdupq_laneq_f64(a_.neon_f64, 0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_shuffle(a_.wasm_v128, a_.wasm_v128, 0, 0); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && defined(SIMDE_SHUFFLE_VECTOR_) + r_.f64 = SIMDE_SHUFFLE_VECTOR_(64, 16, a_.f64, a_.f64, 0, 0); + #else + r_.f64[0] = a_.f64[0]; + r_.f64[1] = a_.f64[0]; + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE3_ENABLE_NATIVE_ALIASES) +# define _mm_movedup_pd(a) simde_mm_movedup_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_movehdup_ps (simde__m128 a) { + #if defined(SIMDE_X86_SSE3_NATIVE) + return _mm_movehdup_ps(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f32 = vtrn2q_f32(a_.neon_f32, a_.neon_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_shuffle(a_.wasm_v128, a_.wasm_v128, 1, 1, 3, 3); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.f32, a_.f32, 1, 1, 3, 3); + #else + r_.f32[0] = a_.f32[1]; + r_.f32[1] = a_.f32[1]; + r_.f32[2] = a_.f32[3]; + r_.f32[3] = a_.f32[3]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE3_ENABLE_NATIVE_ALIASES) +# define _mm_movehdup_ps(a) simde_mm_movehdup_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_moveldup_ps (simde__m128 a) { + #if defined(SIMDE__SSE3_NATIVE) + return _mm_moveldup_ps(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f32 = vtrn1q_f32(a_.neon_f32, a_.neon_f32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_shuffle(a_.wasm_v128, a_.wasm_v128, 0, 0, 2, 2); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_SHUFFLE_VECTOR_) + r_.f32 = SIMDE_SHUFFLE_VECTOR_(32, 16, a_.f32, a_.f32, 0, 0, 2, 2); + #else + r_.f32[0] = a_.f32[0]; + r_.f32[1] = a_.f32[0]; + r_.f32[2] = a_.f32[2]; + r_.f32[3] = a_.f32[2]; + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE3_ENABLE_NATIVE_ALIASES) +# define _mm_moveldup_ps(a) simde_mm_moveldup_ps(a) +#endif + +SIMDE_END_DECLS_ + +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_SSE3_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse4.1.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse4.1.h new file mode 100644 index 00000000..15a197b9 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse4.1.h @@ -0,0 +1,2367 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2017-2020 Evan Nemerson + */ + +#include "sse.h" +#if !defined(SIMDE_X86_SSE4_1_H) +#define SIMDE_X86_SSE4_1_H + +#include "ssse3.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#if !defined(SIMDE_X86_SSE4_1_NATIVE) && defined(SIMDE_ENABLE_NATIVE_ALIASES) +# define SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_blend_epi16 (simde__m128i a, simde__m128i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = ((imm8 >> i) & 1) ? b_.u16[i] : a_.u16[i]; + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) + #define simde_mm_blend_epi16(a, b, imm8) _mm_blend_epi16(a, b, imm8) +#elif defined(SIMDE_SHUFFLE_VECTOR_) + #define simde_mm_blend_epi16(a, b, imm8) \ + (__extension__ ({ \ + simde__m128i_private \ + simde_mm_blend_epi16_a_ = simde__m128i_to_private(a), \ + simde_mm_blend_epi16_b_ = simde__m128i_to_private(b), \ + simde_mm_blend_epi16_r_; \ + \ + simde_mm_blend_epi16_r_.i16 = \ + SIMDE_SHUFFLE_VECTOR_( \ + 16, 16, \ + simde_mm_blend_epi16_a_.i16, \ + simde_mm_blend_epi16_b_.i16, \ + ((imm8) & (1 << 0)) ? 8 : 0, \ + ((imm8) & (1 << 1)) ? 9 : 1, \ + ((imm8) & (1 << 2)) ? 10 : 2, \ + ((imm8) & (1 << 3)) ? 11 : 3, \ + ((imm8) & (1 << 4)) ? 12 : 4, \ + ((imm8) & (1 << 5)) ? 13 : 5, \ + ((imm8) & (1 << 6)) ? 14 : 6, \ + ((imm8) & (1 << 7)) ? 15 : 7 \ + ); \ + \ + simde__m128i_from_private(simde_mm_blend_epi16_r_); \ + })) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_blend_epi16 + #define _mm_blend_epi16(a, b, imm8) simde_mm_blend_epi16(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_blend_pd (simde__m128d a, simde__m128d b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = ((imm8 >> i) & 1) ? b_.f64[i] : a_.f64[i]; + } + return simde__m128d_from_private(r_); +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) + #define simde_mm_blend_pd(a, b, imm8) _mm_blend_pd(a, b, imm8) +#elif defined(SIMDE_SHUFFLE_VECTOR_) + #define simde_mm_blend_pd(a, b, imm8) \ + (__extension__ ({ \ + simde__m128d_private \ + simde_mm_blend_pd_a_ = simde__m128d_to_private(a), \ + simde_mm_blend_pd_b_ = simde__m128d_to_private(b), \ + simde_mm_blend_pd_r_; \ + \ + simde_mm_blend_pd_r_.f64 = \ + SIMDE_SHUFFLE_VECTOR_( \ + 64, 16, \ + simde_mm_blend_pd_a_.f64, \ + simde_mm_blend_pd_b_.f64, \ + ((imm8) & (1 << 0)) ? 2 : 0, \ + ((imm8) & (1 << 1)) ? 3 : 1 \ + ); \ + \ + simde__m128d_from_private(simde_mm_blend_pd_r_); \ + })) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_blend_pd + #define _mm_blend_pd(a, b, imm8) simde_mm_blend_pd(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_blend_ps (simde__m128 a, simde__m128 b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = ((imm8 >> i) & 1) ? b_.f32[i] : a_.f32[i]; + } + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) +# define simde_mm_blend_ps(a, b, imm8) _mm_blend_ps(a, b, imm8) +#elif defined(SIMDE_SHUFFLE_VECTOR_) + #define simde_mm_blend_ps(a, b, imm8) \ + (__extension__ ({ \ + simde__m128_private \ + simde_mm_blend_ps_a_ = simde__m128_to_private(a), \ + simde_mm_blend_ps_b_ = simde__m128_to_private(b), \ + simde_mm_blend_ps_r_; \ + \ + simde_mm_blend_ps_r_.f32 = \ + SIMDE_SHUFFLE_VECTOR_( \ + 32, 16, \ + simde_mm_blend_ps_a_.f32, \ + simde_mm_blend_ps_b_.f32, \ + ((imm8) & (1 << 0)) ? 4 : 0, \ + ((imm8) & (1 << 1)) ? 5 : 1, \ + ((imm8) & (1 << 2)) ? 6 : 2, \ + ((imm8) & (1 << 3)) ? 7 : 3 \ + ); \ + \ + simde__m128_from_private(simde_mm_blend_ps_r_); \ + })) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_blend_ps + #define _mm_blend_ps(a, b, imm8) simde_mm_blend_ps(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_blendv_epi8 (simde__m128i a, simde__m128i b, simde__m128i mask) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_blendv_epi8(a, b, mask); + #elif defined(SIMDE_X86_SSE2_NATIVE) + __m128i m = _mm_cmpgt_epi8(_mm_setzero_si128(), mask); + return _mm_xor_si128(_mm_subs_epu8(_mm_xor_si128(a, b), m), b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + mask_ = simde__m128i_to_private(mask); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + /* Use a signed shift right to create a mask with the sign bit */ + mask_.neon_i8 = vshrq_n_s8(mask_.neon_i8, 7); + r_.neon_i8 = vbslq_s8(mask_.neon_u8, b_.neon_i8, a_.neon_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m = wasm_i8x16_shr(mask_.wasm_v128, 7); + r_.wasm_v128 = wasm_v128_bitselect(b_.wasm_v128, a_.wasm_v128, m); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i8 = vec_sel(a_.altivec_i8, b_.altivec_i8, vec_cmplt(mask_.altivec_i8, vec_splat_s8(0))); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + /* https://software.intel.com/en-us/forums/intel-c-compiler/topic/850087 */ + #if defined(HEDLEY_INTEL_VERSION_CHECK) + __typeof__(mask_.i8) z = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + mask_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(mask_.i8), mask_.i8 < z); + #else + mask_.i8 >>= (CHAR_BIT * sizeof(mask_.i8[0])) - 1; + #endif + + r_.i8 = (mask_.i8 & b_.i8) | (~mask_.i8 & a_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + int8_t m = mask_.i8[i] >> 7; + r_.i8[i] = (m & b_.i8[i]) | (~m & a_.i8[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_blendv_epi8 + #define _mm_blendv_epi8(a, b, mask) simde_mm_blendv_epi8(a, b, mask) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_blendv_epi16 (simde__m128i a, simde__m128i b, simde__m128i mask) { + #if defined(SIMDE_X86_SSE2_NATIVE) + mask = simde_mm_srai_epi16(mask, 15); + return simde_mm_or_si128(simde_mm_and_si128(mask, b), simde_mm_andnot_si128(mask, a)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + mask_ = simde__m128i_to_private(mask); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + mask_ = simde__m128i_to_private(simde_mm_cmplt_epi16(mask, simde_mm_setzero_si128())); + r_.neon_i16 = vbslq_s16(mask_.neon_u16, b_.neon_i16, a_.neon_i16); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i16 = vec_sel(a_.altivec_i16, b_.altivec_i16, vec_cmplt(mask_.altivec_i16, vec_splat_s16(0))); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + #if defined(HEDLEY_INTEL_VERSION_CHECK) + __typeof__(mask_.i16) z = { 0, 0, 0, 0, 0, 0, 0, 0 }; + mask_.i16 = mask_.i16 < z; + #else + mask_.i16 >>= (CHAR_BIT * sizeof(mask_.i16[0])) - 1; + #endif + + r_.i16 = (mask_.i16 & b_.i16) | (~mask_.i16 & a_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + int16_t m = mask_.i16[i] >> 15; + r_.i16[i] = (m & b_.i16[i]) | (~m & a_.i16[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_blendv_epi32 (simde__m128i a, simde__m128i b, simde__m128i mask) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_castps_si128(_mm_blendv_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), _mm_castsi128_ps(mask))); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + mask_ = simde__m128i_to_private(mask); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + mask_ = simde__m128i_to_private(simde_mm_cmplt_epi32(mask, simde_mm_setzero_si128())); + r_.neon_i32 = vbslq_s32(mask_.neon_u32, b_.neon_i32, a_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m = wasm_i32x4_shr(mask_.wasm_v128, 31); + r_.wasm_v128 = wasm_v128_or(wasm_v128_and(b_.wasm_v128, m), wasm_v128_andnot(a_.wasm_v128, m)); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i32 = vec_sel(a_.altivec_i32, b_.altivec_i32, vec_cmplt(mask_.altivec_i32, vec_splat_s32(0))); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + #if defined(HEDLEY_INTEL_VERSION_CHECK) + __typeof__(mask_.i32) z = { 0, 0, 0, 0 }; + mask_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(mask_.i32), mask_.i32 < z); + #else + mask_.i32 >>= (CHAR_BIT * sizeof(mask_.i32[0])) - 1; + #endif + + r_.i32 = (mask_.i32 & b_.i32) | (~mask_.i32 & a_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + int32_t m = mask_.i32[i] >> 31; + r_.i32[i] = (m & b_.i32[i]) | (~m & a_.i32[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_blendv_epi64 (simde__m128i a, simde__m128i b, simde__m128i mask) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_castpd_si128(_mm_blendv_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(b), _mm_castsi128_pd(mask))); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + mask_ = simde__m128i_to_private(mask); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + mask_.neon_u64 = vcltq_s64(mask_.neon_i64, vdupq_n_s64(UINT64_C(0))); + r_.neon_i64 = vbslq_s64(mask_.neon_u64, b_.neon_i64, a_.neon_i64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m = wasm_i64x2_shr(mask_.wasm_v128, 63); + r_.wasm_v128 = wasm_v128_or(wasm_v128_and(b_.wasm_v128, m), wasm_v128_andnot(a_.wasm_v128, m)); + #elif (defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) && !defined(SIMDE_BUG_CLANG_46770)) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i64 = vec_sel(a_.altivec_i64, b_.altivec_i64, vec_cmplt(mask_.altivec_i64, vec_splats(HEDLEY_STATIC_CAST(signed long long, 0)))); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(signed long long) selector = vec_sra(mask_.altivec_i64, vec_splats(HEDLEY_STATIC_CAST(unsigned long long, 63))); + r_.altivec_i32 = vec_sel(a_.altivec_i32, b_.altivec_i32, HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned int), selector)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + #if defined(HEDLEY_INTEL_VERSION_CHECK) + __typeof__(mask_.i64) z = { 0, 0 }; + mask_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(mask_.i64), mask_.i64 < z); + #else + mask_.i64 >>= (CHAR_BIT * sizeof(mask_.i64[0])) - 1; + #endif + + r_.i64 = (mask_.i64 & b_.i64) | (~mask_.i64 & a_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + int64_t m = mask_.i64[i] >> 63; + r_.i64[i] = (m & b_.i64[i]) | (~m & a_.i64[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_blendv_pd (simde__m128d a, simde__m128d b, simde__m128d mask) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_blendv_pd(a, b, mask); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m_ = wasm_i64x2_shr(HEDLEY_REINTERPRET_CAST(v128_t, mask), 63); + return simde__m128d_from_wasm_v128(wasm_v128_bitselect(simde__m128d_to_wasm_v128(b), simde__m128d_to_wasm_v128(a), m_)); + #else + return simde_mm_castsi128_pd(simde_x_mm_blendv_epi64(simde_mm_castpd_si128(a), simde_mm_castpd_si128(b), simde_mm_castpd_si128(mask))); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_blendv_pd + #define _mm_blendv_pd(a, b, mask) simde_mm_blendv_pd(a, b, mask) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_blendv_ps (simde__m128 a, simde__m128 b, simde__m128 mask) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_blendv_ps(a, b, mask); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m_ = wasm_i32x4_shr(HEDLEY_REINTERPRET_CAST(v128_t, mask), 31); + return simde__m128d_from_wasm_v128(wasm_v128_bitselect(simde__m128d_to_wasm_v128(b), simde__m128d_to_wasm_v128(a), m_)); + #else + return simde_mm_castsi128_ps(simde_x_mm_blendv_epi32(simde_mm_castps_si128(a), simde_mm_castps_si128(b), simde_mm_castps_si128(mask))); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_blendv_ps + #define _mm_blendv_ps(a, b, mask) simde_mm_blendv_ps(a, b, mask) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_round_pd (simde__m128d a, int rounding) + SIMDE_REQUIRE_CONSTANT_RANGE(rounding, 0, 15) { + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + /* For architectures which lack a current direction SIMD instruction. */ + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + if ((rounding & 7) == SIMDE_MM_FROUND_CUR_DIRECTION) + rounding = HEDLEY_STATIC_CAST(int, SIMDE_MM_GET_ROUNDING_MODE()) << 13; + #endif + + switch (rounding & ~SIMDE_MM_FROUND_NO_EXC) { + case SIMDE_MM_FROUND_CUR_DIRECTION: + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_round(a_.altivec_f64)); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vrndiq_f64(a_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_nearest(a_.wasm_v128); + #elif defined(simde_math_nearbyint) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_nearbyint(a_.f64[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm_undefined_pd()); + #endif + break; + + case SIMDE_MM_FROUND_TO_NEAREST_INT: + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_round(a_.altivec_f64)); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vrndaq_f64(a_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_nearest(a_.wasm_v128); + #elif defined(simde_math_roundeven) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_roundeven(a_.f64[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm_undefined_pd()); + #endif + break; + + case SIMDE_MM_FROUND_TO_NEG_INF: + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_floor(a_.altivec_f64)); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vrndmq_f64(a_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_floor(a_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_floor(a_.f64[i]); + } + #endif + break; + + case SIMDE_MM_FROUND_TO_POS_INF: + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_ceil(a_.altivec_f64)); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vrndpq_f64(a_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_ceil(a_.wasm_v128); + #elif defined(simde_math_ceil) + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_ceil(a_.f64[i]); + } + #else + HEDLEY_UNREACHABLE_RETURN(simde_mm_undefined_pd()); + #endif + break; + + case SIMDE_MM_FROUND_TO_ZERO: + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_f64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(double), vec_trunc(a_.altivec_f64)); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vrndq_f64(a_.neon_f64); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_f64x2_trunc(a_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_trunc(a_.f64[i]); + } + #endif + break; + + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm_undefined_pd()); + } + + return simde__m128d_from_private(r_); +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) + #define simde_mm_round_pd(a, rounding) _mm_round_pd(a, rounding) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_round_pd + #define _mm_round_pd(a, rounding) simde_mm_round_pd(a, rounding) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_ceil_pd (simde__m128d a) { + #if defined(SIMDE_WASM_SIMD128_NATIVE) + return simde__m128d_from_wasm_v128(wasm_f64x2_ceil(simde__m128d_to_wasm_v128(a))); + #endif + return simde_mm_round_pd(a, SIMDE_MM_FROUND_TO_POS_INF); +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_ceil_pd + #define _mm_ceil_pd(a) simde_mm_ceil_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_ceil_ps (simde__m128 a) { + #if defined(SIMDE_WASM_SIMD128_NATIVE) + return simde__m128_from_wasm_v128(wasm_f32x4_ceil(simde__m128_to_wasm_v128(a))); + #endif + return simde_mm_round_ps(a, SIMDE_MM_FROUND_TO_POS_INF); +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_ceil_ps + #define _mm_ceil_ps(a) simde_mm_ceil_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_ceil_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_ceil_sd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(simde_math_ceilf) + r_ = simde__m128d_to_private(simde_mm_set_pd(a_.f64[1], simde_math_ceil(b_.f64[0]))); + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_ceil_sd + #define _mm_ceil_sd(a, b) simde_mm_ceil_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_ceil_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_ceil_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_ceil_ps(b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_ceil_ps(simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(simde_math_ceilf) + r_ = simde__m128_to_private(simde_mm_set_ps(a_.f32[3], a_.f32[2], a_.f32[1], simde_math_ceilf(b_.f32[0]))); + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_ceil_ss + #define _mm_ceil_ss(a, b) simde_mm_ceil_ss(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cmpeq_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_cmpeq_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vceqq_u64(a_.neon_u64, b_.neon_u64); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + /* (a == b) -> (a_lo == b_lo) && (a_hi == b_hi) */ + uint32x4_t cmp = vceqq_u32(a_.neon_u32, b_.neon_u32); + uint32x4_t swapped = vrev64q_u32(cmp); + r_.neon_u32 = vandq_u32(cmp, swapped); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 == b_.i64); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_i64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed long long), vec_cmpeq(a_.altivec_i64, b_.altivec_i64)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] == b_.u64[i]) ? ~UINT64_C(0) : UINT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpeq_epi64 + #define _mm_cmpeq_epi64(a, b) simde_mm_cmpeq_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtepi8_epi16 (simde__m128i a) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_cvtepi8_epi16(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + return _mm_srai_epi16(_mm_unpacklo_epi8(a, a), 8); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int8x16_t s8x16 = a_.neon_i8; /* xxxx xxxx xxxx DCBA */ + int16x8_t s16x8 = vmovl_s8(vget_low_s8(s8x16)); /* 0x0x 0x0x 0D0C 0B0A */ + r_.neon_i16 = s16x8; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_extend_low_i8x16(a_.wasm_v128); + #elif defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_VECTOR_SCALAR) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), SIMDE_SHUFFLE_VECTOR_(8, 16, a_.i8, a_.i8, + -1, 0, -1, 1, -1, 2, -1, 3, + -1, 4, -1, 5, -1, 6, -1, 7)); + r_.i16 >>= 8; + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i16, a_.m64_private[0].i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtepi8_epi16 + #define _mm_cvtepi8_epi16(a) simde_mm_cvtepi8_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtepi8_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_cvtepi8_epi32(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + __m128i tmp = _mm_unpacklo_epi8(a, a); + tmp = _mm_unpacklo_epi16(tmp, tmp); + return _mm_srai_epi32(tmp, 24); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int8x16_t s8x16 = a_.neon_i8; /* xxxx xxxx xxxx DCBA */ + int16x8_t s16x8 = vmovl_s8(vget_low_s8(s8x16)); /* 0x0x 0x0x 0D0C 0B0A */ + int32x4_t s32x4 = vmovl_s16(vget_low_s16(s16x8)); /* 000D 000C 000B 000A */ + r_.neon_i32 = s32x4; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_extend_low_i16x8(wasm_i16x8_extend_low_i8x16(a_.wasm_v128)); + #elif defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_VECTOR_SCALAR) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), SIMDE_SHUFFLE_VECTOR_(8, 16, a_.i8, a_.i8, + -1, -1, -1, 0, -1, -1, -1, 1, + -1, -1, -1, 2, -1, -1, -1, 3)); + r_.i32 >>= 24; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtepi8_epi32 + #define _mm_cvtepi8_epi32(a) simde_mm_cvtepi8_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtepi8_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_cvtepi8_epi64(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int8x16_t s8x16 = a_.neon_i8; /* xxxx xxxx xxxx xxBA */ + int16x8_t s16x8 = vmovl_s8(vget_low_s8(s8x16)); /* 0x0x 0x0x 0x0x 0B0A */ + int32x4_t s32x4 = vmovl_s16(vget_low_s16(s16x8)); /* 000x 000x 000B 000A */ + int64x2_t s64x2 = vmovl_s32(vget_low_s32(s32x4)); /* 0000 000B 0000 000A */ + r_.neon_i64 = s64x2; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t extra = wasm_i32x4_extend_low_i16x8(wasm_i16x8_extend_low_i8x16(a_.wasm_v128)); + v128_t sign = wasm_i32x4_gt(wasm_i64x2_const(0, 0), extra); + r_.wasm_v128 = wasm_i32x4_shuffle(extra, sign, 0, 4, 1, 5); + #elif (!defined(SIMDE_ARCH_X86) && !defined(SIMDE_ARCH_AMD64)) && defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_VECTOR_SCALAR) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + /* Disabled on x86 due to lack of 64-bit arithmetic shift until + * until AVX-512 (at which point we would be using the native + * _mm_cvtepi_epi64 anyways). */ + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), SIMDE_SHUFFLE_VECTOR_(8, 16, a_.i8, a_.i8, + -1, -1, -1, -1, -1, -1, -1, 0, + -1, -1, -1, -1, -1, -1, -1, 1)); + r_.i64 >>= 56; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtepi8_epi64 + #define _mm_cvtepi8_epi64(a) simde_mm_cvtepi8_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtepu8_epi16 (simde__m128i a) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_cvtepu8_epi16(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + return _mm_unpacklo_epi8(a, _mm_setzero_si128()); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint8x16_t u8x16 = a_.neon_u8; /* xxxx xxxx xxxx DCBA */ + uint16x8_t u16x8 = vmovl_u8(vget_low_u8(u8x16)); /* 0x0x 0x0x 0D0C 0B0A */ + r_.neon_u16 = u16x8; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u16x8_extend_low_u8x16(a_.wasm_v128); + #elif defined(SIMDE_SHUFFLE_VECTOR_) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + __typeof__(r_.i8) z = { 0, }; + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), SIMDE_SHUFFLE_VECTOR_(8, 16, a_.i8, z, + 0, 16, 1, 17, 2, 18, 3, 19, + 4, 20, 5, 21, 6, 22, 7, 23)); + #elif defined(SIMDE_CONVERT_VECTOR_) && !defined(SIMDE_BUG_CLANG_45541) && (!defined(SIMDE_ARCH_POWER) || !defined(__clang__)) + SIMDE_CONVERT_VECTOR_(r_.i16, a_.m64_private[0].u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.u8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtepu8_epi16 + #define _mm_cvtepu8_epi16(a) simde_mm_cvtepu8_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtepu8_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_cvtepu8_epi32(a); + #elif defined(SIMDE_X86_SSSE3_NATIVE) + __m128i s = _mm_set_epi8( + HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x03), + HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x02), + HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x01), + HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x00)); + return _mm_shuffle_epi8(a, s); + #elif defined(SIMDE_X86_SSE2_NATIVE) + __m128i z = _mm_setzero_si128(); + return _mm_unpacklo_epi16(_mm_unpacklo_epi8(a, z), z); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint8x16_t u8x16 = a_.neon_u8; /* xxxx xxxx xxxx DCBA */ + uint16x8_t u16x8 = vmovl_u8(vget_low_u8(u8x16)); /* 0x0x 0x0x 0D0C 0B0A */ + uint32x4_t u32x4 = vmovl_u16(vget_low_u16(u16x8)); /* 000D 000C 000B 000A */ + r_.neon_u32 = u32x4; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u32x4_extend_low_u16x8(wasm_u16x8_extend_low_u8x16(a_.wasm_v128)); + #elif defined(SIMDE_SHUFFLE_VECTOR_) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + __typeof__(r_.i8) z = { 0, }; + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), SIMDE_SHUFFLE_VECTOR_(8, 16, a_.i8, z, + 0, 17, 18, 19, 1, 21, 22, 23, + 2, 25, 26, 27, 3, 29, 30, 31)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.u8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtepu8_epi32 + #define _mm_cvtepu8_epi32(a) simde_mm_cvtepu8_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtepu8_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_cvtepu8_epi64(a); + #elif defined(SIMDE_X86_SSSE3_NATIVE) + __m128i s = _mm_set_epi8( + HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), + HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x01), + HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), + HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x80), HEDLEY_STATIC_CAST(char, 0x00)); + return _mm_shuffle_epi8(a, s); + #elif defined(SIMDE_X86_SSE2_NATIVE) + __m128i z = _mm_setzero_si128(); + return _mm_unpacklo_epi32(_mm_unpacklo_epi16(_mm_unpacklo_epi8(a, z), z), z); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint8x16_t u8x16 = a_.neon_u8; /* xxxx xxxx xxxx xxBA */ + uint16x8_t u16x8 = vmovl_u8(vget_low_u8(u8x16)); /* 0x0x 0x0x 0x0x 0B0A */ + uint32x4_t u32x4 = vmovl_u16(vget_low_u16(u16x8)); /* 000x 000x 000B 000A */ + uint64x2_t u64x2 = vmovl_u32(vget_low_u32(u32x4)); /* 0000 000B 0000 000A */ + r_.neon_u64 = u64x2; + #elif defined(SIMDE_SHUFFLE_VECTOR_) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + __typeof__(r_.i8) z = { 0, }; + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), SIMDE_SHUFFLE_VECTOR_(8, 16, a_.i8, z, + 0, 17, 18, 19, 20, 21, 22, 23, + 1, 25, 26, 27, 28, 29, 30, 31)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.u8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtepu8_epi64 + #define _mm_cvtepu8_epi64(a) simde_mm_cvtepu8_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtepi16_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_cvtepi16_epi32(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + return _mm_srai_epi32(_mm_unpacklo_epi16(a, a), 16); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vmovl_s16(vget_low_s16(a_.neon_i16)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_extend_low_i16x8(a_.wasm_v128); + #elif !defined(SIMDE_ARCH_X86) && defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_VECTOR_SCALAR) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), SIMDE_SHUFFLE_VECTOR_(16, 16, a_.i16, a_.i16, 8, 0, 10, 1, 12, 2, 14, 3)); + r_.i32 >>= 16; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtepi16_epi32 + #define _mm_cvtepi16_epi32(a) simde_mm_cvtepi16_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtepu16_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_cvtepu16_epi32(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + return _mm_unpacklo_epi16(a, _mm_setzero_si128()); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vmovl_u16(vget_low_u16(a_.neon_u16)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u32x4_extend_low_u16x8(a_.wasm_v128); + #elif defined(SIMDE_SHUFFLE_VECTOR_) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + __typeof__(r_.u16) z = { 0, }; + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), SIMDE_SHUFFLE_VECTOR_(16, 16, a_.u16, z, + 0, 9, 1, 11, 2, 13, 3, 15)); + #elif defined(SIMDE_CONVERT_VECTOR_) && !defined(SIMDE_BUG_CLANG_45541) && (!defined(SIMDE_ARCH_POWER) || !defined(__clang__)) + SIMDE_CONVERT_VECTOR_(r_.i32, a_.m64_private[0].u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.u16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtepu16_epi32 + #define _mm_cvtepu16_epi32(a) simde_mm_cvtepu16_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtepu16_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_cvtepu16_epi64(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + __m128i z = _mm_setzero_si128(); + return _mm_unpacklo_epi32(_mm_unpacklo_epi16(a, z), z); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint16x8_t u16x8 = a_.neon_u16; /* xxxx xxxx xxxx 0B0A */ + uint32x4_t u32x4 = vmovl_u16(vget_low_u16(u16x8)); /* 000x 000x 000B 000A */ + uint64x2_t u64x2 = vmovl_u32(vget_low_u32(u32x4)); /* 0000 000B 0000 000A */ + r_.neon_u64 = u64x2; + #elif defined(SIMDE_SHUFFLE_VECTOR_) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + __typeof__(r_.u16) z = { 0, }; + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), SIMDE_SHUFFLE_VECTOR_(16, 16, a_.u16, z, + 0, 9, 10, 11, + 1, 13, 14, 15)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.u16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtepu16_epi64 + #define _mm_cvtepu16_epi64(a) simde_mm_cvtepu16_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtepi16_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_cvtepi16_epi64(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int16x8_t s16x8 = a_.neon_i16; /* xxxx xxxx xxxx 0B0A */ + int32x4_t s32x4 = vmovl_s16(vget_low_s16(s16x8)); /* 000x 000x 000B 000A */ + int64x2_t s64x2 = vmovl_s32(vget_low_s32(s32x4)); /* 0000 000B 0000 000A */ + r_.neon_i64 = s64x2; + #elif (!defined(SIMDE_ARCH_X86) && !defined(SIMDE_ARCH_AMD64)) && defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_VECTOR_SCALAR) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), SIMDE_SHUFFLE_VECTOR_(16, 16, a_.i16, a_.i16, + 8, 9, 10, 0, + 12, 13, 14, 1)); + r_.i64 >>= 48; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtepi16_epi64 + #define _mm_cvtepi16_epi64(a) simde_mm_cvtepi16_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtepi32_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_cvtepi32_epi64(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + __m128i tmp = _mm_shuffle_epi32(a, 0x50); + tmp = _mm_srai_epi32(tmp, 31); + tmp = _mm_shuffle_epi32(tmp, 0xed); + return _mm_unpacklo_epi32(a, tmp); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vmovl_s32(vget_low_s32(a_.neon_i32)); + #elif !defined(SIMDE_ARCH_X86) && defined(SIMDE_SHUFFLE_VECTOR_) && defined(SIMDE_VECTOR_SCALAR) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), SIMDE_SHUFFLE_VECTOR_(32, 16, a_.i32, a_.i32, -1, 0, -1, 1)); + r_.i64 >>= 32; + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i64, a_.m64_private[0].i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtepi32_epi64 + #define _mm_cvtepi32_epi64(a) simde_mm_cvtepi32_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cvtepu32_epi64 (simde__m128i a) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_cvtepu32_epi64(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + return _mm_unpacklo_epi32(a, _mm_setzero_si128()); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u64 = vmovl_u32(vget_low_u32(a_.neon_u32)); + #elif defined(SIMDE_VECTOR_SCALAR) && defined(SIMDE_SHUFFLE_VECTOR_) && (SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_LITTLE) + __typeof__(r_.u32) z = { 0, }; + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), SIMDE_SHUFFLE_VECTOR_(32, 16, a_.u32, z, 0, 4, 1, 6)); + #elif defined(SIMDE_CONVERT_VECTOR_) + SIMDE_CONVERT_VECTOR_(r_.i64, a_.m64_private[0].u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.u32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_cvtepu32_epi64 + #define _mm_cvtepu32_epi64(a) simde_mm_cvtepu32_epi64(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_dp_pd (simde__m128d a, simde__m128d b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vmulq_f64(a_.neon_f64, b_.neon_f64); + + switch (imm8) { + case 0xff: + r_.neon_f64 = vaddq_f64(r_.neon_f64, vextq_f64(r_.neon_f64, r_.neon_f64, 1)); + break; + case 0x13: + r_.neon_f64 = vdupq_lane_f64(vget_low_f64(r_.neon_f64), 0); + break; + default: + { /* imm8 is a compile-time constant, so this all becomes just a load */ + uint64_t mask_data[] = { + (imm8 & (1 << 4)) ? ~UINT64_C(0) : UINT64_C(0), + (imm8 & (1 << 5)) ? ~UINT64_C(0) : UINT64_C(0), + }; + r_.neon_f64 = vreinterpretq_f64_u64(vandq_u64(vld1q_u64(mask_data), vreinterpretq_u64_f64(r_.neon_f64))); + } + + r_.neon_f64 = vdupq_n_f64(vaddvq_f64(r_.neon_f64)); + + { + uint64_t mask_data[] = { + (imm8 & 1) ? ~UINT64_C(0) : UINT64_C(0), + (imm8 & 2) ? ~UINT64_C(0) : UINT64_C(0) + }; + r_.neon_f64 = vreinterpretq_f64_u64(vandq_u64(vld1q_u64(mask_data), vreinterpretq_u64_f64(r_.neon_f64))); + } + break; + } + #else + simde_float64 sum = SIMDE_FLOAT64_C(0.0); + + SIMDE_VECTORIZE_REDUCTION(+:sum) + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + sum += ((imm8 >> (i + 4)) & 1) ? (a_.f64[i] * b_.f64[i]) : 0.0; + } + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = ((imm8 >> i) & 1) ? sum : 0.0; + } + #endif + + return simde__m128d_from_private(r_); +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) +# define simde_mm_dp_pd(a, b, imm8) _mm_dp_pd(a, b, imm8) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_dp_pd + #define _mm_dp_pd(a, b, imm8) simde_mm_dp_pd(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_dp_ps (simde__m128 a, simde__m128 b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f32 = vmulq_f32(a_.neon_f32, b_.neon_f32); + + switch (imm8) { + case 0xff: + r_.neon_f32 = vdupq_n_f32(vaddvq_f32(r_.neon_f32)); + break; + case 0x7f: + r_.neon_f32 = vsetq_lane_f32(0, r_.neon_f32, 3); + r_.neon_f32 = vdupq_n_f32(vaddvq_f32(r_.neon_f32)); + break; + default: + { + { + uint32_t mask_data[] = { + (imm8 & (1 << 4)) ? ~UINT32_C(0) : UINT32_C(0), + (imm8 & (1 << 5)) ? ~UINT32_C(0) : UINT32_C(0), + (imm8 & (1 << 6)) ? ~UINT32_C(0) : UINT32_C(0), + (imm8 & (1 << 7)) ? ~UINT32_C(0) : UINT32_C(0) + }; + r_.neon_f32 = vreinterpretq_f32_u32(vandq_u32(vld1q_u32(mask_data), vreinterpretq_u32_f32(r_.neon_f32))); + } + + r_.neon_f32 = vdupq_n_f32(vaddvq_f32(r_.neon_f32)); + + { + uint32_t mask_data[] = { + (imm8 & 1) ? ~UINT32_C(0) : UINT32_C(0), + (imm8 & 2) ? ~UINT32_C(0) : UINT32_C(0), + (imm8 & 4) ? ~UINT32_C(0) : UINT32_C(0), + (imm8 & 8) ? ~UINT32_C(0) : UINT32_C(0) + }; + r_.neon_f32 = vreinterpretq_f32_u32(vandq_u32(vld1q_u32(mask_data), vreinterpretq_u32_f32(r_.neon_f32))); + } + } + break; + } + #else + simde_float32 sum = SIMDE_FLOAT32_C(0.0); + + SIMDE_VECTORIZE_REDUCTION(+:sum) + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + sum += ((imm8 >> (i + 4)) & 1) ? (a_.f32[i] * b_.f32[i]) : SIMDE_FLOAT32_C(0.0); + } + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = ((imm8 >> i) & 1) ? sum : SIMDE_FLOAT32_C(0.0); + } + #endif + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) + #if defined(HEDLEY_MCST_LCC_VERSION) + #define simde_mm_dp_ps(a, b, imm8) (__extension__ ({ \ + SIMDE_LCC_DISABLE_DEPRECATED_WARNINGS \ + _mm_dp_ps((a), (b), (imm8)); \ + SIMDE_LCC_REVERT_DEPRECATED_WARNINGS \ + })) + #else + #define simde_mm_dp_ps(a, b, imm8) _mm_dp_ps(a, b, imm8) + #endif +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_dp_ps + #define _mm_dp_ps(a, b, imm8) simde_mm_dp_ps(a, b, imm8) +#endif + +#if defined(simde_mm_extract_epi8) +# undef simde_mm_extract_epi8 +#endif +SIMDE_FUNCTION_ATTRIBUTES +int8_t +simde_mm_extract_epi8 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m128i_private + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + #if defined(SIMDE_BUG_GCC_95227) + (void) a_; + (void) imm8; + #endif + return vec_extract(a_.altivec_i8, imm8); + #else + return a_.i8[imm8 & 15]; + #endif +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) && !defined(SIMDE_BUG_GCC_BAD_MM_EXTRACT_EPI8) +# define simde_mm_extract_epi8(a, imm8) HEDLEY_STATIC_CAST(int8_t, _mm_extract_epi8(a, imm8)) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) +# define simde_mm_extract_epi8(a, imm8) vgetq_lane_s8(simde__m128i_to_neon_i8(a), imm8) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) +# define simde_mm_extract_epi8(a, imm8) wasm_u8x16_extract_lane(simde__m128i_to_wasm_v128((a)), (imm8) & 15) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_extract_epi8 + #define _mm_extract_epi8(a, imm8) HEDLEY_STATIC_CAST(int, simde_mm_extract_epi8(a, imm8)) +#endif + +#if defined(simde_mm_extract_epi32) +# undef simde_mm_extract_epi32 +#endif +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm_extract_epi32 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m128i_private + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + #if defined(SIMDE_BUG_GCC_95227) + (void) a_; + (void) imm8; + #endif + return vec_extract(a_.altivec_i32, imm8); + #else + return a_.i32[imm8 & 3]; + #endif +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) +# define simde_mm_extract_epi32(a, imm8) _mm_extract_epi32(a, imm8) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) +# define simde_mm_extract_epi32(a, imm8) vgetq_lane_s32(simde__m128i_to_neon_i32(a), imm8) +#elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) +# define simde_mm_extract_epi32(a, imm8) HEDLEY_STATIC_CAST(int32_t, vec_extract(simde__m128i_to_altivec_i32(a), imm8)) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) +# define simde_mm_extract_epi32(a, imm8) wasm_i32x4_extract_lane(simde__m128i_to_wasm_v128((a)), (imm8) & 3) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_extract_epi32 + #define _mm_extract_epi32(a, imm8) simde_mm_extract_epi32(a, imm8) +#endif + +#if defined(simde_mm_extract_epi64) +# undef simde_mm_extract_epi64 +#endif +SIMDE_FUNCTION_ATTRIBUTES +int64_t +simde_mm_extract_epi64 (simde__m128i a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + simde__m128i_private + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + #if defined(SIMDE_BUG_GCC_95227) + (void) a_; + (void) imm8; + #endif + return vec_extract(a_.altivec_i64, imm8); + #else + return a_.i64[imm8 & 1]; + #endif +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) && defined(SIMDE_ARCH_AMD64) +# define simde_mm_extract_epi64(a, imm8) _mm_extract_epi64(a, imm8) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) +# define simde_mm_extract_epi64(a, imm8) vgetq_lane_s64(simde__m128i_to_neon_i64(a), imm8) +#elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) +# define simde_mm_extract_epi64(a, imm8) HEDLEY_STATIC_CAST(int64_t, vec_extract(simde__m128i_to_altivec_i64(a), imm8)) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) + #undef _mm_extract_epi64 + #define _mm_extract_epi64(a, imm8) simde_mm_extract_epi64(a, imm8) +#endif + +#if defined(simde_mm_extract_ps) +# undef simde_mm_extract_ps +#endif +SIMDE_FUNCTION_ATTRIBUTES +int32_t +simde_mm_extract_ps (simde__m128 a, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m128_private + a_ = simde__m128_to_private(a); + + return a_.i32[imm8 & 3]; +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) + #define simde_mm_extract_ps(a, imm8) _mm_extract_ps(a, imm8) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #define simde_mm_extract_ps(a, imm8) vgetq_lane_s32(simde__m128_to_neon_i32(a), imm8) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) + #define simde_mm_extract_ps(a, imm8) wasm_i32x4_extract_lane(simde__m128_to_wasm_v128((a)), (imm8) & 3) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_extract_ps + #define _mm_extract_ps(a, imm8) simde_mm_extract_ps(a, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_floor_pd (simde__m128d a) { + #if defined(SIMDE_WASM_SIMD128_NATIVE) + return simde__m128d_from_wasm_v128(wasm_f64x2_floor(simde__m128d_to_wasm_v128(a))); + #endif + return simde_mm_round_pd(a, SIMDE_MM_FROUND_TO_NEG_INF); +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_floor_pd + #define _mm_floor_pd(a) simde_mm_floor_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_floor_ps (simde__m128 a) { + #if defined(SIMDE_WASM_SIMD128_NATIVE) + return simde__m128_from_wasm_v128(wasm_f32x4_floor(simde__m128_to_wasm_v128(a))); + #endif + return simde_mm_round_ps(a, SIMDE_MM_FROUND_TO_NEG_INF); +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_floor_ps + #define _mm_floor_ps(a) simde_mm_floor_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_floor_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_floor_sd(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(simde_math_floor) + r_.f64[0] = simde_math_floor(b_.f64[0]); + r_.f64[1] = a_.f64[1]; + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_floor_sd + #define _mm_floor_sd(a, b) simde_mm_floor_sd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_floor_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_floor_ss(a, b); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_FAST_EXCEPTIONS) + return simde_mm_move_ss(a, simde_mm_floor_ps(b)); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + return simde_mm_move_ss(a, simde_mm_floor_ps(simde_x_mm_broadcastlow_ps(b))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(simde_math_floorf) + r_.f32[0] = simde_math_floorf(b_.f32[0]); + for (size_t i = 1 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = a_.f32[i]; + } + #else + HEDLEY_UNREACHABLE(); + #endif + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_floor_ss + #define _mm_floor_ss(a, b) simde_mm_floor_ss(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_insert_epi8 (simde__m128i a, int i, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 15) { + simde__m128i_private + r_ = simde__m128i_to_private(a); + + r_.i8[imm8] = HEDLEY_STATIC_CAST(int8_t, i); + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) + /* clang-3.8 returns an incompatible type, so we need the cast. MSVC + * can't handle the cast ("error C2440: 'type cast': cannot convert + * from '__m128i' to '__m128i'"). */ + #if defined(__clang__) + #define simde_mm_insert_epi8(a, i, imm8) HEDLEY_REINTERPRET_CAST(__m128i, _mm_insert_epi8(a, i, imm8)) + #else + #define simde_mm_insert_epi8(a, i, imm8) _mm_insert_epi8(a, i, imm8) + #endif +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) +# define simde_mm_insert_epi8(a, i, imm8) simde__m128i_from_neon_i8(vsetq_lane_s8(i, simde__m128i_to_neon_i8(a), imm8)) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) +# define simde_mm_insert_epi8(a, i, imm8) simde__m128i_from_wasm_v128(wasm_i8x16_replace_lane(simde__m128i_to_wasm_v128((a)), (imm8) & 15, HEDLEY_STATIC_CAST(int8_t, (i)))) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_insert_epi8 + #define _mm_insert_epi8(a, i, imm8) simde_mm_insert_epi8(a, i, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_insert_epi32 (simde__m128i a, int i, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 3) { + simde__m128i_private + r_ = simde__m128i_to_private(a); + + r_.i32[imm8] = HEDLEY_STATIC_CAST(int32_t, i); + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) + #if defined(__clang__) + #define simde_mm_insert_epi32(a, i, imm8) HEDLEY_REINTERPRET_CAST(__m128i, _mm_insert_epi32(a, i, imm8)) + #else + #define simde_mm_insert_epi32(a, i, imm8) _mm_insert_epi32(a, i, imm8) + #endif +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) +# define simde_mm_insert_epi32(a, i, imm8) simde__m128i_from_neon_i32(vsetq_lane_s32(i, simde__m128i_to_neon_i32(a), imm8)) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) +# define simde_mm_insert_epi32(a, i, imm8) simde__m128i_from_wasm_v128(wasm_i32x4_replace_lane(simde__m128i_to_wasm_v128((a)), (imm8) & 3, (i))) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_insert_epi32 + #define _mm_insert_epi32(a, i, imm8) simde_mm_insert_epi32(a, i, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_insert_epi64 (simde__m128i a, int64_t i, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 1) { + #if defined(SIMDE_BUG_GCC_94482) + simde__m128i_private + a_ = simde__m128i_to_private(a); + + switch(imm8) { + case 0: + return simde_mm_set_epi64x(a_.i64[1], i); + break; + case 1: + return simde_mm_set_epi64x(i, a_.i64[0]); + break; + default: + HEDLEY_UNREACHABLE(); + break; + } + #else + simde__m128i_private + r_ = simde__m128i_to_private(a); + + r_.i64[imm8] = i; + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) && defined(SIMDE_ARCH_AMD64) +# define simde_mm_insert_epi64(a, i, imm8) _mm_insert_epi64(a, i, imm8) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) +# define simde_mm_insert_epi64(a, i, imm8) simde__m128i_from_neon_i64(vsetq_lane_s64(i, simde__m128i_to_neon_i64(a), imm8)) +#elif defined(SIMDE_WASM_SIMD128_NATIVE) +# define simde_mm_insert_epi64(a, i, imm8) simde__m128i_from_wasm_v128(wasm_i64x2_replace_lane(simde__m128i_to_wasm_v128((a)), (imm8) & 1, (i))) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) + #undef _mm_insert_epi64 + #define _mm_insert_epi64(a, i, imm8) simde_mm_insert_epi64(a, i, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_insert_ps (simde__m128 a, simde__m128 b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + float tmp1_ = b_.f32[(imm8 >> 6) & 3]; + a_.f32[(imm8 >> 4) & 3] = tmp1_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = ((imm8 >> i) & 1 ) ? SIMDE_FLOAT32_C(0.0) : a_.f32[i]; + } + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) +# define simde_mm_insert_ps(a, b, imm8) _mm_insert_ps(a, b, imm8) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_insert_ps + #define _mm_insert_ps(a, b, imm8) simde_mm_insert_ps(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_max_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) && !defined(__PGI) + return _mm_max_epi8(a, b); + #elif defined(SIMDE_X86_SSE2_NATIVE) + __m128i m = _mm_cmpgt_epi8(a, b); + return _mm_or_si128(_mm_and_si128(m, a), _mm_andnot_si128(m, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vmaxq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_max(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i8 = vec_max(a_.altivec_i8, b_.altivec_i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] > b_.i8[i] ? a_.i8[i] : b_.i8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_max_epi8 + #define _mm_max_epi8(a, b) simde_mm_max_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_max_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) && !defined(__PGI) + return _mm_max_epi32(a, b); + #elif defined(SIMDE_X86_SSE2_NATIVE) + __m128i m = _mm_cmpgt_epi32(a, b); + return _mm_or_si128(_mm_and_si128(m, a), _mm_andnot_si128(m, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vmaxq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_max(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i32 = vec_max(a_.altivec_i32, b_.altivec_i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] > b_.i32[i] ? a_.i32[i] : b_.i32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_max_epi32 + #define _mm_max_epi32(a, b) simde_mm_max_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_max_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_max_epu16(a, b); + #elif defined(SIMDE_X86_SSE2_NATIVE) + /* https://github.com/simd-everywhere/simde/issues/855#issuecomment-881656284 */ + return _mm_add_epi16(b, _mm_subs_epu16(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vmaxq_u16(a_.neon_u16, b_.neon_u16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u16x8_max(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u16 = vec_max(a_.altivec_u16, b_.altivec_u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = a_.u16[i] > b_.u16[i] ? a_.u16[i] : b_.u16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_max_epu16 + #define _mm_max_epu16(a, b) simde_mm_max_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_max_epu32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_max_epu32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vmaxq_u32(a_.neon_u32, b_.neon_u32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u32x4_max(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u32 = vec_max(a_.altivec_u32, b_.altivec_u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] > b_.u32[i] ? a_.u32[i] : b_.u32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_max_epu32 + #define _mm_max_epu32(a, b) simde_mm_max_epu32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_min_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) && !defined(__PGI) + return _mm_min_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vminq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_min(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i8 = vec_min(a_.altivec_i8, b_.altivec_i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] < b_.i8[i] ? a_.i8[i] : b_.i8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_min_epi8 + #define _mm_min_epi8(a, b) simde_mm_min_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_min_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) && !defined(__PGI) + return _mm_min_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vminq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_min(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i32 = vec_min(a_.altivec_i32, b_.altivec_i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] < b_.i32[i] ? a_.i32[i] : b_.i32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_min_epi32 + #define _mm_min_epi32(a, b) simde_mm_min_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_min_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_min_epu16(a, b); + #elif defined(SIMDE_X86_SSE2_NATIVE) + /* https://github.com/simd-everywhere/simde/issues/855#issuecomment-881656284 */ + return _mm_sub_epi16(a, _mm_subs_epu16(a, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vminq_u16(a_.neon_u16, b_.neon_u16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u16x8_min(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u16 = vec_min(a_.altivec_u16, b_.altivec_u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = a_.u16[i] < b_.u16[i] ? a_.u16[i] : b_.u16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_min_epu16 + #define _mm_min_epu16(a, b) simde_mm_min_epu16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_min_epu32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_min_epu32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vminq_u32(a_.neon_u32, b_.neon_u32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u32x4_min(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_u32 = vec_min(a_.altivec_u32, b_.altivec_u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] < b_.u32[i] ? a_.u32[i] : b_.u32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_min_epu32 + #define _mm_min_epu32(a, b) simde_mm_min_epu32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_minpos_epu16 (simde__m128i a) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_minpos_epu16(a); + #else + simde__m128i_private + r_ = simde__m128i_to_private(simde_mm_setzero_si128()), + a_ = simde__m128i_to_private(a); + + r_.u16[0] = UINT16_MAX; + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + if (a_.u16[i] < r_.u16[0]) { + r_.u16[0] = a_.u16[i]; + r_.u16[1] = HEDLEY_STATIC_CAST(uint16_t, i); + } + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_minpos_epu16 + #define _mm_minpos_epu16(a) simde_mm_minpos_epu16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mpsadbw_epu8 (simde__m128i a, simde__m128i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + const int a_offset = imm8 & 4; + const int b_offset = (imm8 & 3) << 2; + +#if defined(simde_math_abs) + for (int i = 0 ; i < HEDLEY_STATIC_CAST(int, (sizeof(r_.u16) / sizeof(r_.u16[0]))) ; i++) { + r_.u16[i] = + HEDLEY_STATIC_CAST(uint16_t, simde_math_abs(HEDLEY_STATIC_CAST(int, a_.u8[a_offset + i + 0] - b_.u8[b_offset + 0]))) + + HEDLEY_STATIC_CAST(uint16_t, simde_math_abs(HEDLEY_STATIC_CAST(int, a_.u8[a_offset + i + 1] - b_.u8[b_offset + 1]))) + + HEDLEY_STATIC_CAST(uint16_t, simde_math_abs(HEDLEY_STATIC_CAST(int, a_.u8[a_offset + i + 2] - b_.u8[b_offset + 2]))) + + HEDLEY_STATIC_CAST(uint16_t, simde_math_abs(HEDLEY_STATIC_CAST(int, a_.u8[a_offset + i + 3] - b_.u8[b_offset + 3]))); + } +#else + HEDLEY_UNREACHABLE(); +#endif + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) && !defined(SIMDE_BUG_PGI_30107) +# define simde_mm_mpsadbw_epu8(a, b, imm8) _mm_mpsadbw_epu8(a, b, imm8) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_mpsadbw_epu8 + #define _mm_mpsadbw_epu8(a, b, imm8) simde_mm_mpsadbw_epu8(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mul_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_mul_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + // vmull_s32 upcasts instead of masking, so we downcast. + int32x2_t a_lo = vmovn_s64(a_.neon_i64); + int32x2_t b_lo = vmovn_s64(b_.neon_i64); + r_.neon_i64 = vmull_s32(a_lo, b_lo); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_make( + wasm_i32x4_extract_lane(a_.wasm_v128, 0) * HEDLEY_STATIC_CAST(int64_t, wasm_i32x4_extract_lane(b_.wasm_v128, 0)), + wasm_i32x4_extract_lane(a_.wasm_v128, 2) * HEDLEY_STATIC_CAST(int64_t, wasm_i32x4_extract_lane(b_.wasm_v128, 2))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = + HEDLEY_STATIC_CAST(int64_t, a_.i32[i * 2]) * + HEDLEY_STATIC_CAST(int64_t, b_.i32[i * 2]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_mul_epi32 + #define _mm_mul_epi32(a, b) simde_mm_mul_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mullo_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_mullo_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vmulq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + (void) a_; + (void) b_; + r_.altivec_i32 = vec_mul(a_.altivec_i32, b_.altivec_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_mul(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.u32[i] = HEDLEY_STATIC_CAST(uint32_t, (HEDLEY_STATIC_CAST(uint64_t, (HEDLEY_STATIC_CAST(int64_t, a_.i32[i]) * HEDLEY_STATIC_CAST(int64_t, b_.i32[i]))) & 0xffffffff)); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_mullo_epi32 + #define _mm_mullo_epi32(a, b) simde_mm_mullo_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_x_mm_mullo_epu32 (simde__m128i a, simde__m128i b) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vmulq_u32(a_.neon_u32, b_.neon_u32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = a_.u32 * b_.u32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] * b_.u32[i]; + } + #endif + + return simde__m128i_from_private(r_); +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_packus_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_packus_epi32(a, b); + #elif defined(SIMDE_X86_SSE2_NATIVE) + const __m128i max = _mm_set1_epi32(UINT16_MAX); + const __m128i tmpa = _mm_andnot_si128(_mm_srai_epi32(a, 31), a); + const __m128i tmpb = _mm_andnot_si128(_mm_srai_epi32(b, 31), b); + return + _mm_packs_epi32( + _mm_srai_epi32(_mm_slli_epi32(_mm_or_si128(tmpa, _mm_cmpgt_epi32(tmpa, max)), 16), 16), + _mm_srai_epi32(_mm_slli_epi32(_mm_or_si128(tmpb, _mm_cmpgt_epi32(tmpb, max)), 16), 16) + ); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + r_; + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + #if defined(SIMDE_BUG_CLANG_46840) + r_.neon_u16 = vqmovun_high_s32(vreinterpret_s16_u16(vqmovun_s32(a_.neon_i32)), b_.neon_i32); + #else + r_.neon_u16 = vqmovun_high_s32(vqmovun_s32(a_.neon_i32), b_.neon_i32); + #endif + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = + vcombine_u16( + vqmovun_s32(a_.neon_i32), + vqmovun_s32(b_.neon_i32) + ); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r_.altivec_u16 = vec_packsu(a_.altivec_i32, b_.altivec_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u16x8_narrow_i32x4(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_CONVERT_VECTOR_) && HEDLEY_HAS_BUILTIN(__builtin_shufflevector) && defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + int32_t v SIMDE_VECTOR(32) = SIMDE_SHUFFLE_VECTOR_(32, 32, a_.i32, b_.i32, 0, 1, 2, 3, 4, 5, 6, 7); + + v &= ~(v >> 31); + v |= HEDLEY_REINTERPRET_CAST(__typeof__(v), v > UINT16_MAX); + + SIMDE_CONVERT_VECTOR_(r_.i16, v); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + int32_t v = (i < (sizeof(a_.i32) / sizeof(a_.i32[0]))) ? a_.i32[i] : b_.i32[i & 3]; + r_.u16[i] = (v < 0) ? UINT16_C(0) : ((v > UINT16_MAX) ? UINT16_MAX : HEDLEY_STATIC_CAST(uint16_t, v)); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_packus_epi32 + #define _mm_packus_epi32(a, b) simde_mm_packus_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_round_sd (simde__m128d a, simde__m128d b, int rounding) + SIMDE_REQUIRE_CONSTANT_RANGE(rounding, 0, 15) { + simde__m128d_private + r_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + switch (rounding & ~SIMDE_MM_FROUND_NO_EXC) { + #if defined(simde_math_nearbyint) + case SIMDE_MM_FROUND_TO_NEAREST_INT: + case SIMDE_MM_FROUND_CUR_DIRECTION: + r_.f64[0] = simde_math_nearbyint(b_.f64[0]); + break; + #endif + + #if defined(simde_math_floor) + case SIMDE_MM_FROUND_TO_NEG_INF: + r_.f64[0] = simde_math_floor(b_.f64[0]); + break; + #endif + + #if defined(simde_math_ceil) + case SIMDE_MM_FROUND_TO_POS_INF: + r_.f64[0] = simde_math_ceil(b_.f64[0]); + break; + #endif + + #if defined(simde_math_trunc) + case SIMDE_MM_FROUND_TO_ZERO: + r_.f64[0] = simde_math_trunc(b_.f64[0]); + break; + #endif + + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm_undefined_pd()); + } + + return simde__m128d_from_private(r_); +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) +# define simde_mm_round_sd(a, b, rounding) _mm_round_sd(a, b, rounding) +#elif SIMDE_NATURAL_VECTOR_SIZE_GE(128) && defined(SIMDE_FAST_EXCEPTIONS) +# define simde_mm_round_sd(a, b, rounding) simde_mm_move_sd(a, simde_mm_round_pd(b, rounding)) +#elif SIMDE_NATURAL_VECTOR_SIZE_GE(128) + #define simde_mm_round_sd(a, b, rounding) simde_mm_move_sd(a, simde_mm_round_pd(simde_x_mm_broadcastlow_pd(b), rounding)) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_round_sd + #define _mm_round_sd(a, b, rounding) simde_mm_round_sd(a, b, rounding) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_round_ss (simde__m128 a, simde__m128 b, int rounding) + SIMDE_REQUIRE_CONSTANT_RANGE(rounding, 0, 15) { + simde__m128_private + r_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + switch (rounding & ~SIMDE_MM_FROUND_NO_EXC) { + #if defined(simde_math_nearbyintf) + case SIMDE_MM_FROUND_TO_NEAREST_INT: + case SIMDE_MM_FROUND_CUR_DIRECTION: + r_.f32[0] = simde_math_nearbyintf(b_.f32[0]); + break; + #endif + + #if defined(simde_math_floorf) + case SIMDE_MM_FROUND_TO_NEG_INF: + r_.f32[0] = simde_math_floorf(b_.f32[0]); + break; + #endif + + #if defined(simde_math_ceilf) + case SIMDE_MM_FROUND_TO_POS_INF: + r_.f32[0] = simde_math_ceilf(b_.f32[0]); + break; + #endif + + #if defined(simde_math_truncf) + case SIMDE_MM_FROUND_TO_ZERO: + r_.f32[0] = simde_math_truncf(b_.f32[0]); + break; + #endif + + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm_undefined_pd()); + } + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_SSE4_1_NATIVE) + #define simde_mm_round_ss(a, b, rounding) _mm_round_ss(a, b, rounding) +#elif SIMDE_NATURAL_VECTOR_SIZE > 0 && defined(SIMDE_FAST_EXCEPTIONS) + #define simde_mm_round_ss(a, b, rounding) simde_mm_move_ss((a), simde_mm_round_ps((b), (rounding))) +#elif SIMDE_NATURAL_VECTOR_SIZE > 0 + #define simde_mm_round_ss(a, b, rounding) simde_mm_move_ss((a), simde_mm_round_ps(simde_x_mm_broadcastlow_ps(b), (rounding))) +#endif +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_round_ss + #define _mm_round_ss(a, b, rounding) simde_mm_round_ss(a, b, rounding) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_stream_load_si128 (const simde__m128i* mem_addr) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_stream_load_si128(HEDLEY_CONST_CAST(simde__m128i*, mem_addr)); + #elif HEDLEY_HAS_BUILTIN(__builtin_nontemporal_load) && ( \ + defined(SIMDE_ARM_NEON_A32V7_NATIVE) || defined(SIMDE_VECTOR_SUBSCRIPT) || \ + defined(SIMDE_WASM_SIMD128_NATIVE) || defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || \ + defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE)) + return __builtin_nontemporal_load(mem_addr); + #else + return simde_mm_load_si128(mem_addr); + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_stream_load_si128 + #define _mm_stream_load_si128(mem_addr) simde_mm_stream_load_si128(mem_addr) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_test_all_ones (simde__m128i a) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_test_all_ones(a); + #else + simde__m128i_private a_ = simde__m128i_to_private(a); + int r; + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r = vec_all_eq(a_.altivec_i32, vec_splats(~0)); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r = ((vgetq_lane_s64(a_.neon_i64, 0) & vgetq_lane_s64(a_.neon_i64, 1)) == ~HEDLEY_STATIC_CAST(int64_t, 0)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r = HEDLEY_STATIC_CAST(unsigned long long, wasm_i64x2_extract_lane(a_.wasm_v128, 0) & wasm_i64x2_extract_lane(a_.wasm_v128, 1)) == 0xFFFFFFFFFFFFFFFFull; + #else + int_fast32_t r_ = ~HEDLEY_STATIC_CAST(int_fast32_t, 0); + + SIMDE_VECTORIZE_REDUCTION(&:r_) + for (size_t i = 0 ; i < (sizeof(a_.i32f) / sizeof(a_.i32f[0])) ; i++) { + r_ &= a_.i32f[i]; + } + + r = (r_ == ~HEDLEY_STATIC_CAST(int_fast32_t, 0)); + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_test_all_ones + #define _mm_test_all_ones(a) simde_mm_test_all_ones(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_test_all_zeros (simde__m128i a, simde__m128i mask) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_test_all_zeros(a, mask); + #else + simde__m128i_private tmp_ = simde__m128i_to_private(simde_mm_and_si128(a, mask)); + int r; + + #if defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + r = vec_all_eq(tmp_.altivec_i32, vec_splats(0)); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r = !(vgetq_lane_s64(tmp_.neon_i64, 0) | vgetq_lane_s64(tmp_.neon_i64, 1)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r = (wasm_i64x2_extract_lane(tmp_.wasm_v128, 0) | wasm_i64x2_extract_lane(tmp_.wasm_v128, 1)) == 0; + #else + int_fast32_t r_ = HEDLEY_STATIC_CAST(int_fast32_t, 0); + + SIMDE_VECTORIZE_REDUCTION(|:r_) + for (size_t i = 0 ; i < (sizeof(tmp_.i32f) / sizeof(tmp_.i32f[0])) ; i++) { + r_ |= tmp_.i32f[i]; + } + + r = !r_; + #endif + + return r; + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_test_all_zeros + #define _mm_test_all_zeros(a, mask) simde_mm_test_all_zeros(a, mask) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_test_mix_ones_zeros (simde__m128i a, simde__m128i mask) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_test_mix_ones_zeros(a, mask); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + mask_ = simde__m128i_to_private(mask); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int64x2_t s640 = vandq_s64(a_.neon_i64, mask_.neon_i64); + int64x2_t s641 = vandq_s64(vreinterpretq_s64_s32(vmvnq_s32(vreinterpretq_s32_s64(a_.neon_i64))), mask_.neon_i64); + return (((vgetq_lane_s64(s640, 0) | vgetq_lane_s64(s640, 1)) & (vgetq_lane_s64(s641, 0) | vgetq_lane_s64(s641, 1)))!=0); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m = wasm_v128_and(a_.wasm_v128, mask_.wasm_v128); + long long c0 = wasm_i64x2_extract_lane(m, 0); + long long c1 = wasm_i64x2_extract_lane(m, 1); + long long ones = c0 | c1; + long long zeros = ~(c0 & c1); + return ones && zeros; + #else + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) + if (((a_.u64[i] & mask_.u64[i]) != 0) && ((~a_.u64[i] & mask_.u64[i]) != 0)) + return 1; + + return 0; + #endif + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_test_mix_ones_zeros + #define _mm_test_mix_ones_zeros(a, mask) simde_mm_test_mix_ones_zeros(a, mask) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_testc_si128 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_testc_si128(a, b); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int64x2_t s64 = vbicq_s64(b_.neon_i64, a_.neon_i64); + return !(vgetq_lane_s64(s64, 0) | vgetq_lane_s64(s64, 1)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m = wasm_v128_andnot(b_.wasm_v128, a_.wasm_v128); + return (wasm_i64x2_extract_lane(m, 0) | wasm_i64x2_extract_lane(m, 1)) == 0; + #else + int_fast32_t r = 0; + + SIMDE_VECTORIZE_REDUCTION(|:r) + for (size_t i = 0 ; i < (sizeof(a_.i32f) / sizeof(a_.i32f[0])) ; i++) { + r |= ~a_.i32f[i] & b_.i32f[i]; + } + + return HEDLEY_STATIC_CAST(int, !r); + #endif + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_testc_si128 + #define _mm_testc_si128(a, b) simde_mm_testc_si128(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_testnzc_si128 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_testnzc_si128(a, b); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int64x2_t s640 = vandq_s64(b_.neon_i64, a_.neon_i64); + int64x2_t s641 = vbicq_s64(b_.neon_i64, a_.neon_i64); + return !( !(vgetq_lane_s64(s641, 0) || vgetq_lane_s64(s641, 1)) \ + || !(vgetq_lane_s64(s640, 0) || vgetq_lane_s64(s640, 1)) ); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m1 = wasm_v128_and(a_.wasm_v128, b_.wasm_v128); + v128_t m2 = wasm_v128_andnot(b_.wasm_v128, a_.wasm_v128); + return (wasm_i64x2_extract_lane(m1, 0) | wasm_i64x2_extract_lane(m1, 1)) \ + && (wasm_i64x2_extract_lane(m2, 0) | wasm_i64x2_extract_lane(m2, 1)); + #else + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + if (((a_.u64[i] & b_.u64[i]) != 0) && ((~a_.u64[i] & b_.u64[i]) != 0)) + return 1; + } + + return 0; + #endif + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_testnzc_si128 + #define _mm_testnzc_si128(a, b) simde_mm_testnzc_si128(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_testz_si128 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_1_NATIVE) + return _mm_testz_si128(a, b); + #else + simde__m128i_private + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int64x2_t s64 = vandq_s64(a_.neon_i64, b_.neon_i64); + return !(vgetq_lane_s64(s64, 0) | vgetq_lane_s64(s64, 1)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t m = wasm_v128_and(a_.wasm_v128, b_.wasm_v128); + return (wasm_i64x2_extract_lane(m, 0) | wasm_i64x2_extract_lane(m, 1)) == 0; + #elif defined(SIMDE_HAVE_INT128_) + if ((a_.u128[0] & b_.u128[0]) == 0) { + return 1; + } + return 0; + #else + for (size_t i = 0 ; i < (sizeof(a_.u64) / sizeof(a_.u64[0])) ; i++) { + if ((a_.u64[i] & b_.u64[i]) > 0) + return 0; + } + #endif + + return 1; + #endif +} +#if defined(SIMDE_X86_SSE4_1_ENABLE_NATIVE_ALIASES) + #undef _mm_testz_si128 + #define _mm_testz_si128(a, b) simde_mm_testz_si128(a, b) +#endif + +SIMDE_END_DECLS_ + +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_SSE4_1_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse4.2.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse4.2.h new file mode 100644 index 00000000..ae9e7569 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/sse4.2.h @@ -0,0 +1,381 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2017 Evan Nemerson + * 2020 Hidayat Khan + */ + +#if !defined(SIMDE_X86_SSE4_2_H) +#define SIMDE_X86_SSE4_2_H + +#include "sse4.1.h" + +#if defined(__ARM_ACLE) || (defined(__GNUC__) && defined(__ARM_FEATURE_CRC32)) + #include +#endif + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +#if defined(SIMDE_X86_SSE4_2_NATIVE) + #define SIMDE_SIDD_UBYTE_OPS _SIDD_UBYTE_OPS + #define SIMDE_SIDD_UWORD_OPS _SIDD_UWORD_OPS + #define SIMDE_SIDD_SBYTE_OPS _SIDD_SBYTE_OPS + #define SIMDE_SIDD_SWORD_OPS _SIDD_SWORD_OPS + #define SIMDE_SIDD_CMP_EQUAL_ANY _SIDD_CMP_EQUAL_ANY + #define SIMDE_SIDD_CMP_RANGES _SIDD_CMP_RANGES + #define SIMDE_SIDD_CMP_EQUAL_EACH _SIDD_CMP_EQUAL_EACH + #define SIMDE_SIDD_CMP_EQUAL_ORDERED _SIDD_CMP_EQUAL_ORDERED + #define SIMDE_SIDD_POSITIVE_POLARITY _SIDD_POSITIVE_POLARITY + #define SIMDE_SIDD_NEGATIVE_POLARITY _SIDD_NEGATIVE_POLARITY + #define SIMDE_SIDD_MASKED_POSITIVE_POLARITY _SIDD_MASKED_POSITIVE_POLARITY + #define SIMDE_SIDD_MASKED_NEGATIVE_POLARITY _SIDD_MASKED_NEGATIVE_POLARITY + #define SIMDE_SIDD_LEAST_SIGNIFICANT _SIDD_LEAST_SIGNIFICANT + #define SIMDE_SIDD_MOST_SIGNIFICANT _SIDD_MOST_SIGNIFICANT + #define SIMDE_SIDD_BIT_MASK _SIDD_BIT_MASK + #define SIMDE_SIDD_UNIT_MASK _SIDD_UNIT_MASK +#else + #define SIMDE_SIDD_UBYTE_OPS 0x00 + #define SIMDE_SIDD_UWORD_OPS 0x01 + #define SIMDE_SIDD_SBYTE_OPS 0x02 + #define SIMDE_SIDD_SWORD_OPS 0x03 + #define SIMDE_SIDD_CMP_EQUAL_ANY 0x00 + #define SIMDE_SIDD_CMP_RANGES 0x04 + #define SIMDE_SIDD_CMP_EQUAL_EACH 0x08 + #define SIMDE_SIDD_CMP_EQUAL_ORDERED 0x0c + #define SIMDE_SIDD_POSITIVE_POLARITY 0x00 + #define SIMDE_SIDD_NEGATIVE_POLARITY 0x10 + #define SIMDE_SIDD_MASKED_POSITIVE_POLARITY 0x20 + #define SIMDE_SIDD_MASKED_NEGATIVE_POLARITY 0x30 + #define SIMDE_SIDD_LEAST_SIGNIFICANT 0x00 + #define SIMDE_SIDD_MOST_SIGNIFICANT 0x40 + #define SIMDE_SIDD_BIT_MASK 0x00 + #define SIMDE_SIDD_UNIT_MASK 0x40 +#endif + +#if defined(SIMDE_X86_SSE4_2_ENABLE_NATIVE_ALIASES) && !defined(_SIDD_UBYTE_OPS) + #define _SIDD_UBYTE_OPS SIMDE_SIDD_UBYTE_OPS + #define _SIDD_UWORD_OPS SIMDE_SIDD_UWORD_OPS + #define _SIDD_SBYTE_OPS SIMDE_SIDD_SBYTE_OPS + #define _SIDD_SWORD_OPS SIMDE_SIDD_SWORD_OPS + #define _SIDD_CMP_EQUAL_ANY SIMDE_SIDD_CMP_EQUAL_ANY + #define _SIDD_CMP_RANGES SIMDE_SIDD_CMP_RANGES + #define _SIDD_CMP_EQUAL_EACH SIMDE_SIDD_CMP_EQUAL_EACH + #define _SIDD_CMP_EQUAL_ORDERED SIMDE_SIDD_CMP_EQUAL_ORDERED + #define _SIDD_POSITIVE_POLARITY SIMDE_SIDD_POSITIVE_POLARITY + #define _SIDD_NEGATIVE_POLARITY SIMDE_SIDD_NEGATIVE_POLARITY + #define _SIDD_MASKED_POSITIVE_POLARITY SIMDE_SIDD_MASKED_POSITIVE_POLARITY + #define _SIDD_MASKED_NEGATIVE_POLARITY SIMDE_SIDD_MASKED_NEGATIVE_POLARITY + #define _SIDD_LEAST_SIGNIFICANT SIMDE_SIDD_LEAST_SIGNIFICANT + #define _SIDD_MOST_SIGNIFICANT SIMDE_SIDD_MOST_SIGNIFICANT + #define _SIDD_BIT_MASK SIMDE_SIDD_BIT_MASK + #define _SIDD_UNIT_MASK SIMDE_SIDD_UNIT_MASK +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int simde_mm_cmpestrs (simde__m128i a, int la, simde__m128i b, int lb, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + #if !defined(HEDLEY_PGI_VERSION) + /* https://www.pgroup.com/userforum/viewtopic.php?f=4&p=27590&sid=cf89f8bf30be801831fe4a2ff0a2fa6c */ + (void) a; + (void) b; + #endif + (void) la; + (void) lb; + return la <= ((128 / ((imm8 & SIMDE_SIDD_UWORD_OPS) ? 16 : 8)) - 1); +} +#if defined(SIMDE_X86_SSE4_2_NATIVE) + #if defined(__clang__) && !SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + #define simde_mm_cmpestrs(a, la, b, lb, imm8) \ + _mm_cmpestrs( \ + HEDLEY_REINTERPRET_CAST(__v16qi, a), la, \ + HEDLEY_REINTERPRET_CAST(__v16qi, b), lb, \ + imm8) + #else + #define simde_mm_cmpestrs(a, la, b, lb, imm8) _mm_cmpestrs(a, la, b, lb, imm8) + #endif +#endif +#if defined(SIMDE_X86_SSE4_2_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpestrs + #define _mm_cmpestrs(a, la, b, lb, imm8) simde_mm_cmpestrs(a, la, b, lb, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int simde_mm_cmpestrz (simde__m128i a, int la, simde__m128i b, int lb, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 255) { + #if !defined(HEDLEY_PGI_VERSION) + /* https://www.pgroup.com/userforum/viewtopic.php?f=4&p=27590&sid=cf89f8bf30be801831fe4a2ff0a2fa6c */ + (void) a; + (void) b; + #endif + (void) la; + (void) lb; + return lb <= ((128 / ((imm8 & SIMDE_SIDD_UWORD_OPS) ? 16 : 8)) - 1); +} +#if defined(SIMDE_X86_SSE4_2_NATIVE) + #if defined(__clang__) && !SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + #define simde_mm_cmpestrz(a, la, b, lb, imm8) \ + _mm_cmpestrz( \ + HEDLEY_REINTERPRET_CAST(__v16qi, a), la, \ + HEDLEY_REINTERPRET_CAST(__v16qi, b), lb, \ + imm8) + #else + #define simde_mm_cmpestrz(a, la, b, lb, imm8) _mm_cmpestrz(a, la, b, lb, imm8) + #endif +#endif +#if defined(SIMDE_X86_SSE4_2_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpestrz + #define _mm_cmpestrz(a, la, b, lb, imm8) simde_mm_cmpestrz(a, la, b, lb, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cmpgt_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSE4_2_NATIVE) + return _mm_cmpgt_epi64(a, b); + #elif defined(SIMDE_X86_SSE2_NATIVE) + /* https://stackoverflow.com/a/65175746/501126 */ + __m128i r = _mm_and_si128(_mm_cmpeq_epi32(a, b), _mm_sub_epi64(b, a)); + r = _mm_or_si128(r, _mm_cmpgt_epi32(a, b)); + return _mm_shuffle_epi32(r, _MM_SHUFFLE(3, 3, 1, 1)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcgtq_s64(a_.neon_i64, b_.neon_i64); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + /* https://stackoverflow.com/a/65223269/501126 */ + r_.neon_i64 = vshrq_n_s64(vqsubq_s64(b_.neon_i64, a_.neon_i64), 63); + #elif defined(SIMDE_POWER_ALTIVEC_P8_NATIVE) + r_.altivec_u64 = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned long long), vec_cmpgt(a_.altivec_i64, b_.altivec_i64)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x2_gt(a_.wasm_v128, b_.wasm_v128); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 > b_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] > b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSE4_2_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpgt_epi64 + #define _mm_cmpgt_epi64(a, b) simde_mm_cmpgt_epi64(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_cmpistrs_8_(simde__m128i a) { + simde__m128i_private a_= simde__m128i_to_private(a); + const int upper_bound = (128 / 8) - 1; + int a_invalid = 0; + SIMDE_VECTORIZE + for (int i = 0 ; i <= upper_bound ; i++) { + if(!a_.i8[i]) + a_invalid = 1; + } + return a_invalid; +} + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_cmpistrs_16_(simde__m128i a) { + simde__m128i_private a_= simde__m128i_to_private(a); + const int upper_bound = (128 / 16) - 1; + int a_invalid = 0; + SIMDE_VECTORIZE + for (int i = 0 ; i <= upper_bound ; i++) { + if(!a_.i16[i]) + a_invalid = 1; + } + return a_invalid; +} + +#if defined(SIMDE_X86_SSE4_2_NATIVE) + #if defined(__clang__) && !SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + #define simde_mm_cmpistrs(a, b, imm8) \ + _mm_cmpistrs( \ + HEDLEY_REINTERPRET_CAST(__v16qi, a), \ + HEDLEY_REINTERPRET_CAST(__v16qi, b), \ + imm8) + #else + #define simde_mm_cmpistrs(a, b, imm8) _mm_cmpistrs(a, b, imm8) + #endif +#else + #define simde_mm_cmpistrs(a, b, imm8) \ + (((imm8) & SIMDE_SIDD_UWORD_OPS) \ + ? simde_mm_cmpistrs_16_((a)) \ + : simde_mm_cmpistrs_8_((a))) +#endif +#if defined(SIMDE_X86_SSE4_2_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpistrs + #define _mm_cmpistrs(a, b, imm8) simde_mm_cmpistrs(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_cmpistrz_8_(simde__m128i b) { + simde__m128i_private b_= simde__m128i_to_private(b); + const int upper_bound = (128 / 8) - 1; + int b_invalid = 0; + SIMDE_VECTORIZE + for (int i = 0 ; i <= upper_bound ; i++) { + if(!b_.i8[i]) + b_invalid = 1; + } + return b_invalid; +} + +SIMDE_FUNCTION_ATTRIBUTES +int +simde_mm_cmpistrz_16_(simde__m128i b) { + simde__m128i_private b_= simde__m128i_to_private(b); + const int upper_bound = (128 / 16) - 1; + int b_invalid = 0; + SIMDE_VECTORIZE + for (int i = 0 ; i <= upper_bound ; i++) { + if(!b_.i16[i]) + b_invalid = 1; + } + return b_invalid; +} + +#if defined(SIMDE_X86_SSE4_2_NATIVE) + #if defined(__clang__) && !SIMDE_DETECT_CLANG_VERSION_CHECK(3,8,0) + #define simde_mm_cmpistrz(a, b, imm8) \ + _mm_cmpistrz( \ + HEDLEY_REINTERPRET_CAST(__v16qi, a), \ + HEDLEY_REINTERPRET_CAST(__v16qi, b), \ + imm8) + #else + #define simde_mm_cmpistrz(a, b, imm8) _mm_cmpistrz(a, b, imm8) + #endif +#else + #define simde_mm_cmpistrz(a, b, imm8) \ + (((imm8) & SIMDE_SIDD_UWORD_OPS) \ + ? simde_mm_cmpistrz_16_((b)) \ + : simde_mm_cmpistrz_8_((b))) +#endif +#if defined(SIMDE_X86_SSE4_2_ENABLE_NATIVE_ALIASES) + #undef _mm_cmpistrz + #define _mm_cmpistrz(a, b, imm8) simde_mm_cmpistrz(a, b, imm8) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +uint32_t +simde_mm_crc32_u8(uint32_t prevcrc, uint8_t v) { + #if defined(SIMDE_X86_SSE4_2_NATIVE) + return _mm_crc32_u8(prevcrc, v); + #else + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) && defined(__ARM_FEATURE_CRC32) + return __crc32cb(prevcrc, v); + #else + uint32_t crc = prevcrc; + crc ^= v; + for(int bit = 0 ; bit < 8 ; bit++) { + if (crc & 1) + crc = (crc >> 1) ^ UINT32_C(0x82f63b78); + else + crc = (crc >> 1); + } + return crc; + #endif + #endif +} +#if defined(SIMDE_X86_SSE4_2_ENABLE_NATIVE_ALIASES) + #define _mm_crc32_u8(prevcrc, v) simde_mm_crc32_u8(prevcrc, v) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +uint32_t +simde_mm_crc32_u16(uint32_t prevcrc, uint16_t v) { + #if defined(SIMDE_X86_SSE4_2_NATIVE) + return _mm_crc32_u16(prevcrc, v); + #else + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) && defined(__ARM_FEATURE_CRC32) + return __crc32ch(prevcrc, v); + #else + uint32_t crc = prevcrc; + crc = simde_mm_crc32_u8(crc, v & 0xff); + crc = simde_mm_crc32_u8(crc, (v >> 8) & 0xff); + return crc; + #endif + #endif +} +#if defined(SIMDE_X86_SSE4_2_ENABLE_NATIVE_ALIASES) + #define _mm_crc32_u16(prevcrc, v) simde_mm_crc32_u16(prevcrc, v) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +uint32_t +simde_mm_crc32_u32(uint32_t prevcrc, uint32_t v) { + #if defined(SIMDE_X86_SSE4_2_NATIVE) + return _mm_crc32_u32(prevcrc, v); + #else + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) && defined(__ARM_FEATURE_CRC32) + return __crc32cw(prevcrc, v); + #else + uint32_t crc = prevcrc; + crc = simde_mm_crc32_u16(crc, v & 0xffff); + crc = simde_mm_crc32_u16(crc, (v >> 16) & 0xffff); + return crc; + #endif + #endif +} +#if defined(SIMDE_X86_SSE4_2_ENABLE_NATIVE_ALIASES) + #define _mm_crc32_u32(prevcrc, v) simde_mm_crc32_u32(prevcrc, v) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +uint64_t +simde_mm_crc32_u64(uint64_t prevcrc, uint64_t v) { + #if defined(SIMDE_X86_SSE4_2_NATIVE) && defined(SIMDE_ARCH_AMD64) + return _mm_crc32_u64(prevcrc, v); + #else + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) && defined(__ARM_FEATURE_CRC32) + return __crc32cd(HEDLEY_STATIC_CAST(uint32_t, prevcrc), v); + #else + uint64_t crc = prevcrc; + crc = simde_mm_crc32_u32(HEDLEY_STATIC_CAST(uint32_t, crc), v & 0xffffffff); + crc = simde_mm_crc32_u32(HEDLEY_STATIC_CAST(uint32_t, crc), (v >> 32) & 0xffffffff); + return crc; + #endif + #endif +} +#if defined(SIMDE_X86_SSE4_2_ENABLE_NATIVE_ALIASES) || (defined(SIMDE_ENABLE_NATIVE_ALIASES) && !defined(SIMDE_ARCH_AMD64)) + #define _mm_crc32_u64(prevcrc, v) simde_mm_crc32_u64(prevcrc, v) +#endif + +SIMDE_END_DECLS_ + +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_SSE4_2_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/ssse3.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/ssse3.h new file mode 100644 index 00000000..6c4c12d5 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/ssse3.h @@ -0,0 +1,1057 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2017-2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_SSSE3_H) +#define SIMDE_X86_SSSE3_H + +#include "sse3.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_abs_epi8 (simde__m128i a) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_abs_epi8(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + return _mm_min_epu8(a, _mm_sub_epi8(_mm_setzero_si128(), a)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vabsq_s8(a_.neon_i8); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i8 = vec_abs(a_.altivec_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_abs(a_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.u8[i] = HEDLEY_STATIC_CAST(uint8_t, (a_.i8[i] < 0) ? (- a_.i8[i]) : a_.i8[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_abs_epi8(a) simde_mm_abs_epi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_abs_epi16 (simde__m128i a) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_abs_epi16(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + return _mm_max_epi16(a, _mm_sub_epi16(_mm_setzero_si128(), a)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vabsq_s16(a_.neon_i16); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i16 = vec_abs(a_.altivec_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_abs(a_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, (a_.i16[i] < 0) ? (- a_.i16[i]) : a_.i16[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_abs_epi16(a) simde_mm_abs_epi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_abs_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_abs_epi32(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + const __m128i m = _mm_cmpgt_epi32(_mm_setzero_si128(), a); + return _mm_sub_epi32(_mm_xor_si128(a, m), m); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vabsq_s32(a_.neon_i32); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) || defined(SIMDE_ZARCH_ZVECTOR_13_NATIVE) + r_.altivec_i32 = vec_abs(a_.altivec_i32); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_abs(a_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + #if defined(_MSC_VER) + HEDLEY_DIAGNOSTIC_PUSH + #pragma warning(disable:4146) + #endif + r_.u32[i] = (a_.i32[i] < 0) ? (- HEDLEY_STATIC_CAST(uint32_t, a_.i32[i])) : HEDLEY_STATIC_CAST(uint32_t, a_.i32[i]); + #if defined(_MSC_VER) + HEDLEY_DIAGNOSTIC_POP + #endif + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_abs_epi32(a) simde_mm_abs_epi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_abs_pi8 (simde__m64 a) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_abs_pi8(a); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vabs_s8(a_.neon_i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.u8[i] = HEDLEY_STATIC_CAST(uint8_t, (a_.i8[i] < 0) ? (- a_.i8[i]) : a_.i8[i]); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_abs_pi8(a) simde_mm_abs_pi8(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_abs_pi16 (simde__m64 a) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_abs_pi16(a); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vabs_s16(a_.neon_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, (a_.i16[i] < 0) ? (- a_.i16[i]) : a_.i16[i]); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_abs_pi16(a) simde_mm_abs_pi16(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_abs_pi32 (simde__m64 a) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_abs_pi32(a); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vabs_s32(a_.neon_i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.u32[i] = HEDLEY_STATIC_CAST(uint32_t, (a_.i32[i] < 0) ? (- a_.i32[i]) : a_.i32[i]); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_abs_pi32(a) simde_mm_abs_pi32(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_alignr_epi8 (simde__m128i a, simde__m128i b, int count) + SIMDE_REQUIRE_CONSTANT_RANGE(count, 0, 255) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + if (HEDLEY_UNLIKELY(count > 31)) + return simde_mm_setzero_si128(); + + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + const int srcpos = count + HEDLEY_STATIC_CAST(int, i); + if (srcpos > 31) { + r_.i8[i] = 0; + } else if (srcpos > 15) { + r_.i8[i] = a_.i8[(srcpos) & 15]; + } else { + r_.i8[i] = b_.i8[srcpos]; + } + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_SSSE3_NATIVE) + #define simde_mm_alignr_epi8(a, b, count) _mm_alignr_epi8(a, b, count) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #define simde_mm_alignr_epi8(a, b, count) \ + ( \ + ((count) > 31) \ + ? simde__m128i_from_neon_i8(vdupq_n_s8(0)) \ + : ( \ + ((count) > 15) \ + ? (simde__m128i_from_neon_i8(vextq_s8(simde__m128i_to_neon_i8(a), vdupq_n_s8(0), (count) & 15))) \ + : (simde__m128i_from_neon_i8(vextq_s8(simde__m128i_to_neon_i8(b), simde__m128i_to_neon_i8(a), ((count) & 15)))))) +#endif +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) + #define _mm_alignr_epi8(a, b, count) simde_mm_alignr_epi8(a, b, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_alignr_pi8 (simde__m64 a, simde__m64 b, const int count) + SIMDE_REQUIRE_CONSTANT(count) { + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + if (HEDLEY_UNLIKELY(count > 15)) + return simde_mm_setzero_si64(); + + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + const int srcpos = count + HEDLEY_STATIC_CAST(int, i); + if (srcpos > 15) { + r_.i8[i] = 0; + } else if (srcpos > 7) { + r_.i8[i] = a_.i8[(srcpos) & 7]; + } else { + r_.i8[i] = b_.i8[srcpos]; + } + } + + return simde__m64_from_private(r_); +} +#if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) +# define simde_mm_alignr_pi8(a, b, count) _mm_alignr_pi8(a, b, count) +#elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + #define simde_mm_alignr_pi8(a, b, count) \ + ( \ + ((count) > 15) \ + ? simde__m64_from_neon_i8(vdup_n_s8(0)) \ + : ( \ + ((count) > 7) \ + ? (simde__m64_from_neon_i8(vext_s8(simde__m64_to_neon_i8(a), vdup_n_s8(0), (count) & 7))) \ + : (simde__m64_from_neon_i8(vext_s8(simde__m64_to_neon_i8(b), simde__m64_to_neon_i8(a), ((count) & 7)))))) +#endif +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_alignr_pi8(a, b, count) simde_mm_alignr_pi8(a, b, count) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_shuffle_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_shuffle_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i8 = vqtbl1q_s8(a_.neon_i8, vandq_u8(b_.neon_u8, vdupq_n_u8(0x8F))); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + /* Mask out the bits we're not interested in. vtbl will result in 0 + * for any values outside of [0, 15], so if the high bit is set it + * will return 0, just like in SSSE3. */ + b_.neon_i8 = vandq_s8(b_.neon_i8, vdupq_n_s8(HEDLEY_STATIC_CAST(int8_t, (1 << 7) | 15))); + + /* Convert a from an int8x16_t to an int8x8x2_t */ + int8x8x2_t i; + i.val[0] = vget_low_s8(a_.neon_i8); + i.val[1] = vget_high_s8(a_.neon_i8); + + /* Table lookups */ + int8x8_t l = vtbl2_s8(i, vget_low_s8(b_.neon_i8)); + int8x8_t h = vtbl2_s8(i, vget_high_s8(b_.neon_i8)); + + r_.neon_i8 = vcombine_s8(l, h); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + /* This is a bit ugly because of the casts and the awful type + * macros (SIMDE_POWER_ALTIVEC_VECTOR), but it's really just + * vec_sel(vec_perm(a, a, b), 0, vec_cmplt(b, 0)) */ + SIMDE_POWER_ALTIVEC_VECTOR(signed char) z = { 0, }; + SIMDE_POWER_ALTIVEC_VECTOR(signed char) msb_mask = HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(signed char), vec_cmplt(b_.altivec_i8, z)); + SIMDE_POWER_ALTIVEC_VECTOR(signed char) c = vec_perm(a_.altivec_i8, a_.altivec_i8, HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), b_.altivec_i8)); + r_.altivec_i8 = vec_sel(c, z, HEDLEY_REINTERPRET_CAST(SIMDE_POWER_ALTIVEC_VECTOR(unsigned char), msb_mask)); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x16_swizzle( + a_.wasm_v128, wasm_v128_and(b_.wasm_v128, wasm_i8x16_splat(0x8F))); + #else + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[b_.i8[i] & 15] & (~(b_.i8[i]) >> 7); + } + #endif + + return simde__m128i_from_private(r_); +#endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_shuffle_epi8(a, b) simde_mm_shuffle_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_shuffle_pi8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_shuffle_pi8(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + b_.neon_i8 = vand_s8(b_.neon_i8, vdup_n_s8(HEDLEY_STATIC_CAST(int8_t, (1 << 7) | 7))); + r_.neon_i8 = vtbl1_s8(a_.neon_i8, b_.neon_i8); + #else + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.i8[i] = a_.i8[b_.i8[i] & 7] & (~(b_.i8[i]) >> 7); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_shuffle_pi8(a, b) simde_mm_shuffle_pi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_hadd_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_hadd_epi16(a, b); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return simde__m128i_from_neon_i16(vpaddq_s16(simde__m128i_to_neon_i16(a), simde__m128i_to_neon_i16(b))); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int16x8x2_t t = vuzpq_s16(simde__m128i_to_neon_i16(a), simde__m128i_to_neon_i16(b)); + return simde__m128i_from_neon_i16(vaddq_s16(t.val[0], t.val[1])); + #else + return simde_mm_add_epi16(simde_x_mm_deinterleaveeven_epi16(a, b), simde_x_mm_deinterleaveodd_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hadd_epi16(a, b) simde_mm_hadd_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_hadd_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_hadd_epi32(a, b); + #elif defined(SIMDE_ARM_NEON_A64V8_NATIVE) + return simde__m128i_from_neon_i32(vpaddq_s32(simde__m128i_to_neon_i32(a), simde__m128i_to_neon_i32(b))); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int32x4x2_t t = vuzpq_s32(simde__m128i_to_neon_i32(a), simde__m128i_to_neon_i32(b)); + return simde__m128i_from_neon_i32(vaddq_s32(t.val[0], t.val[1])); + #else + return simde_mm_add_epi32(simde_x_mm_deinterleaveeven_epi32(a, b), simde_x_mm_deinterleaveodd_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hadd_epi32(a, b) simde_mm_hadd_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_hadd_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_hadd_pi16(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i16 = vpadd_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int16x4x2_t t = vuzp_s16(a_.neon_i16, b_.neon_i16); + r_.neon_i16 = vadd_s16(t.val[0], t.val[1]); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_SHUFFLE_VECTOR_) + r_.i16 = + SIMDE_SHUFFLE_VECTOR_(16, 8, a_.i16, b_.i16, 0, 2, 4, 6) + + SIMDE_SHUFFLE_VECTOR_(16, 8, a_.i16, b_.i16, 1, 3, 5, 7); + #else + r_.i16[0] = a_.i16[0] + a_.i16[1]; + r_.i16[1] = a_.i16[2] + a_.i16[3]; + r_.i16[2] = b_.i16[0] + b_.i16[1]; + r_.i16[3] = b_.i16[2] + b_.i16[3]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hadd_pi16(a, b) simde_mm_hadd_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_hadd_pi32 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_hadd_pi32(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_i32 = vpadd_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int32x2x2_t t = vuzp_s32(a_.neon_i32, b_.neon_i32); + r_.neon_i32 = vadd_s32(t.val[0], t.val[1]); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_SHUFFLE_VECTOR_) + r_.i32 = + SIMDE_SHUFFLE_VECTOR_(32, 8, a_.i32, b_.i32, 0, 2) + + SIMDE_SHUFFLE_VECTOR_(32, 8, a_.i32, b_.i32, 1, 3); + #else + r_.i32[0] = a_.i32[0] + a_.i32[1]; + r_.i32[1] = b_.i32[0] + b_.i32[1]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hadd_pi32(a, b) simde_mm_hadd_pi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_hadds_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_hadds_epi16(a, b); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int16x8x2_t t = vuzpq_s16(simde__m128i_to_neon_i16(a), simde__m128i_to_neon_i16(b)); + return simde__m128i_from_neon_i16(vqaddq_s16(t.val[0], t.val[1])); + #else + return simde_mm_adds_epi16(simde_x_mm_deinterleaveeven_epi16(a, b), simde_x_mm_deinterleaveodd_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hadds_epi16(a, b) simde_mm_hadds_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_hadds_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_hadds_pi16(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int16x4x2_t t = vuzp_s16(a_.neon_i16, b_.neon_i16); + r_.neon_i16 = vqadd_s16(t.val[0], t.val[1]); + #else + for (size_t i = 0 ; i < ((sizeof(r_.i16) / sizeof(r_.i16[0])) / 2) ; i++) { + int32_t ta = HEDLEY_STATIC_CAST(int32_t, a_.i16[i * 2]) + HEDLEY_STATIC_CAST(int32_t, a_.i16[(i * 2) + 1]); + r_.i16[ i ] = HEDLEY_LIKELY(ta > INT16_MIN) ? (HEDLEY_LIKELY(ta < INT16_MAX) ? HEDLEY_STATIC_CAST(int16_t, ta) : INT16_MAX) : INT16_MIN; + int32_t tb = HEDLEY_STATIC_CAST(int32_t, b_.i16[i * 2]) + HEDLEY_STATIC_CAST(int32_t, b_.i16[(i * 2) + 1]); + r_.i16[i + 2] = HEDLEY_LIKELY(tb > INT16_MIN) ? (HEDLEY_LIKELY(tb < INT16_MAX) ? HEDLEY_STATIC_CAST(int16_t, tb) : INT16_MAX) : INT16_MIN; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hadds_pi16(a, b) simde_mm_hadds_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_hsub_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_hsub_epi16(a, b); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int16x8x2_t t = vuzpq_s16(simde__m128i_to_neon_i16(a), simde__m128i_to_neon_i16(b)); + return simde__m128i_from_neon_i16(vsubq_s16(t.val[0], t.val[1])); + #else + return simde_mm_sub_epi16(simde_x_mm_deinterleaveeven_epi16(a, b), simde_x_mm_deinterleaveodd_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hsub_epi16(a, b) simde_mm_hsub_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_hsub_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_hsub_epi32(a, b); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int32x4x2_t t = vuzpq_s32(simde__m128i_to_neon_i32(a), simde__m128i_to_neon_i32(b)); + return simde__m128i_from_neon_i32(vsubq_s32(t.val[0], t.val[1])); + #else + return simde_mm_sub_epi32(simde_x_mm_deinterleaveeven_epi32(a, b), simde_x_mm_deinterleaveodd_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hsub_epi32(a, b) simde_mm_hsub_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_hsub_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_hsub_pi16(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int16x4x2_t t = vuzp_s16(a_.neon_i16, b_.neon_i16); + r_.neon_i16 = vsub_s16(t.val[0], t.val[1]); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_SHUFFLE_VECTOR_) + r_.i16 = + SIMDE_SHUFFLE_VECTOR_(16, 8, a_.i16, b_.i16, 0, 2, 4, 6) - + SIMDE_SHUFFLE_VECTOR_(16, 8, a_.i16, b_.i16, 1, 3, 5, 7); + #else + r_.i16[0] = a_.i16[0] - a_.i16[1]; + r_.i16[1] = a_.i16[2] - a_.i16[3]; + r_.i16[2] = b_.i16[0] - b_.i16[1]; + r_.i16[3] = b_.i16[2] - b_.i16[3]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hsub_pi16(a, b) simde_mm_hsub_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_hsub_pi32 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_hsub_pi32(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int32x2x2_t t = vuzp_s32(a_.neon_i32, b_.neon_i32); + r_.neon_i32 = vsub_s32(t.val[0], t.val[1]); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) && defined(SIMDE_SHUFFLE_VECTOR_) + r_.i32 = + SIMDE_SHUFFLE_VECTOR_(32, 8, a_.i32, b_.i32, 0, 2) - + SIMDE_SHUFFLE_VECTOR_(32, 8, a_.i32, b_.i32, 1, 3); + #else + r_.i32[0] = a_.i32[0] - a_.i32[1]; + r_.i32[1] = b_.i32[0] - b_.i32[1]; + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hsub_pi32(a, b) simde_mm_hsub_pi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_hsubs_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_hsubs_epi16(a, b); + #elif defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int16x8x2_t t = vuzpq_s16(simde__m128i_to_neon_i16(a), simde__m128i_to_neon_i16(b)); + return simde__m128i_from_neon_i16(vqsubq_s16(t.val[0], t.val[1])); + #else + return simde_mm_subs_epi16(simde_x_mm_deinterleaveeven_epi16(a, b), simde_x_mm_deinterleaveodd_epi16(a, b)); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hsubs_epi16(a, b) simde_mm_hsubs_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_hsubs_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_hsubs_pi16(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + int16x4x2_t t = vuzp_s16(a_.neon_i16, b_.neon_i16); + r_.neon_i16 = vqsub_s16(t.val[0], t.val[1]); + #else + for (size_t i = 0 ; i < ((sizeof(r_.i16) / sizeof(r_.i16[0])) / 2) ; i++) { + r_.i16[ i ] = simde_math_subs_i16(a_.i16[i * 2], a_.i16[(i * 2) + 1]); + r_.i16[i + 2] = simde_math_subs_i16(b_.i16[i * 2], b_.i16[(i * 2) + 1]); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_hsubs_pi16(a, b) simde_mm_hsubs_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maddubs_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_maddubs_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + /* Zero extend a */ + int16x8_t a_odd = vreinterpretq_s16_u16(vshrq_n_u16(a_.neon_u16, 8)); + int16x8_t a_even = vreinterpretq_s16_u16(vbicq_u16(a_.neon_u16, vdupq_n_u16(0xff00))); + + /* Sign extend by shifting left then shifting right. */ + int16x8_t b_even = vshrq_n_s16(vshlq_n_s16(b_.neon_i16, 8), 8); + int16x8_t b_odd = vshrq_n_s16(b_.neon_i16, 8); + + /* multiply */ + int16x8_t prod1 = vmulq_s16(a_even, b_even); + int16x8_t prod2 = vmulq_s16(a_odd, b_odd); + + /* saturated add */ + r_.neon_i16 = vqaddq_s16(prod1, prod2); + #else + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + const int idx = HEDLEY_STATIC_CAST(int, i) << 1; + int32_t ts = + (HEDLEY_STATIC_CAST(int16_t, a_.u8[ idx ]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[ idx ])) + + (HEDLEY_STATIC_CAST(int16_t, a_.u8[idx + 1]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[idx + 1])); + r_.i16[i] = (ts > INT16_MIN) ? ((ts < INT16_MAX) ? HEDLEY_STATIC_CAST(int16_t, ts) : INT16_MAX) : INT16_MIN; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_maddubs_epi16(a, b) simde_mm_maddubs_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_maddubs_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_maddubs_pi16(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + int16x8_t ai = vreinterpretq_s16_u16(vmovl_u8(a_.neon_u8)); + int16x8_t bi = vmovl_s8(b_.neon_i8); + int16x8_t p = vmulq_s16(ai, bi); + int16x4_t l = vget_low_s16(p); + int16x4_t h = vget_high_s16(p); + r_.neon_i16 = vqadd_s16(vuzp1_s16(l, h), vuzp2_s16(l, h)); + #else + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + const int idx = HEDLEY_STATIC_CAST(int, i) << 1; + int32_t ts = + (HEDLEY_STATIC_CAST(int16_t, a_.u8[ idx ]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[ idx ])) + + (HEDLEY_STATIC_CAST(int16_t, a_.u8[idx + 1]) * HEDLEY_STATIC_CAST(int16_t, b_.i8[idx + 1])); + r_.i16[i] = (ts > INT16_MIN) ? ((ts < INT16_MAX) ? HEDLEY_STATIC_CAST(int16_t, ts) : INT16_MAX) : INT16_MIN; + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_maddubs_pi16(a, b) simde_mm_maddubs_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_mulhrs_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_mulhrs_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + /* Multiply */ + int32x4_t mul_lo = vmull_s16(vget_low_s16(a_.neon_i16), + vget_low_s16(b_.neon_i16)); + int32x4_t mul_hi = vmull_s16(vget_high_s16(a_.neon_i16), + vget_high_s16(b_.neon_i16)); + + /* Rounding narrowing shift right + * narrow = (int16_t)((mul + 16384) >> 15); */ + int16x4_t narrow_lo = vrshrn_n_s32(mul_lo, 15); + int16x4_t narrow_hi = vrshrn_n_s32(mul_hi, 15); + + /* Join together */ + r_.neon_i16 = vcombine_s16(narrow_lo, narrow_hi); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + v128_t __lo = wasm_i32x4_mul(wasm_i32x4_extend_low_i16x8(a_.wasm_v128), wasm_i32x4_extend_low_i16x8(b_.wasm_v128)); + v128_t __hi = wasm_i32x4_mul(wasm_i32x4_extend_high_i16x8(a_.wasm_v128), wasm_i32x4_extend_high_i16x8(b_.wasm_v128)); + const v128_t __inc = wasm_i32x4_splat(0x4000); + __lo = wasm_i32x4_add(__lo, __inc); + __hi = wasm_i32x4_add(__hi, __inc); + __lo = wasm_i32x4_add(__lo, __lo); + __hi = wasm_i32x4_add(__hi, __hi); + r_.wasm_v128 = wasm_i16x8_shuffle(__lo, __hi, 1, 3, 5, 7, 9, 11, 13, 15); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, (((HEDLEY_STATIC_CAST(int32_t, a_.i16[i]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[i])) + 0x4000) >> 15)); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_mulhrs_epi16(a, b) simde_mm_mulhrs_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_mulhrs_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_mulhrs_pi16(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + /* Multiply */ + int32x4_t mul = vmull_s16(a_.neon_i16, b_.neon_i16); + + /* Rounding narrowing shift right + * narrow = (int16_t)((mul + 16384) >> 15); */ + int16x4_t narrow = vrshrn_n_s32(mul, 15); + + /* Join together */ + r_.neon_i16 = narrow; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, (((HEDLEY_STATIC_CAST(int32_t, a_.i16[i]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[i])) + 0x4000) >> 15)); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_mulhrs_pi16(a, b) simde_mm_mulhrs_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sign_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_sign_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint8x16_t aneg_mask = vreinterpretq_u8_s8(vshrq_n_s8(b_.neon_i8, 7)); + uint8x16_t bnz_mask; + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + bnz_mask = vceqzq_s8(b_.neon_i8); + #else + bnz_mask = vceqq_s8(b_.neon_i8, vdupq_n_s8(0)); + #endif + bnz_mask = vmvnq_u8(bnz_mask); + + r_.neon_i8 = vbslq_s8(aneg_mask, vnegq_s8(a_.neon_i8), vandq_s8(a_.neon_i8, vreinterpretq_s8_u8(bnz_mask))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + simde__m128i mask = wasm_i8x16_shr(b_.wasm_v128, 7); + simde__m128i zeromask = simde_mm_cmpeq_epi8(b_.wasm_v128, simde_mm_setzero_si128()); + r_.wasm_v128 = simde_mm_andnot_si128(zeromask, simde_mm_xor_si128(simde_mm_add_epi8(a_.wasm_v128, mask), mask)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (b_.i8[i] < 0) ? (- a_.i8[i]) : ((b_.i8[i] != 0) ? (a_.i8[i]) : INT8_C(0)); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_sign_epi8(a, b) simde_mm_sign_epi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sign_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_sign_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint16x8_t aneg_mask = vreinterpretq_u16_s16(vshrq_n_s16(b_.neon_i16, 15)); + uint16x8_t bnz_mask; + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + bnz_mask = vceqzq_s16(b_.neon_i16); + #else + bnz_mask = vceqq_s16(b_.neon_i16, vdupq_n_s16(0)); + #endif + bnz_mask = vmvnq_u16(bnz_mask); + + r_.neon_i16 = vbslq_s16(aneg_mask, vnegq_s16(a_.neon_i16), vandq_s16(a_.neon_i16, vreinterpretq_s16_u16(bnz_mask))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + simde__m128i mask = simde_mm_srai_epi16(b_.wasm_v128, 15); + simde__m128i zeromask = simde_mm_cmpeq_epi16(b_.wasm_v128, simde_mm_setzero_si128()); + r_.wasm_v128 = simde_mm_andnot_si128(zeromask, simde_mm_xor_si128(simde_mm_add_epi16(a_.wasm_v128, mask), mask)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (b_.i16[i] < 0) ? (- a_.i16[i]) : ((b_.i16[i] != 0) ? (a_.i16[i]) : INT16_C(0)); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_sign_epi16(a, b) simde_mm_sign_epi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sign_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_sign_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x4_t aneg_mask = vreinterpretq_u32_s32(vshrq_n_s32(b_.neon_i32, 31)); + uint32x4_t bnz_mask; + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + bnz_mask = vceqzq_s32(b_.neon_i32); + #else + bnz_mask = vceqq_s32(b_.neon_i32, vdupq_n_s32(0)); + #endif + bnz_mask = vmvnq_u32(bnz_mask); + + r_.neon_i32 = vbslq_s32(aneg_mask, vnegq_s32(a_.neon_i32), vandq_s32(a_.neon_i32, vreinterpretq_s32_u32(bnz_mask))); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + simde__m128i mask = simde_mm_srai_epi32(b_.wasm_v128, 31); + simde__m128i zeromask = simde_mm_cmpeq_epi32(b_.wasm_v128, simde_mm_setzero_si128()); + r_.wasm_v128 = simde_mm_andnot_si128(zeromask, simde_mm_xor_si128(simde_mm_add_epi32(a_.wasm_v128, mask), mask)); + #else + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (b_.i32[i] < 0) ? (- a_.i32[i]) : ((b_.i32[i] != 0) ? (a_.i32[i]) : INT32_C(0)); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_sign_epi32(a, b) simde_mm_sign_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_sign_pi8 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_sign_pi8(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint8x8_t aneg_mask = vreinterpret_u8_s8(vshr_n_s8(b_.neon_i8, 7)); + uint8x8_t bnz_mask; + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + bnz_mask = vceqz_s8(b_.neon_i8); + #else + bnz_mask = vceq_s8(b_.neon_i8, vdup_n_s8(0)); + #endif + bnz_mask = vmvn_u8(bnz_mask); + + r_.neon_i8 = vbsl_s8(aneg_mask, vneg_s8(a_.neon_i8), vand_s8(a_.neon_i8, vreinterpret_s8_u8(bnz_mask))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (b_.i8[i] < 0) ? (- a_.i8[i]) : ((b_.i8[i] != 0) ? (a_.i8[i]) : INT8_C(0)); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_sign_pi8(a, b) simde_mm_sign_pi8(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_sign_pi16 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_sign_pi16(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint16x4_t aneg_mask = vreinterpret_u16_s16(vshr_n_s16(b_.neon_i16, 15)); + uint16x4_t bnz_mask; + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + bnz_mask = vceqz_s16(b_.neon_i16); + #else + bnz_mask = vceq_s16(b_.neon_i16, vdup_n_s16(0)); + #endif + bnz_mask = vmvn_u16(bnz_mask); + + r_.neon_i16 = vbsl_s16(aneg_mask, vneg_s16(a_.neon_i16), vand_s16(a_.neon_i16, vreinterpret_s16_u16(bnz_mask))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (b_.i16[i] < 0) ? (- a_.i16[i]) : ((b_.i16[i] > 0) ? (a_.i16[i]) : INT16_C(0)); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_sign_pi16(a, b) simde_mm_sign_pi16(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m64 +simde_mm_sign_pi32 (simde__m64 a, simde__m64 b) { + #if defined(SIMDE_X86_SSSE3_NATIVE) && defined(SIMDE_X86_MMX_NATIVE) + return _mm_sign_pi32(a, b); + #else + simde__m64_private + r_, + a_ = simde__m64_to_private(a), + b_ = simde__m64_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + uint32x2_t aneg_mask = vreinterpret_u32_s32(vshr_n_s32(b_.neon_i32, 31)); + uint32x2_t bnz_mask; + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + bnz_mask = vceqz_s32(b_.neon_i32); + #else + bnz_mask = vceq_s32(b_.neon_i32, vdup_n_s32(0)); + #endif + bnz_mask = vmvn_u32(bnz_mask); + + r_.neon_i32 = vbsl_s32(aneg_mask, vneg_s32(a_.neon_i32), vand_s32(a_.neon_i32, vreinterpret_s32_u32(bnz_mask))); + #else + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (b_.i32[i] < 0) ? (- a_.i32[i]) : ((b_.i32[i] > 0) ? (a_.i32[i]) : INT32_C(0)); + } + #endif + + return simde__m64_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SSSE3_ENABLE_NATIVE_ALIASES) +# define _mm_sign_pi32(a, b) simde_mm_sign_pi32(a, b) +#endif + +SIMDE_END_DECLS_ + +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_SSE2_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/svml.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/svml.h new file mode 100644 index 00000000..40fe0cd6 --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/svml.h @@ -0,0 +1,12129 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + * 2020 Himanshi Mathur + */ + +#if !defined(SIMDE_X86_SVML_H) +#define SIMDE_X86_SVML_H + +#include "fma.h" +#include "avx2.h" +#include "avx512/abs.h" +#include "avx512/add.h" +#include "avx512/cmp.h" +#include "avx512/copysign.h" +#include "avx512/xorsign.h" +#include "avx512/div.h" +#include "avx512/fmadd.h" +#include "avx512/mov.h" +#include "avx512/mul.h" +#include "avx512/negate.h" +#include "avx512/or.h" +#include "avx512/set1.h" +#include "avx512/setone.h" +#include "avx512/setzero.h" +#include "avx512/sqrt.h" +#include "avx512/sub.h" + +#include "../simde-complex.h" + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_acos_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_acos_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_acosf4_u10(a); + #else + return Sleef_acosf4_u35(a); + #endif + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_acosf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_acos_ps + #define _mm_acos_ps(a) simde_mm_acos_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_acos_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_acos_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_acosd2_u10(a); + #else + return Sleef_acosd2_u35(a); + #endif + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_acos(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_acos_pd + #define _mm_acos_pd(a) simde_mm_acos_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_acos_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_acos_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_acosf8_u10(a); + #else + return Sleef_acosf8_u35(a); + #endif + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_acos_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_acosf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_acos_ps + #define _mm256_acos_ps(a) simde_mm256_acos_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_acos_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_acos_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_acosd4_u10(a); + #else + return Sleef_acosd4_u35(a); + #endif + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_acos_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_acos(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_acos_pd + #define _mm256_acos_pd(a) simde_mm256_acos_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_acos_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_acos_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_acosf16_u10(a); + #else + return Sleef_acosf16_u35(a); + #endif + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_acos_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_acosf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_acos_ps + #define _mm512_acos_ps(a) simde_mm512_acos_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_acos_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_acos_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_acosd8_u10(a); + #else + return Sleef_acosd8_u35(a); + #endif + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_acos_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_acos(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_acos_pd + #define _mm512_acos_pd(a) simde_mm512_acos_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_acos_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_acos_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_acos_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_acos_ps + #define _mm512_mask_acos_ps(src, k, a) simde_mm512_mask_acos_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_acos_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_acos_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_acos_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_acos_pd + #define _mm512_mask_acos_pd(src, k, a) simde_mm512_mask_acos_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_acosh_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_acosh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_acoshf4_u10(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_acoshf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_acosh_ps + #define _mm_acosh_ps(a) simde_mm_acosh_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_acosh_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_acosh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_acoshd2_u10(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_acosh(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_acosh_pd + #define _mm_acosh_pd(a) simde_mm_acosh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_acosh_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_acosh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_acoshf8_u10(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_acosh_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_acoshf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_acosh_ps + #define _mm256_acosh_ps(a) simde_mm256_acosh_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_acosh_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_acosh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_acoshd4_u10(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_acosh_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_acosh(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_acosh_pd + #define _mm256_acosh_pd(a) simde_mm256_acosh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_acosh_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_acosh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_acoshf16_u10(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_acosh_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_acoshf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_acosh_ps + #define _mm512_acosh_ps(a) simde_mm512_acosh_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_acosh_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_acosh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_acoshd8_u10(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_acosh_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_acosh(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_acosh_pd + #define _mm512_acosh_pd(a) simde_mm512_acosh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_acosh_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_acosh_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_acosh_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_acosh_ps + #define _mm512_mask_acosh_ps(src, k, a) simde_mm512_mask_acosh_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_acosh_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_acosh_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_acosh_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_acosh_pd + #define _mm512_mask_acosh_pd(src, k, a) simde_mm512_mask_acosh_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_asin_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_asin_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_asinf4_u10(a); + #else + return Sleef_asinf4_u35(a); + #endif + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_asinf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_asin_ps + #define _mm_asin_ps(a) simde_mm_asin_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_asin_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_asin_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_asind2_u10(a); + #else + return Sleef_asind2_u35(a); + #endif + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_asin(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_asin_pd + #define _mm_asin_pd(a) simde_mm_asin_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_asin_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_asin_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_asinf8_u10(a); + #else + return Sleef_asinf8_u35(a); + #endif + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_asin_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_asinf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_asin_ps + #define _mm256_asin_ps(a) simde_mm256_asin_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_asin_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_asin_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_asind4_u10(a); + #else + return Sleef_asind4_u35(a); + #endif + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_asin_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_asin(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_asin_pd + #define _mm256_asin_pd(a) simde_mm256_asin_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_asin_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_asin_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_asinf16_u10(a); + #else + return Sleef_asinf16_u35(a); + #endif + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_asin_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_asinf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_asin_ps + #define _mm512_asin_ps(a) simde_mm512_asin_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_asin_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_asin_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_asind8_u10(a); + #else + return Sleef_asind8_u35(a); + #endif + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_asin_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_asin(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_asin_pd + #define _mm512_asin_pd(a) simde_mm512_asin_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_asin_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_asin_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_asin_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_asin_ps + #define _mm512_mask_asin_ps(src, k, a) simde_mm512_mask_asin_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_asin_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_asin_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_asin_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_asin_pd + #define _mm512_mask_asin_pd(src, k, a) simde_mm512_mask_asin_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_asinh_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_asinh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_asinhf4_u10(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_asinhf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_asinh_ps + #define _mm_asinh_ps(a) simde_mm_asinh_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_asinh_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_asinh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_asinhd2_u10(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_asinh(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_asinh_pd + #define _mm_asinh_pd(a) simde_mm_asinh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_asinh_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_asinh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_asinhf8_u10(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_asinh_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_asinhf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_asinh_ps + #define _mm256_asinh_ps(a) simde_mm256_asinh_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_asinh_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_asinh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_asinhd4_u10(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_asinh_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_asinh(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_asinh_pd + #define _mm256_asinh_pd(a) simde_mm256_asinh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_asinh_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_asinh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_asinhf16_u10(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_asinh_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_asinhf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_asinh_ps + #define _mm512_asinh_ps(a) simde_mm512_asinh_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_asinh_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_asinh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_asinhd8_u10(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_asinh_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_asinh(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_asinh_pd + #define _mm512_asinh_pd(a) simde_mm512_asinh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_asinh_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_asinh_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_asinh_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_asinh_ps + #define _mm512_mask_asinh_ps(src, k, a) simde_mm512_mask_asinh_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_asinh_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_asinh_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_asinh_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_asinh_pd + #define _mm512_mask_asinh_pd(src, k, a) simde_mm512_mask_asinh_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_atan_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_atan_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_atanf4_u10(a); + #else + return Sleef_atanf4_u35(a); + #endif + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_atanf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_atan_ps + #define _mm_atan_ps(a) simde_mm_atan_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_atan_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_atan_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_atand2_u10(a); + #else + return Sleef_atand2_u35(a); + #endif + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_atan(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_atan_pd + #define _mm_atan_pd(a) simde_mm_atan_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_atan_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_atan_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_atanf8_u10(a); + #else + return Sleef_atanf8_u35(a); + #endif + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_atan_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_atanf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_atan_ps + #define _mm256_atan_ps(a) simde_mm256_atan_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_atan_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_atan_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_atand4_u10(a); + #else + return Sleef_atand4_u35(a); + #endif + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_atan_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_atan(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_atan_pd + #define _mm256_atan_pd(a) simde_mm256_atan_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_atan_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_atan_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_atanf16_u10(a); + #else + return Sleef_atanf16_u35(a); + #endif + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_atan_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_atanf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_atan_ps + #define _mm512_atan_ps(a) simde_mm512_atan_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_atan_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_atan_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_atand8_u10(a); + #else + return Sleef_atand8_u35(a); + #endif + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_atan_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_atan(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_atan_pd + #define _mm512_atan_pd(a) simde_mm512_atan_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_atan_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_atan_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_atan_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_atan_ps + #define _mm512_mask_atan_ps(src, k, a) simde_mm512_mask_atan_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_atan_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_atan_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_atan_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_atan_pd + #define _mm512_mask_atan_pd(src, k, a) simde_mm512_mask_atan_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_atan2_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_atan2_ps(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_atan2f4_u10(a, b); + #else + return Sleef_atan2f4_u35(a, b); + #endif + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_atan2f(a_.f32[i], b_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_atan2_ps + #define _mm_atan2_ps(a, b) simde_mm_atan2_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_atan2_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_atan2_pd(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_atan2d2_u10(a, b); + #else + return Sleef_atan2d2_u35(a, b); + #endif + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_atan2(a_.f64[i], b_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_atan2_pd + #define _mm_atan2_pd(a, b) simde_mm_atan2_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_atan2_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_atan2_ps(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_atan2f8_u10(a, b); + #else + return Sleef_atan2f8_u35(a, b); + #endif + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_atan2_ps(a_.m128[i], b_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_atan2f(a_.f32[i], b_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_atan2_ps + #define _mm256_atan2_ps(a, b) simde_mm256_atan2_ps(a, b) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_atan2_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_atan2_pd(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_atan2d4_u10(a, b); + #else + return Sleef_atan2d4_u35(a, b); + #endif + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_atan2_pd(a_.m128d[i], b_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_atan2(a_.f64[i], b_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_atan2_pd + #define _mm256_atan2_pd(a, b) simde_mm256_atan2_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_atan2_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_atan2_ps(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_atan2f16_u10(a, b); + #else + return Sleef_atan2f16_u35(a, b); + #endif + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_atan2_ps(a_.m256[i], b_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_atan2f(a_.f32[i], b_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_atan2_ps + #define _mm512_atan2_ps(a, b) simde_mm512_atan2_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_atan2_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_atan2_pd(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_atan2d8_u10(a, b); + #else + return Sleef_atan2d8_u35(a, b); + #endif + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_atan2_pd(a_.m256d[i], b_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_atan2(a_.f64[i], b_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_atan2_pd + #define _mm512_atan2_pd(a, b) simde_mm512_atan2_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_atan2_ps(simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_atan2_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_atan2_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_atan2_ps + #define _mm512_mask_atan2_ps(src, k, a, b) simde_mm512_mask_atan2_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_atan2_pd(simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_atan2_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_atan2_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_atan2_pd + #define _mm512_mask_atan2_pd(src, k, a, b) simde_mm512_mask_atan2_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_atanh_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_atanh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_atanhf4_u10(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_atanhf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_atanh_ps + #define _mm_atanh_ps(a) simde_mm_atanh_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_atanh_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_atanh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_atanhd2_u10(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_atanh(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_atanh_pd + #define _mm_atanh_pd(a) simde_mm_atanh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_atanh_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_atanh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_atanhf8_u10(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_atanh_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_atanhf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_atanh_ps + #define _mm256_atanh_ps(a) simde_mm256_atanh_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_atanh_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_atanh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_atanhd4_u10(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_atanh_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_atanh(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_atanh_pd + #define _mm256_atanh_pd(a) simde_mm256_atanh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_atanh_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_atanh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_atanhf16_u10(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_atanh_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_atanhf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_atanh_ps + #define _mm512_atanh_ps(a) simde_mm512_atanh_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_atanh_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_atanh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_atanhd8_u10(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_atanh_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_atanh(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_atanh_pd + #define _mm512_atanh_pd(a) simde_mm512_atanh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_atanh_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_atanh_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_atanh_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_atanh_ps + #define _mm512_mask_atanh_ps(src, k, a) simde_mm512_mask_atanh_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_atanh_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_atanh_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_atanh_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_atanh_pd + #define _mm512_mask_atanh_pd(src, k, a) simde_mm512_mask_atanh_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cbrt_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_cbrt_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_cbrtf4_u10(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_cbrtf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_cbrt_ps + #define _mm_cbrt_ps(a) simde_mm_cbrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cbrt_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_cbrt_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_cbrtd2_u10(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cbrt(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_cbrt_pd + #define _mm_cbrt_pd(a) simde_mm_cbrt_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_cbrt_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cbrt_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_cbrtf8_u10(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_cbrt_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_cbrtf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_cbrt_ps + #define _mm256_cbrt_ps(a) simde_mm256_cbrt_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_cbrt_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cbrt_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_cbrtd4_u10(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_cbrt_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cbrt(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_cbrt_pd + #define _mm256_cbrt_pd(a) simde_mm256_cbrt_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_cbrt_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cbrt_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_cbrtf16_u10(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_cbrt_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_cbrtf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_cbrt_ps + #define _mm512_cbrt_ps(a) simde_mm512_cbrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_cbrt_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cbrt_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_cbrtd8_u10(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_cbrt_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cbrt(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_cbrt_pd + #define _mm512_cbrt_pd(a) simde_mm512_cbrt_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_cbrt_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cbrt_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_cbrt_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cbrt_ps + #define _mm512_mask_cbrt_ps(src, k, a) simde_mm512_mask_cbrt_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_cbrt_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cbrt_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_cbrt_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cbrt_pd + #define _mm512_mask_cbrt_pd(src, k, a) simde_mm512_mask_cbrt_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cexp_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_cexp_ps(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i+=2) { + simde_cfloat32 val = simde_math_cexpf(SIMDE_MATH_CMPLXF(a_.f32[i], a_.f32[i+1])); + r_.f32[ i ] = simde_math_crealf(val); + r_.f32[i + 1] = simde_math_cimagf(val); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_cexp_ps + #define _mm_cexp_ps(a) simde_mm_cexp_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_cexp_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cexp_ps(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i+=2) { + simde_cfloat32 val = simde_math_cexpf(SIMDE_MATH_CMPLXF(a_.f32[i], a_.f32[i+1])); + r_.f32[ i ] = simde_math_crealf(val); + r_.f32[i + 1] = simde_math_cimagf(val); + } + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_cexp_ps + #define _mm256_cexp_ps(a) simde_mm256_cexp_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cos_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_cos_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_cosf4_u10(a); + #else + return Sleef_cosf4_u35(a); + #endif + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_cosf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_cos_ps + #define _mm_cos_ps(a) simde_mm_cos_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cos_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_cos_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_cosd2_u10(a); + #else + return Sleef_cosd2_u35(a); + #endif + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cos(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_cos_pd + #define _mm_cos_pd(a) simde_mm_cos_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_cos_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cos_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_cosf8_u10(a); + #else + return Sleef_cosf8_u35(a); + #endif + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_cos_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_cosf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_cos_ps + #define _mm256_cos_ps(a) simde_mm256_cos_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_cos_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cos_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_cosd4_u10(a); + #else + return Sleef_cosd4_u35(a); + #endif + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_cos_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cos(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_cos_pd + #define _mm256_cos_pd(a) simde_mm256_cos_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_cos_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cos_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_cosf16_u10(a); + #else + return Sleef_cosf16_u35(a); + #endif + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_cos_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_cosf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_cos_ps + #define _mm512_cos_ps(a) simde_mm512_cos_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_cos_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cos_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_cosd8_u10(a); + #else + return Sleef_cosd8_u35(a); + #endif + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_cos_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cos(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_cos_pd + #define _mm512_cos_pd(a) simde_mm512_cos_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_cos_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cos_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_cos_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cos_ps + #define _mm512_mask_cos_ps(src, k, a) simde_mm512_mask_cos_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_cos_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cos_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_cos_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cos_pd + #define _mm512_mask_cos_pd(src, k, a) simde_mm512_mask_cos_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_x_mm_deg2rad_ps(simde__m128 a) { + #if SIMDE_NATURAL_VECTOR_SIZE_GE(128) + return simde_mm_mul_ps(a, simde_mm_set1_ps(SIMDE_MATH_PI_OVER_180F)); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_f32 = vmulq_n_f32(a_.neon_i32, SIMDE_MATH_PI_OVER_180F); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && !defined(SIMDE_BUG_GCC_53784) + r_.f32 = a_.f32 * SIMDE_MATH_PI_OVER_180F; + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + const __typeof__(r_.f32) tmp = { SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F }; + r_.f32 = a_.f32 * tmp; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_deg2radf(a_.f32[i]); + } + + #endif + return simde__m128_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_x_mm_deg2rad_pd(simde__m128d a) { + #if SIMDE_NATURAL_VECTOR_SIZE_GE(128) + return simde_mm_mul_pd(a, simde_mm_set1_pd(SIMDE_MATH_PI_OVER_180)); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_f64 = vmulq_n_f64(a_.neon_i64, SIMDE_MATH_PI_OVER_180); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && !defined(SIMDE_BUG_GCC_53784) + r_.f64 = a_.f64 * SIMDE_MATH_PI_OVER_180; + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + const __typeof__(r_.f64) tmp = { SIMDE_MATH_PI_OVER_180, SIMDE_MATH_PI_OVER_180 }; + r_.f64 = a_.f64 * tmp; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_deg2rad(a_.f64[i]); + } + + #endif + return simde__m128d_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_x_mm256_deg2rad_ps(simde__m256 a) { + #if SIMDE_NATURAL_VECTOR_SIZE_GE(256) + return simde_mm256_mul_ps(a, simde_mm256_set1_ps(SIMDE_MATH_PI_OVER_180F)); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_x_mm_deg2rad_ps(a_.m128[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && !defined(SIMDE_BUG_GCC_53784) + r_.f32 = a_.f32 * SIMDE_MATH_PI_OVER_180F; + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + const __typeof__(r_.f32) tmp = { + SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, + SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F + }; + r_.f32 = a_.f32 * tmp; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_deg2radf(a_.f32[i]); + } + + #endif + return simde__m256_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_x_mm256_deg2rad_pd(simde__m256d a) { + #if SIMDE_NATURAL_VECTOR_SIZE_GE(256) + return simde_mm256_mul_pd(a, simde_mm256_set1_pd(SIMDE_MATH_PI_OVER_180)); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_x_mm_deg2rad_pd(a_.m128d[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && !defined(SIMDE_BUG_GCC_53784) + r_.f64 = a_.f64 * SIMDE_MATH_PI_OVER_180; + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + const __typeof__(r_.f64) tmp = { SIMDE_MATH_PI_OVER_180, SIMDE_MATH_PI_OVER_180, SIMDE_MATH_PI_OVER_180, SIMDE_MATH_PI_OVER_180 }; + r_.f64 = a_.f64 * tmp; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_deg2rad(a_.f64[i]); + } + + #endif + return simde__m256d_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_x_mm512_deg2rad_ps(simde__m512 a) { + #if SIMDE_NATURAL_VECTOR_SIZE_GE(512) + return simde_mm512_mul_ps(a, simde_mm512_set1_ps(SIMDE_MATH_PI_OVER_180F)); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_x_mm256_deg2rad_ps(a_.m256[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && !defined(SIMDE_BUG_GCC_53784) + r_.f32 = a_.f32 * SIMDE_MATH_PI_OVER_180F; + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + const __typeof__(r_.f32) tmp = { + SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, + SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, + SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, + SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F, SIMDE_MATH_PI_OVER_180F + }; + r_.f32 = a_.f32 * tmp; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_deg2radf(a_.f32[i]); + } + + #endif + return simde__m512_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_x_mm512_deg2rad_pd(simde__m512d a) { + #if SIMDE_NATURAL_VECTOR_SIZE_GE(512) + return simde_mm512_mul_pd(a, simde_mm512_set1_pd(SIMDE_MATH_PI_OVER_180)); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_x_mm256_deg2rad_pd(a_.m256d[i]); + } + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) && !defined(SIMDE_BUG_GCC_53784) + r_.f64 = a_.f64 * SIMDE_MATH_PI_OVER_180; + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + const __typeof__(r_.f64) tmp = { + SIMDE_MATH_PI_OVER_180, SIMDE_MATH_PI_OVER_180, SIMDE_MATH_PI_OVER_180, SIMDE_MATH_PI_OVER_180, + SIMDE_MATH_PI_OVER_180, SIMDE_MATH_PI_OVER_180, SIMDE_MATH_PI_OVER_180, SIMDE_MATH_PI_OVER_180 + }; + r_.f64 = a_.f64 * tmp; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_deg2rad(a_.f64[i]); + } + + #endif + return simde__m512d_from_private(r_); + #endif +} + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cosd_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_cosd_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_cosf4_u10(simde_x_mm_deg2rad_ps(a)); + #else + return Sleef_cosf4_u35(simde_x_mm_deg2rad_ps(a)); + #endif + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_cosf(simde_math_deg2radf(a_.f32[i])); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_cosd_ps + #define _mm_cosd_ps(a) simde_mm_cosd_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cosd_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_cosd_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_cosd2_u10(simde_x_mm_deg2rad_pd(a)); + #else + return Sleef_cosd2_u35(simde_x_mm_deg2rad_pd(a)); + #endif + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cos(simde_math_deg2rad(a_.f64[i])); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_cosd_pd + #define _mm_cosd_pd(a) simde_mm_cosd_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_cosd_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cosd_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_cosf8_u10(simde_x_mm256_deg2rad_ps(a)); + #else + return Sleef_cosf8_u35(simde_x_mm256_deg2rad_ps(a)); + #endif + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_cosd_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_cosf(simde_math_deg2radf(a_.f32[i])); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_cosd_ps + #define _mm256_cosd_ps(a) simde_mm256_cosd_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_cosd_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cosd_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_cosd4_u10(simde_x_mm256_deg2rad_pd(a)); + #else + return Sleef_cosd4_u35(simde_x_mm256_deg2rad_pd(a)); + #endif + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_cosd_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cos(simde_math_deg2rad(a_.f64[i])); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_cosd_pd + #define _mm256_cosd_pd(a) simde_mm256_cosd_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_cosd_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cosd_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_cosf16_u10(simde_x_mm512_deg2rad_ps(a)); + #else + return Sleef_cosf16_u35(simde_x_mm512_deg2rad_ps(a)); + #endif + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_cosd_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_cosf(simde_math_deg2radf(a_.f32[i])); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_cosd_ps + #define _mm512_cosd_ps(a) simde_mm512_cosd_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_cosd_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cosd_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_cosd8_u10(simde_x_mm512_deg2rad_pd(a)); + #else + return Sleef_cosd8_u35(simde_x_mm512_deg2rad_pd(a)); + #endif + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_cosd_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cos(simde_math_deg2rad(a_.f64[i])); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_cosd_pd + #define _mm512_cosd_pd(a) simde_mm512_cosd_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_cosd_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cosd_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_cosd_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cosd_ps + #define _mm512_mask_cosd_ps(src, k, a) simde_mm512_mask_cosd_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_cosd_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cosd_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_cosd_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cosd_pd + #define _mm512_mask_cosd_pd(src, k, a) simde_mm512_mask_cosd_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cosh_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_cosh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_coshf4_u10(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_coshf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_cosh_ps + #define _mm_cosh_ps(a) simde_mm_cosh_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cosh_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_cosh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_coshd2_u10(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cosh(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_cosh_pd + #define _mm_cosh_pd(a) simde_mm_cosh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_cosh_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cosh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_coshf8_u10(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_cosh_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_coshf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_cosh_ps + #define _mm256_cosh_ps(a) simde_mm256_cosh_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_cosh_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cosh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_coshd4_u10(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_cosh_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cosh(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_cosh_pd + #define _mm256_cosh_pd(a) simde_mm256_cosh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_cosh_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cosh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_coshf16_u10(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_cosh_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_coshf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_cosh_ps + #define _mm512_cosh_ps(a) simde_mm512_cosh_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_cosh_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cosh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_coshd8_u10(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_cosh_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cosh(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_cosh_pd + #define _mm512_cosh_pd(a) simde_mm512_cosh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_cosh_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cosh_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_cosh_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cosh_ps + #define _mm512_mask_cosh_ps(src, k, a) simde_mm512_mask_cosh_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_cosh_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cosh_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_cosh_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cosh_pd + #define _mm512_mask_cosh_pd(src, k, a) simde_mm512_mask_cosh_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_div_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_div_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = a_.i8 / b_.i8; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i8x4_div(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] / b_.i8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_div_epi8 + #define _mm_div_epi8(a, b) simde_mm_div_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_div_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_div_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = a_.i16 / b_.i16; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x4_div(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] / b_.i16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_div_epi16 + #define _mm_div_epi16(a, b) simde_mm_div_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_div_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_div_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 / b_.i32; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_div(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] / b_.i32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#define simde_mm_idiv_epi32(a, b) simde_mm_div_epi32(a, b) +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_div_epi32 + #define _mm_div_epi32(a, b) simde_mm_div_epi32(a, b) + #undef _mm_idiv_epi32 + #define _mm_idiv_epi32(a, b) simde_mm_div_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_div_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_div_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 / b_.i64; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i64x4_div(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] / b_.i64[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_div_epi64 + #define _mm_div_epi64(a, b) simde_mm_div_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_div_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_div_epu8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = a_.u8 / b_.u8; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u8x16_div(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = a_.u8[i] / b_.u8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_div_epu8 + #define _mm_div_epu8(a, b) simde_mm_div_epu8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_div_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_div_epu16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = a_.u16 / b_.u16; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u16x16_div(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = a_.u16[i] / b_.u16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_div_epu16 + #define _mm_div_epu16(a, b) simde_mm_div_epu16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_div_epu32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_div_epu32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = a_.u32 / b_.u32; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u32x16_div(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] / b_.u32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#define simde_mm_udiv_epi32(a, b) simde_mm_div_epu32(a, b) +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_div_epu32 + #define _mm_div_epu32(a, b) simde_mm_div_epu32(a, b) + #undef _mm_udiv_epi32 + #define _mm_udiv_epi32(a, b) simde_mm_div_epu32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_div_epu64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_div_epu64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = a_.u64 / b_.u64; + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u64x16_div(a_.wasm_v128, b_.wasm_v128); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = a_.u64[i] / b_.u64[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_div_epu64 + #define _mm_div_epu64(a, b) simde_mm_div_epu64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_div_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_div_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = a_.i8 / b_.i8; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_div_epi8(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] / b_.i8[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_div_epi8 + #define _mm256_div_epi8(a, b) simde_mm256_div_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_div_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_div_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = a_.i16 / b_.i16; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_div_epi16(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] / b_.i16[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_div_epi16 + #define _mm256_div_epi16(a, b) simde_mm256_div_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_div_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_div_epi32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 / b_.i32; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_div_epi32(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] / b_.i32[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#define simde_mm256_idiv_epi32(a, b) simde_mm256_div_epi32(a, b) +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_div_epi32 + #define _mm256_div_epi32(a, b) simde_mm256_div_epi32(a, b) + #undef _mm256_idiv_epi32 + #define _mm256_idiv_epi32(a, b) simde_mm256_div_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_div_epi64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_div_epi64(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 / b_.i64; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_div_epi64(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] / b_.i64[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_div_epi64 + #define _mm256_div_epi64(a, b) simde_mm256_div_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_div_epu8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_div_epu8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = a_.u8 / b_.u8; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_div_epu8(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = a_.u8[i] / b_.u8[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_div_epu8 + #define _mm256_div_epu8(a, b) simde_mm256_div_epu8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_div_epu16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_div_epu16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = a_.u16 / b_.u16; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_div_epu16(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = a_.u16[i] / b_.u16[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_div_epu16 + #define _mm256_div_epu16(a, b) simde_mm256_div_epu16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_div_epu32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_div_epu32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = a_.u32 / b_.u32; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_div_epu32(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] / b_.u32[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#define simde_mm256_udiv_epi32(a, b) simde_mm256_div_epu32(a, b) +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_div_epu32 + #define _mm256_div_epu32(a, b) simde_mm256_div_epu32(a, b) + #undef _mm256_udiv_epi32 + #define _mm256_udiv_epi32(a, b) simde_mm256_div_epu32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_div_epu64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_div_epu64(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = a_.u64 / b_.u64; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_div_epu64(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = a_.u64[i] / b_.u64[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_div_epu64 + #define _mm256_div_epu64(a, b) simde_mm256_div_epu64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_div_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_div_epi8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = a_.i8 / b_.i8; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_div_epi8(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] / b_.i8[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_div_epi8 + #define _mm512_div_epi8(a, b) simde_mm512_div_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_div_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_div_epi16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = a_.i16 / b_.i16; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_div_epi16(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] / b_.i16[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_div_epi16 + #define _mm512_div_epi16(a, b) simde_mm512_div_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_div_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_div_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = a_.i32 / b_.i32; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_div_epi32(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] / b_.i32[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_div_epi32 + #define _mm512_div_epi32(a, b) simde_mm512_div_epi32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_div_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_div_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_div_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_div_epi32 + #define _mm512_mask_div_epi32(src, k, a, b) simde_mm512_mask_div_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_div_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_div_epi64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = a_.i64 / b_.i64; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_div_epi64(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] / b_.i64[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_div_epi64 + #define _mm512_div_epi64(a, b) simde_mm512_div_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_div_epu8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_div_epu8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = a_.u8 / b_.u8; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_div_epu8(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = a_.u8[i] / b_.u8[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_div_epu8 + #define _mm512_div_epu8(a, b) simde_mm512_div_epu8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_div_epu16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_div_epu16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = a_.u16 / b_.u16; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_div_epu16(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = a_.u16[i] / b_.u16[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_div_epu16 + #define _mm512_div_epu16(a, b) simde_mm512_div_epu16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_div_epu32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_div_epu32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = a_.u32 / b_.u32; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_div_epu32(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] / b_.u32[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_div_epu32 + #define _mm512_div_epu32(a, b) simde_mm512_div_epu32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_div_epu32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_div_epu32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_div_epu32(a, b)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_div_epu32 + #define _mm512_mask_div_epu32(src, k, a, b) simde_mm512_mask_div_epu32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_div_epu64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_div_epu64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = a_.u64 / b_.u64; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_div_epu64(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = a_.u64[i] / b_.u64[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_div_epu64 + #define _mm512_div_epu64(a, b) simde_mm512_div_epu64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_erf_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_erf_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_erff4_u10(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_erff(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_erf_ps + #define _mm_erf_ps(a) simde_mm_erf_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_erf_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_erf_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_erfd2_u10(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_erf(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_erf_pd + #define _mm_erf_pd(a) simde_mm_erf_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_erf_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_erf_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_erff8_u10(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_erf_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_erff(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_erf_ps + #define _mm256_erf_ps(a) simde_mm256_erf_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_erf_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_erf_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_erfd4_u10(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_erf_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_erf(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_erf_pd + #define _mm256_erf_pd(a) simde_mm256_erf_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_erf_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_erf_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_erff16_u10(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_erf_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_erff(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_erf_ps + #define _mm512_erf_ps(a) simde_mm512_erf_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_erf_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_erf_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_erfd8_u10(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_erf_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_erf(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_erf_pd + #define _mm512_erf_pd(a) simde_mm512_erf_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_erf_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_erf_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_erf_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_erf_ps + #define _mm512_mask_erf_ps(src, k, a) simde_mm512_mask_erf_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_erf_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_erf_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_erf_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_erf_pd + #define _mm512_mask_erf_pd(src, k, a) simde_mm512_mask_erf_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_erfc_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_erfc_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_erfcf4_u15(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_erfcf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_erfc_ps + #define _mm_erfc_ps(a) simde_mm_erfc_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_erfc_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_erfc_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_erfcd2_u15(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_erfc(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_erfc_pd + #define _mm_erfc_pd(a) simde_mm_erfc_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_erfc_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_erfc_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_erfcf8_u15(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_erfc_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_erfcf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_erfc_ps + #define _mm256_erfc_ps(a) simde_mm256_erfc_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_erfc_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_erfc_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_erfcd4_u15(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_erfc_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_erfc(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_erfc_pd + #define _mm256_erfc_pd(a) simde_mm256_erfc_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_erfc_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_erfc_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_erfcf16_u15(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_erfc_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_erfcf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_erfc_ps + #define _mm512_erfc_ps(a) simde_mm512_erfc_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_erfc_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_erfc_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_erfcd8_u15(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_erfc_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_erfc(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_erfc_pd + #define _mm512_erfc_pd(a) simde_mm512_erfc_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_erfc_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_erfc_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_erfc_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_erfc_ps + #define _mm512_mask_erfc_ps(src, k, a) simde_mm512_mask_erfc_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_erfc_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_erfc_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_erfc_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_erfc_pd + #define _mm512_mask_erfc_pd(src, k, a) simde_mm512_mask_erfc_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_exp_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_exp_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_expf4_u10(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_expf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_exp_ps + #define _mm_exp_ps(a) simde_mm_exp_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_exp_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_exp_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_expd2_u10(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_exp(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_exp_pd + #define _mm_exp_pd(a) simde_mm_exp_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_exp_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_exp_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_expf8_u10(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_exp_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_expf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_exp_ps + #define _mm256_exp_ps(a) simde_mm256_exp_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_exp_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_exp_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_expd4_u10(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_exp_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_exp(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_exp_pd + #define _mm256_exp_pd(a) simde_mm256_exp_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_exp_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_exp_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_expf16_u10(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_exp_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_expf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_exp_ps + #define _mm512_exp_ps(a) simde_mm512_exp_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_exp_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_exp_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_expd8_u10(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_exp_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_exp(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_exp_pd + #define _mm512_exp_pd(a) simde_mm512_exp_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_exp_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_exp_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_exp_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_exp_ps + #define _mm512_mask_exp_ps(src, k, a) simde_mm512_mask_exp_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_exp_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_exp_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_exp_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_exp_pd + #define _mm512_mask_exp_pd(src, k, a) simde_mm512_mask_exp_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_expm1_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_expm1_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_expm1f4_u10(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_expm1f(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_expm1_ps + #define _mm_expm1_ps(a) simde_mm_expm1_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_expm1_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_expm1_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_expm1d2_u10(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_expm1(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_expm1_pd + #define _mm_expm1_pd(a) simde_mm_expm1_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_expm1_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_expm1_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_expm1f8_u10(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_expm1_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_expm1f(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_expm1_ps + #define _mm256_expm1_ps(a) simde_mm256_expm1_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_expm1_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_expm1_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_expm1d4_u10(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_expm1_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_expm1(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_expm1_pd + #define _mm256_expm1_pd(a) simde_mm256_expm1_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_expm1_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_expm1_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_expm1f16_u10(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_expm1_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_expm1f(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_expm1_ps + #define _mm512_expm1_ps(a) simde_mm512_expm1_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_expm1_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_expm1_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_expm1d8_u10(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_expm1_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_expm1(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_expm1_pd + #define _mm512_expm1_pd(a) simde_mm512_expm1_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_expm1_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_expm1_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_expm1_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_expm1_ps + #define _mm512_mask_expm1_ps(src, k, a) simde_mm512_mask_expm1_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_expm1_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_expm1_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_expm1_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_expm1_pd + #define _mm512_mask_expm1_pd(src, k, a) simde_mm512_mask_expm1_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_exp2_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_exp2_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_exp2f4_u10(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_exp2f(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_exp2_ps + #define _mm_exp2_ps(a) simde_mm_exp2_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_exp2_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_exp2_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_exp2d2_u10(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_exp2(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_exp2_pd + #define _mm_exp2_pd(a) simde_mm_exp2_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_exp2_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_exp2_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_exp2f8_u10(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_exp2_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_exp2f(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_exp2_ps + #define _mm256_exp2_ps(a) simde_mm256_exp2_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_exp2_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_exp2_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_exp2d4_u10(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_exp2_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_exp2(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_exp2_pd + #define _mm256_exp2_pd(a) simde_mm256_exp2_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_exp2_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_exp2_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_exp2f16_u10(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_exp2_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_exp2f(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_exp2_ps + #define _mm512_exp2_ps(a) simde_mm512_exp2_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_exp2_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_exp2_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_exp2d8_u10(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_exp2_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_exp2(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_exp2_pd + #define _mm512_exp2_pd(a) simde_mm512_exp2_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_exp2_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_exp2_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_exp2_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_exp2_ps + #define _mm512_mask_exp2_ps(src, k, a) simde_mm512_mask_exp2_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_exp2_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_exp2_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_exp2_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_exp2_pd + #define _mm512_mask_exp2_pd(src, k, a) simde_mm512_mask_exp2_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_exp10_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_exp10_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_exp10f4_u10(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_exp10f(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_exp10_ps + #define _mm_exp10_ps(a) simde_mm_exp10_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_exp10_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_exp10_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_exp10d2_u10(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_exp10(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_exp10_pd + #define _mm_exp10_pd(a) simde_mm_exp10_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_exp10_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_exp10_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_exp10f8_u10(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_exp10_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_exp10f(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_exp10_ps + #define _mm256_exp10_ps(a) simde_mm256_exp10_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_exp10_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_exp10_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_exp10d4_u10(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_exp10_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_exp10(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_exp10_pd + #define _mm256_exp10_pd(a) simde_mm256_exp10_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_exp10_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_exp10_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_exp10f16_u10(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_exp10_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_exp10f(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_exp10_ps + #define _mm512_exp10_ps(a) simde_mm512_exp10_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_exp10_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_exp10_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_exp10d8_u10(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_exp10_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_exp10(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_exp10_pd + #define _mm512_exp10_pd(a) simde_mm512_exp10_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_exp10_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_exp10_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_exp10_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_exp10_ps + #define _mm512_mask_exp10_ps(src, k, a) simde_mm512_mask_exp10_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_exp10_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_exp10_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_exp10_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_exp10_pd + #define _mm512_mask_exp10_pd(src, k, a) simde_mm512_mask_exp10_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cdfnorm_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_cdfnorm_ps(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + /* https://www.johndcook.com/blog/cpp_phi/ */ + const simde__m128 a1 = simde_mm_set1_ps(SIMDE_FLOAT32_C( 0.254829592)); + const simde__m128 a2 = simde_mm_set1_ps(SIMDE_FLOAT32_C(-0.284496736)); + const simde__m128 a3 = simde_mm_set1_ps(SIMDE_FLOAT32_C(1.421413741)); + const simde__m128 a4 = simde_mm_set1_ps(SIMDE_FLOAT32_C(-1.453152027)); + const simde__m128 a5 = simde_mm_set1_ps(SIMDE_FLOAT32_C(1.061405429)); + const simde__m128 p = simde_mm_set1_ps(SIMDE_FLOAT32_C(0.3275911)); + const simde__m128 one = simde_mm_set1_ps(SIMDE_FLOAT32_C(1.0)); + + /* simde_math_fabsf(x) / sqrtf(2.0) */ + const simde__m128 x = simde_mm_div_ps(simde_x_mm_abs_ps(a), simde_mm_sqrt_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C(2.0)))); + + /* 1.0 / (1.0 + p * x) */ + const simde__m128 t = simde_mm_div_ps(one, simde_mm_add_ps(one, simde_mm_mul_ps(p, x))); + + /* 1.0 - (((((a5 * t + a4) * t) + a3) * t + a2) * t + a1) * t * exp(-x * x) */ + simde__m128 y = simde_mm_mul_ps(a5, t); + y = simde_mm_add_ps(y, a4); + y = simde_mm_mul_ps(y, t); + y = simde_mm_add_ps(y, a3); + y = simde_mm_mul_ps(y, t); + y = simde_mm_add_ps(y, a2); + y = simde_mm_mul_ps(y, t); + y = simde_mm_add_ps(y, a1); + y = simde_mm_mul_ps(y, t); + y = simde_mm_mul_ps(y, simde_mm_exp_ps(simde_mm_mul_ps(x, simde_x_mm_negate_ps(x)))); + y = simde_mm_sub_ps(one, y); + + /* 0.5 * (1.0 + ((a < 0.0) ? -y : y)) */ + return simde_mm_mul_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C(0.5)), simde_mm_add_ps(one, simde_x_mm_xorsign_ps(y, a))); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_cdfnormf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_cdfnorm_ps + #define _mm_cdfnorm_ps(a) simde_mm_cdfnorm_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cdfnorm_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_cdfnorm_pd(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + /* https://www.johndcook.com/blog/cpp_phi/ */ + const simde__m128d a1 = simde_mm_set1_pd(SIMDE_FLOAT64_C( 0.254829592)); + const simde__m128d a2 = simde_mm_set1_pd(SIMDE_FLOAT64_C(-0.284496736)); + const simde__m128d a3 = simde_mm_set1_pd(SIMDE_FLOAT64_C(1.421413741)); + const simde__m128d a4 = simde_mm_set1_pd(SIMDE_FLOAT64_C(-1.453152027)); + const simde__m128d a5 = simde_mm_set1_pd(SIMDE_FLOAT64_C(1.061405429)); + const simde__m128d p = simde_mm_set1_pd(SIMDE_FLOAT64_C(0.6475911)); + const simde__m128d one = simde_mm_set1_pd(SIMDE_FLOAT64_C(1.0)); + + /* simde_math_fabs(x) / sqrt(2.0) */ + const simde__m128d x = simde_mm_div_pd(simde_x_mm_abs_pd(a), simde_mm_sqrt_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(2.0)))); + + /* 1.0 / (1.0 + p * x) */ + const simde__m128d t = simde_mm_div_pd(one, simde_mm_add_pd(one, simde_mm_mul_pd(p, x))); + + /* 1.0 - (((((a5 * t + a4) * t) + a3) * t + a2) * t + a1) * t * exp(-x * x) */ + simde__m128d y = simde_mm_mul_pd(a5, t); + y = simde_mm_add_pd(y, a4); + y = simde_mm_mul_pd(y, t); + y = simde_mm_add_pd(y, a3); + y = simde_mm_mul_pd(y, t); + y = simde_mm_add_pd(y, a2); + y = simde_mm_mul_pd(y, t); + y = simde_mm_add_pd(y, a1); + y = simde_mm_mul_pd(y, t); + y = simde_mm_mul_pd(y, simde_mm_exp_pd(simde_mm_mul_pd(x, simde_x_mm_negate_pd(x)))); + y = simde_mm_sub_pd(one, y); + + /* 0.5 * (1.0 + ((a < 0.0) ? -y : y)) */ + return simde_mm_mul_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(0.5)), simde_mm_add_pd(one, simde_x_mm_xorsign_pd(y, a))); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cdfnorm(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_cdfnorm_pd + #define _mm_cdfnorm_pd(a) simde_mm_cdfnorm_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_cdfnorm_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cdfnorm_ps(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + /* https://www.johndcook.com/blog/cpp_phi/ */ + const simde__m256 a1 = simde_mm256_set1_ps(SIMDE_FLOAT32_C( 0.254829592)); + const simde__m256 a2 = simde_mm256_set1_ps(SIMDE_FLOAT32_C(-0.284496736)); + const simde__m256 a3 = simde_mm256_set1_ps(SIMDE_FLOAT32_C(1.421413741)); + const simde__m256 a4 = simde_mm256_set1_ps(SIMDE_FLOAT32_C(-1.453152027)); + const simde__m256 a5 = simde_mm256_set1_ps(SIMDE_FLOAT32_C(1.061405429)); + const simde__m256 p = simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.3275911)); + const simde__m256 one = simde_mm256_set1_ps(SIMDE_FLOAT32_C(1.0)); + + /* simde_math_fabsf(x) / sqrtf(2.0) */ + const simde__m256 x = simde_mm256_div_ps(simde_x_mm256_abs_ps(a), simde_mm256_sqrt_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(2.0)))); + + /* 1.0 / (1.0 + p * x) */ + const simde__m256 t = simde_mm256_div_ps(one, simde_mm256_add_ps(one, simde_mm256_mul_ps(p, x))); + + /* 1.0 - (((((a5 * t + a4) * t) + a3) * t + a2) * t + a1) * t * exp(-x * x) */ + simde__m256 y = simde_mm256_mul_ps(a5, t); + y = simde_mm256_add_ps(y, a4); + y = simde_mm256_mul_ps(y, t); + y = simde_mm256_add_ps(y, a3); + y = simde_mm256_mul_ps(y, t); + y = simde_mm256_add_ps(y, a2); + y = simde_mm256_mul_ps(y, t); + y = simde_mm256_add_ps(y, a1); + y = simde_mm256_mul_ps(y, t); + y = simde_mm256_mul_ps(y, simde_mm256_exp_ps(simde_mm256_mul_ps(x, simde_x_mm256_negate_ps(x)))); + y = simde_mm256_sub_ps(one, y); + + /* 0.5 * (1.0 + ((a < 0.0) ? -y : y)) */ + return simde_mm256_mul_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.5)), simde_mm256_add_ps(one, simde_x_mm256_xorsign_ps(y, a))); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_cdfnorm_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_cdfnormf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_cdfnorm_ps + #define _mm256_cdfnorm_ps(a) simde_mm256_cdfnorm_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_cdfnorm_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cdfnorm_pd(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + /* https://www.johndcook.com/blog/cpp_phi/ */ + const simde__m256d a1 = simde_mm256_set1_pd(SIMDE_FLOAT64_C( 0.254829592)); + const simde__m256d a2 = simde_mm256_set1_pd(SIMDE_FLOAT64_C(-0.284496736)); + const simde__m256d a3 = simde_mm256_set1_pd(SIMDE_FLOAT64_C(1.421413741)); + const simde__m256d a4 = simde_mm256_set1_pd(SIMDE_FLOAT64_C(-1.453152027)); + const simde__m256d a5 = simde_mm256_set1_pd(SIMDE_FLOAT64_C(1.061405429)); + const simde__m256d p = simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.6475911)); + const simde__m256d one = simde_mm256_set1_pd(SIMDE_FLOAT64_C(1.0)); + + /* simde_math_fabs(x) / sqrt(2.0) */ + const simde__m256d x = simde_mm256_div_pd(simde_x_mm256_abs_pd(a), simde_mm256_sqrt_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(2.0)))); + + /* 1.0 / (1.0 + p * x) */ + const simde__m256d t = simde_mm256_div_pd(one, simde_mm256_add_pd(one, simde_mm256_mul_pd(p, x))); + + /* 1.0 - (((((a5 * t + a4) * t) + a3) * t + a2) * t + a1) * t * exp(-x * x) */ + simde__m256d y = simde_mm256_mul_pd(a5, t); + y = simde_mm256_add_pd(y, a4); + y = simde_mm256_mul_pd(y, t); + y = simde_mm256_add_pd(y, a3); + y = simde_mm256_mul_pd(y, t); + y = simde_mm256_add_pd(y, a2); + y = simde_mm256_mul_pd(y, t); + y = simde_mm256_add_pd(y, a1); + y = simde_mm256_mul_pd(y, t); + y = simde_mm256_mul_pd(y, simde_mm256_exp_pd(simde_mm256_mul_pd(x, simde_x_mm256_negate_pd(x)))); + y = simde_mm256_sub_pd(one, y); + + /* 0.5 * (1.0 + ((a < 0.0) ? -y : y)) */ + return simde_mm256_mul_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.5)), simde_mm256_add_pd(one, simde_x_mm256_xorsign_pd(y, a))); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_cdfnorm_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cdfnorm(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_cdfnorm_pd + #define _mm256_cdfnorm_pd(a) simde_mm256_cdfnorm_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_cdfnorm_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cdfnorm_ps(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + /* https://www.johndcook.com/blog/cpp_phi/ */ + const simde__m512 a1 = simde_mm512_set1_ps(SIMDE_FLOAT32_C( 0.254829592)); + const simde__m512 a2 = simde_mm512_set1_ps(SIMDE_FLOAT32_C(-0.284496736)); + const simde__m512 a3 = simde_mm512_set1_ps(SIMDE_FLOAT32_C(1.421413741)); + const simde__m512 a4 = simde_mm512_set1_ps(SIMDE_FLOAT32_C(-1.453152027)); + const simde__m512 a5 = simde_mm512_set1_ps(SIMDE_FLOAT32_C(1.061405429)); + const simde__m512 p = simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.3275911)); + const simde__m512 one = simde_mm512_set1_ps(SIMDE_FLOAT32_C(1.0)); + + /* simde_math_fabsf(x) / sqrtf(2.0) */ + const simde__m512 x = simde_mm512_div_ps(simde_mm512_abs_ps(a), simde_mm512_sqrt_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(2.0)))); + + /* 1.0 / (1.0 + p * x) */ + const simde__m512 t = simde_mm512_div_ps(one, simde_mm512_add_ps(one, simde_mm512_mul_ps(p, x))); + + /* 1.0 - (((((a5 * t + a4) * t) + a3) * t + a2) * t + a1) * t * exp(-x * x) */ + simde__m512 y = simde_mm512_mul_ps(a5, t); + y = simde_mm512_add_ps(y, a4); + y = simde_mm512_mul_ps(y, t); + y = simde_mm512_add_ps(y, a3); + y = simde_mm512_mul_ps(y, t); + y = simde_mm512_add_ps(y, a2); + y = simde_mm512_mul_ps(y, t); + y = simde_mm512_add_ps(y, a1); + y = simde_mm512_mul_ps(y, t); + y = simde_mm512_mul_ps(y, simde_mm512_exp_ps(simde_mm512_mul_ps(x, simde_x_mm512_negate_ps(x)))); + y = simde_mm512_sub_ps(one, y); + + /* 0.5 * (1.0 + ((a < 0.0) ? -y : y)) */ + return simde_mm512_mul_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.5)), simde_mm512_add_ps(one, simde_x_mm512_xorsign_ps(y, a))); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_cdfnorm_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_cdfnormf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_cdfnorm_ps + #define _mm512_cdfnorm_ps(a) simde_mm512_cdfnorm_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_cdfnorm_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cdfnorm_pd(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + /* https://www.johndcook.com/blog/cpp_phi/ */ + const simde__m512d a1 = simde_mm512_set1_pd(SIMDE_FLOAT64_C( 0.254829592)); + const simde__m512d a2 = simde_mm512_set1_pd(SIMDE_FLOAT64_C(-0.284496736)); + const simde__m512d a3 = simde_mm512_set1_pd(SIMDE_FLOAT64_C(1.421413741)); + const simde__m512d a4 = simde_mm512_set1_pd(SIMDE_FLOAT64_C(-1.453152027)); + const simde__m512d a5 = simde_mm512_set1_pd(SIMDE_FLOAT64_C(1.061405429)); + const simde__m512d p = simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.6475911)); + const simde__m512d one = simde_mm512_set1_pd(SIMDE_FLOAT64_C(1.0)); + + /* simde_math_fabs(x) / sqrt(2.0) */ + const simde__m512d x = simde_mm512_div_pd(simde_mm512_abs_pd(a), simde_mm512_sqrt_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(2.0)))); + + /* 1.0 / (1.0 + p * x) */ + const simde__m512d t = simde_mm512_div_pd(one, simde_mm512_add_pd(one, simde_mm512_mul_pd(p, x))); + + /* 1.0 - (((((a5 * t + a4) * t) + a3) * t + a2) * t + a1) * t * exp(-x * x) */ + simde__m512d y = simde_mm512_mul_pd(a5, t); + y = simde_mm512_add_pd(y, a4); + y = simde_mm512_mul_pd(y, t); + y = simde_mm512_add_pd(y, a3); + y = simde_mm512_mul_pd(y, t); + y = simde_mm512_add_pd(y, a2); + y = simde_mm512_mul_pd(y, t); + y = simde_mm512_add_pd(y, a1); + y = simde_mm512_mul_pd(y, t); + y = simde_mm512_mul_pd(y, simde_mm512_exp_pd(simde_mm512_mul_pd(x, simde_x_mm512_negate_pd(x)))); + y = simde_mm512_sub_pd(one, y); + + /* 0.5 * (1.0 + ((a < 0.0) ? -y : y)) */ + return simde_mm512_mul_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.5)), simde_mm512_add_pd(one, simde_x_mm512_xorsign_pd(y, a))); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_cdfnorm_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cdfnorm(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_cdfnorm_pd + #define _mm512_cdfnorm_pd(a) simde_mm512_cdfnorm_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_cdfnorm_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cdfnorm_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_cdfnorm_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cdfnorm_ps + #define _mm512_mask_cdfnorm_ps(src, k, a) simde_mm512_mask_cdfnorm_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_cdfnorm_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cdfnorm_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_cdfnorm_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cdfnorm_pd + #define _mm512_mask_cdfnorm_pd(src, k, a) simde_mm512_mask_cdfnorm_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_idivrem_epi32 (simde__m128i* mem_addr, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_idivrem_epi32(HEDLEY_REINTERPRET_CAST(__m128i*, mem_addr), a, b); + #else + simde__m128i r; + + r = simde_mm_div_epi32(a, b); + *mem_addr = simde_mm_sub_epi32(a, simde_mm_mullo_epi32(r, b)); + + return r; + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_idivrem_epi32 + #define _mm_idivrem_epi32(mem_addr, a, b) simde_mm_idivrem_epi32((mem_addr),(a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_idivrem_epi32 (simde__m256i* mem_addr, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_idivrem_epi32(HEDLEY_REINTERPRET_CAST(__m256i*, mem_addr), a, b); + #else + simde__m256i r; + + r = simde_mm256_div_epi32(a, b); + *mem_addr = simde_mm256_sub_epi32(a, simde_mm256_mullo_epi32(r, b)); + + return r; + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_idivrem_epi32 + #define _mm256_idivrem_epi32(mem_addr, a, b) simde_mm256_idivrem_epi32((mem_addr),(a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_hypot_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_hypot_ps(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_hypotf4_u05(a, b); + #else + return Sleef_hypotf4_u35(a, b); + #endif + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_hypotf(a_.f32[i], b_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_hypot_ps + #define _mm_hypot_ps(a, b) simde_mm_hypot_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_hypot_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_hypot_pd(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_hypotd2_u05(a, b); + #else + return Sleef_hypotd2_u35(a, b); + #endif + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_hypot(a_.f64[i], b_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_hypot_pd + #define _mm_hypot_pd(a, b) simde_mm_hypot_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_hypot_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_hypot_ps(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_hypotf8_u05(a, b); + #else + return Sleef_hypotf8_u35(a, b); + #endif + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_hypot_ps(a_.m128[i], b_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_hypotf(a_.f32[i], b_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_hypot_ps + #define _mm256_hypot_ps(a, b) simde_mm256_hypot_ps(a, b) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_hypot_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_hypot_pd(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_hypotd4_u05(a, b); + #else + return Sleef_hypotd4_u35(a, b); + #endif + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_hypot_pd(a_.m128d[i], b_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_hypot(a_.f64[i], b_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_hypot_pd + #define _mm256_hypot_pd(a, b) simde_mm256_hypot_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_hypot_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_hypot_ps(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_hypotf16_u05(a, b); + #else + return Sleef_hypotf16_u35(a, b); + #endif + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_hypot_ps(a_.m256[i], b_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_hypotf(a_.f32[i], b_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_hypot_ps + #define _mm512_hypot_ps(a, b) simde_mm512_hypot_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_hypot_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_hypot_pd(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_hypotd8_u05(a, b); + #else + return Sleef_hypotd8_u35(a, b); + #endif + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_hypot_pd(a_.m256d[i], b_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_hypot(a_.f64[i], b_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_hypot_pd + #define _mm512_hypot_pd(a, b) simde_mm512_hypot_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_hypot_ps(simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_hypot_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_hypot_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_hypot_ps + #define _mm512_mask_hypot_ps(src, k, a, b) simde_mm512_mask_hypot_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_hypot_pd(simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_hypot_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_hypot_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_hypot_pd + #define _mm512_mask_hypot_pd(src, k, a, b) simde_mm512_mask_hypot_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_invcbrt_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_invcbrt_ps(a); + #else + return simde_mm_rcp_ps(simde_mm_cbrt_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_invcbrt_ps + #define _mm_invcbrt_ps(a) simde_mm_invcbrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_invcbrt_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_invcbrt_pd(a); + #else + return simde_mm_div_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(1.0)), simde_mm_cbrt_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_invcbrt_pd + #define _mm_invcbrt_pd(a) simde_mm_invcbrt_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_invcbrt_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_invcbrt_ps(a); + #else + return simde_mm256_rcp_ps(simde_mm256_cbrt_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_invcbrt_ps + #define _mm256_invcbrt_ps(a) simde_mm256_invcbrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_invcbrt_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_invcbrt_pd(a); + #else + return simde_mm256_div_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(1.0)), simde_mm256_cbrt_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_invcbrt_pd + #define _mm256_invcbrt_pd(a) simde_mm256_invcbrt_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_invsqrt_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_invsqrt_ps(a); + #else + return simde_mm_rcp_ps(simde_mm_sqrt_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_invsqrt_ps + #define _mm_invsqrt_ps(a) simde_mm_invsqrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_invsqrt_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_invsqrt_pd(a); + #else + return simde_mm_div_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(1.0)), simde_mm_sqrt_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_invsqrt_pd + #define _mm_invsqrt_pd(a) simde_mm_invsqrt_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_invsqrt_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_invsqrt_ps(a); + #else + return simde_mm256_rcp_ps(simde_mm256_sqrt_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_invsqrt_ps + #define _mm256_invsqrt_ps(a) simde_mm256_invsqrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_invsqrt_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_invsqrt_pd(a); + #else + return simde_mm256_div_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(1.0)), simde_mm256_sqrt_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_invsqrt_pd + #define _mm256_invsqrt_pd(a) simde_mm256_invsqrt_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_invsqrt_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_invsqrt_ps(a); + #else + return simde_mm512_div_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(1.0)), simde_mm512_sqrt_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_invsqrt_ps + #define _mm512_invsqrt_ps(a) simde_mm512_invsqrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_invsqrt_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_invsqrt_pd(a); + #else + return simde_mm512_div_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(1.0)), simde_mm512_sqrt_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_invsqrt_pd + #define _mm512_invsqrt_pd(a) simde_mm512_invsqrt_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_invsqrt_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_invsqrt_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_invsqrt_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_invsqrt_ps + #define _mm512_mask_invsqrt_ps(src, k, a) simde_mm512_mask_invsqrt_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_invsqrt_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_invsqrt_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_invsqrt_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_invsqrt_pd + #define _mm512_mask_invsqrt_pd(src, k, a) simde_mm512_mask_invsqrt_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_log_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_log_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_logf4_u10(a); + #else + return Sleef_logf4_u35(a); + #endif + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_logf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_log_ps + #define _mm_log_ps(a) simde_mm_log_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_log_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_log_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_logd2_u10(a); + #else + return Sleef_logd2_u35(a); + #endif + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_log(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_log_pd + #define _mm_log_pd(a) simde_mm_log_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_log_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_log_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_logf8_u10(a); + #else + return Sleef_logf8_u35(a); + #endif + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_log_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_logf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_log_ps + #define _mm256_log_ps(a) simde_mm256_log_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_log_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_log_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_logd4_u10(a); + #else + return Sleef_logd4_u35(a); + #endif + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_log_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_log(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_log_pd + #define _mm256_log_pd(a) simde_mm256_log_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_log_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_log_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_logf16_u10(a); + #else + return Sleef_logf16_u35(a); + #endif + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_log_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_logf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_log_ps + #define _mm512_log_ps(a) simde_mm512_log_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_log_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_log_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_logd8_u10(a); + #else + return Sleef_logd8_u35(a); + #endif + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_log_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_log(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_log_pd + #define _mm512_log_pd(a) simde_mm512_log_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_log_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_log_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_log_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_log_ps + #define _mm512_mask_log_ps(src, k, a) simde_mm512_mask_log_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_log_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_log_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_log_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_log_pd + #define _mm512_mask_log_pd(src, k, a) simde_mm512_mask_log_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_cdfnorminv_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_cdfnorminv_ps(a); + #elif SIMDE_NATURAL_VECTOR_SIZE_GE(128) + simde__m128 matched, retval = simde_mm_setzero_ps(); + + { /* if (a < 0 || a > 1) */ + matched = simde_mm_or_ps(simde_mm_cmplt_ps(a, simde_mm_set1_ps(SIMDE_FLOAT32_C(0.0))), simde_mm_cmpgt_ps(a, simde_mm_set1_ps(SIMDE_FLOAT32_C(1.0)))); + + /* We don't actually need to do anything here since we initialize + * retval to 0.0. */ + } + + { /* else if (a == 0) */ + simde__m128 mask = simde_mm_cmpeq_ps(a, simde_mm_set1_ps(SIMDE_FLOAT32_C(0.0))); + mask = simde_mm_andnot_ps(matched, mask); + matched = simde_mm_or_ps(matched, mask); + + simde__m128 res = simde_mm_set1_ps(-SIMDE_MATH_INFINITYF); + + retval = simde_mm_or_ps(retval, simde_mm_and_ps(mask, res)); + } + + { /* else if (a == 1) */ + simde__m128 mask = simde_mm_cmpeq_ps(a, simde_mm_set1_ps(SIMDE_FLOAT32_C(1.0))); + mask = simde_mm_andnot_ps(matched, mask); + matched = simde_mm_or_ps(matched, mask); + + simde__m128 res = simde_mm_set1_ps(SIMDE_MATH_INFINITYF); + + retval = simde_mm_or_ps(retval, simde_mm_and_ps(mask, res)); + } + + { /* Remaining conditions. + * + * Including the else case in this complicates things a lot, but + * we're using cheap operations to get rid of expensive multiply + * and add functions. This should be a small improvement on SSE + * prior to 4.1. On SSE 4.1 we can use _mm_blendv_ps which is + * very fast and this becomes a huge win. NEON, AltiVec, and + * WASM also have blend operations, so this should be a big win + * there, too. */ + + /* else if (a < 0.02425) */ + simde__m128 mask_lo = simde_mm_cmplt_ps(a, simde_mm_set1_ps(SIMDE_FLOAT32_C(0.02425))); + /* else if (a > 0.97575) */ + simde__m128 mask_hi = simde_mm_cmpgt_ps(a, simde_mm_set1_ps(SIMDE_FLOAT32_C(0.97575))); + + simde__m128 mask = simde_mm_or_ps(mask_lo, mask_hi); + matched = simde_mm_or_ps(matched, mask); + + /* else */ + simde__m128 mask_el = simde_x_mm_not_ps(matched); + mask = simde_mm_or_ps(mask, mask_el); + + /* r = a - 0.5f */ + simde__m128 r = simde_mm_sub_ps(a, simde_mm_set1_ps(SIMDE_FLOAT32_C(0.5))); + + /* lo: q = a + * hi: q = (1.0 - a) */ + simde__m128 q = simde_mm_and_ps(mask_lo, a); + q = simde_mm_or_ps(q, simde_mm_and_ps(mask_hi, simde_mm_sub_ps(simde_mm_set1_ps(1.0f), a))); + + /* q = simde_math_sqrtf(-2.0f * simde_math_logf(q)) */ + q = simde_mm_log_ps(q); + q = simde_mm_mul_ps(q, simde_mm_set1_ps(SIMDE_FLOAT32_C(-2.0))); + q = simde_mm_sqrt_ps(q); + + /* el: q = r * r */ + q = simde_x_mm_select_ps(q, simde_mm_mul_ps(r, r), mask_el); + + /* lo: float numerator = ((((((c_c[0] * q + c_c[1]) * q + c_c[2]) * q + c_c[3]) * q + c_c[4]) * q + c_c[5]) * 1.0f); */ + /* hi: float numerator = ((((((c_c[0] * q + c_c[1]) * q + c_c[2]) * q + c_c[3]) * q + c_c[4]) * q + c_c[5]) * -1.0f); */ + /* el: float numerator = ((((((c_a[0] * q + c_a[1]) * q + c_a[2]) * q + c_a[3]) * q + c_a[4]) * q + c_a[5]) * r); */ + simde__m128 numerator = simde_x_mm_select_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C(-7.784894002430293e-03)), simde_mm_set1_ps(SIMDE_FLOAT32_C(-3.969683028665376e+01)), mask_el); + numerator = simde_mm_fmadd_ps(numerator, q, simde_x_mm_select_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C(-3.223964580411365e-01)), simde_mm_set1_ps(SIMDE_FLOAT32_C( 2.209460984245205e+02)), mask_el)); + numerator = simde_mm_fmadd_ps(numerator, q, simde_x_mm_select_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C(-2.400758277161838e+00)), simde_mm_set1_ps(SIMDE_FLOAT32_C(-2.759285104469687e+02)), mask_el)); + numerator = simde_mm_fmadd_ps(numerator, q, simde_x_mm_select_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C(-2.549732539343734e+00)), simde_mm_set1_ps(SIMDE_FLOAT32_C( 1.383577518672690e+02)), mask_el)); + numerator = simde_mm_fmadd_ps(numerator, q, simde_x_mm_select_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C( 4.374664141464968e+00)), simde_mm_set1_ps(SIMDE_FLOAT32_C(-3.066479806614716e+01)), mask_el)); + numerator = simde_mm_fmadd_ps(numerator, q, simde_x_mm_select_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C( 2.938163982698783e+00)), simde_mm_set1_ps(SIMDE_FLOAT32_C( 2.506628277459239e+00)), mask_el)); + { + simde__m128 multiplier; + multiplier = simde_mm_and_ps(mask_lo, simde_mm_set1_ps(SIMDE_FLOAT32_C( 1.0))); + multiplier = simde_mm_or_ps(multiplier, simde_mm_and_ps(mask_hi, simde_mm_set1_ps(SIMDE_FLOAT32_C(-1.0)))); + multiplier = simde_mm_or_ps(multiplier, simde_mm_and_ps(mask_el, r)); + numerator = simde_mm_mul_ps(numerator, multiplier); + } + + /* lo/hi: float denominator = (((((c_d[0] * q + c_d[1]) * q + c_d[2]) * q + c_d[3]) * 1 + 0.0f) * q + 1); */ + /* el: float denominator = (((((c_b[0] * q + c_b[1]) * q + c_b[2]) * q + c_b[3]) * q + c_b[4]) * q + 1); */ + simde__m128 denominator = simde_x_mm_select_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C( 7.784695709041462e-03)), simde_mm_set1_ps(SIMDE_FLOAT32_C(-5.447609879822406e+01)), mask_el); + denominator = simde_mm_fmadd_ps(denominator, q, simde_x_mm_select_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C( 3.224671290700398e-01)), simde_mm_set1_ps(SIMDE_FLOAT32_C( 1.615858368580409e+02)), mask_el)); + denominator = simde_mm_fmadd_ps(denominator, q, simde_x_mm_select_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C( 2.445134137142996e+00)), simde_mm_set1_ps(SIMDE_FLOAT32_C(-1.556989798598866e+02)), mask_el)); + denominator = simde_mm_fmadd_ps(denominator, q, simde_x_mm_select_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C( 3.754408661907416e+00)), simde_mm_set1_ps(SIMDE_FLOAT32_C( 6.680131188771972e+01)), mask_el)); + denominator = simde_mm_fmadd_ps(denominator, simde_x_mm_select_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C( 1.0)), q, mask_el), + simde_x_mm_select_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C( 0.0)), simde_mm_set1_ps(SIMDE_FLOAT32_C(-1.328068155288572e+01)), mask_el)); + denominator = simde_mm_fmadd_ps(denominator, q, simde_mm_set1_ps(SIMDE_FLOAT32_C(1.0))); + + /* res = numerator / denominator; */ + simde__m128 res = simde_mm_div_ps(numerator, denominator); + + retval = simde_mm_or_ps(retval, simde_mm_and_ps(mask, res)); + } + + return retval; + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_cdfnorminvf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_cdfnorminv_ps + #define _mm_cdfnorminv_ps(a) simde_mm_cdfnorminv_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_cdfnorminv_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_cdfnorminv_pd(a); + #elif SIMDE_NATURAL_VECTOR_SIZE_GE(128) + simde__m128d matched, retval = simde_mm_setzero_pd(); + + { /* if (a < 0 || a > 1) */ + matched = simde_mm_or_pd(simde_mm_cmplt_pd(a, simde_mm_set1_pd(SIMDE_FLOAT64_C(0.0))), simde_mm_cmpgt_pd(a, simde_mm_set1_pd(SIMDE_FLOAT64_C(1.0)))); + + /* We don't actually need to do anything here since we initialize + * retval to 0.0. */ + } + + { /* else if (a == 0) */ + simde__m128d mask = simde_mm_cmpeq_pd(a, simde_mm_set1_pd(SIMDE_FLOAT64_C(0.0))); + mask = simde_mm_andnot_pd(matched, mask); + matched = simde_mm_or_pd(matched, mask); + + simde__m128d res = simde_mm_set1_pd(-SIMDE_MATH_INFINITY); + + retval = simde_mm_or_pd(retval, simde_mm_and_pd(mask, res)); + } + + { /* else if (a == 1) */ + simde__m128d mask = simde_mm_cmpeq_pd(a, simde_mm_set1_pd(SIMDE_FLOAT64_C(1.0))); + mask = simde_mm_andnot_pd(matched, mask); + matched = simde_mm_or_pd(matched, mask); + + simde__m128d res = simde_mm_set1_pd(SIMDE_MATH_INFINITY); + + retval = simde_mm_or_pd(retval, simde_mm_and_pd(mask, res)); + } + + { /* Remaining conditions. + * + * Including the else case in this complicates things a lot, but + * we're using cheap operations to get rid of expensive multiply + * and add functions. This should be a small improvement on SSE + * prior to 4.1. On SSE 4.1 we can use _mm_blendv_pd which is + * very fast and this becomes a huge win. NEON, AltiVec, and + * WASM also have blend operations, so this should be a big win + * there, too. */ + + /* else if (a < 0.02425) */ + simde__m128d mask_lo = simde_mm_cmplt_pd(a, simde_mm_set1_pd(SIMDE_FLOAT64_C(0.02425))); + /* else if (a > 0.97575) */ + simde__m128d mask_hi = simde_mm_cmpgt_pd(a, simde_mm_set1_pd(SIMDE_FLOAT64_C(0.97575))); + + simde__m128d mask = simde_mm_or_pd(mask_lo, mask_hi); + matched = simde_mm_or_pd(matched, mask); + + /* else */ + simde__m128d mask_el = simde_x_mm_not_pd(matched); + mask = simde_mm_or_pd(mask, mask_el); + + /* r = a - 0.5 */ + simde__m128d r = simde_mm_sub_pd(a, simde_mm_set1_pd(SIMDE_FLOAT64_C(0.5))); + + /* lo: q = a + * hi: q = (1.0 - a) */ + simde__m128d q = simde_mm_and_pd(mask_lo, a); + q = simde_mm_or_pd(q, simde_mm_and_pd(mask_hi, simde_mm_sub_pd(simde_mm_set1_pd(1.0), a))); + + /* q = simde_math_sqrt(-2.0 * simde_math_log(q)) */ + q = simde_mm_log_pd(q); + q = simde_mm_mul_pd(q, simde_mm_set1_pd(SIMDE_FLOAT64_C(-2.0))); + q = simde_mm_sqrt_pd(q); + + /* el: q = r * r */ + q = simde_x_mm_select_pd(q, simde_mm_mul_pd(r, r), mask_el); + + /* lo: double numerator = ((((((c_c[0] * q + c_c[1]) * q + c_c[2]) * q + c_c[3]) * q + c_c[4]) * q + c_c[5]) * 1.0); */ + /* hi: double numerator = ((((((c_c[0] * q + c_c[1]) * q + c_c[2]) * q + c_c[3]) * q + c_c[4]) * q + c_c[5]) * -1.0); */ + /* el: double numerator = ((((((c_a[0] * q + c_a[1]) * q + c_a[2]) * q + c_a[3]) * q + c_a[4]) * q + c_a[5]) * r); */ + simde__m128d numerator = simde_x_mm_select_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(-7.784894002430293e-03)), simde_mm_set1_pd(SIMDE_FLOAT64_C(-3.969683028665376e+01)), mask_el); + numerator = simde_mm_fmadd_pd(numerator, q, simde_x_mm_select_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(-3.223964580411365e-01)), simde_mm_set1_pd(SIMDE_FLOAT64_C( 2.209460984245205e+02)), mask_el)); + numerator = simde_mm_fmadd_pd(numerator, q, simde_x_mm_select_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(-2.400758277161838e+00)), simde_mm_set1_pd(SIMDE_FLOAT64_C(-2.759285104469687e+02)), mask_el)); + numerator = simde_mm_fmadd_pd(numerator, q, simde_x_mm_select_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(-2.549732539343734e+00)), simde_mm_set1_pd(SIMDE_FLOAT64_C( 1.383577518672690e+02)), mask_el)); + numerator = simde_mm_fmadd_pd(numerator, q, simde_x_mm_select_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C( 4.374664141464968e+00)), simde_mm_set1_pd(SIMDE_FLOAT64_C(-3.066479806614716e+01)), mask_el)); + numerator = simde_mm_fmadd_pd(numerator, q, simde_x_mm_select_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C( 2.938163982698783e+00)), simde_mm_set1_pd(SIMDE_FLOAT64_C( 2.506628277459239e+00)), mask_el)); + { + simde__m128d multiplier; + multiplier = simde_mm_and_pd(mask_lo, simde_mm_set1_pd(SIMDE_FLOAT64_C( 1.0))); + multiplier = simde_mm_or_pd(multiplier, simde_mm_and_pd(mask_hi, simde_mm_set1_pd(SIMDE_FLOAT64_C(-1.0)))); + multiplier = simde_mm_or_pd(multiplier, simde_mm_and_pd(mask_el, r)); + numerator = simde_mm_mul_pd(numerator, multiplier); + } + + /* lo/hi: double denominator = (((((c_d[0] * q + c_d[1]) * q + c_d[2]) * q + c_d[3]) * 1 + 0.0f) * q + 1); */ + /* el: double denominator = (((((c_b[0] * q + c_b[1]) * q + c_b[2]) * q + c_b[3]) * q + c_b[4]) * q + 1); */ + simde__m128d denominator = simde_x_mm_select_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C( 7.784695709041462e-03)), simde_mm_set1_pd(SIMDE_FLOAT64_C(-5.447609879822406e+01)), mask_el); + denominator = simde_mm_fmadd_pd(denominator, q, simde_x_mm_select_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C( 3.224671290700398e-01)), simde_mm_set1_pd(SIMDE_FLOAT64_C( 1.615858368580409e+02)), mask_el)); + denominator = simde_mm_fmadd_pd(denominator, q, simde_x_mm_select_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C( 2.445134137142996e+00)), simde_mm_set1_pd(SIMDE_FLOAT64_C(-1.556989798598866e+02)), mask_el)); + denominator = simde_mm_fmadd_pd(denominator, q, simde_x_mm_select_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C( 3.754408661907416e+00)), simde_mm_set1_pd(SIMDE_FLOAT64_C( 6.680131188771972e+01)), mask_el)); + denominator = simde_mm_fmadd_pd(denominator, simde_x_mm_select_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C( 1.0)), q, mask_el), + simde_x_mm_select_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C( 0.0)), simde_mm_set1_pd(SIMDE_FLOAT64_C(-1.328068155288572e+01)), mask_el)); + denominator = simde_mm_fmadd_pd(denominator, q, simde_mm_set1_pd(SIMDE_FLOAT64_C(1.0))); + + /* res = numerator / denominator; */ + simde__m128d res = simde_mm_div_pd(numerator, denominator); + + retval = simde_mm_or_pd(retval, simde_mm_and_pd(mask, res)); + } + + return retval; + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cdfnorminv(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_cdfnorminv_pd + #define _mm_cdfnorminv_pd(a) simde_mm_cdfnorminv_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_cdfnorminv_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cdfnorminv_ps(a); + #elif SIMDE_NATURAL_VECTOR_SIZE_GE(256) + simde__m256 matched, retval = simde_mm256_setzero_ps(); + + { /* if (a < 0 || a > 1) */ + matched = simde_mm256_or_ps(simde_mm256_cmp_ps(a, simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.0)), SIMDE_CMP_LT_OQ), simde_mm256_cmp_ps(a, simde_mm256_set1_ps(SIMDE_FLOAT32_C(1.0)), SIMDE_CMP_GT_OQ)); + + /* We don't actually need to do anything here since we initialize + * retval to 0.0. */ + } + + { /* else if (a == 0) */ + simde__m256 mask = simde_mm256_cmp_ps(a, simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.0)), SIMDE_CMP_EQ_OQ); + mask = simde_mm256_andnot_ps(matched, mask); + matched = simde_mm256_or_ps(matched, mask); + + simde__m256 res = simde_mm256_set1_ps(-SIMDE_MATH_INFINITYF); + + retval = simde_mm256_or_ps(retval, simde_mm256_and_ps(mask, res)); + } + + { /* else if (a == 1) */ + simde__m256 mask = simde_mm256_cmp_ps(a, simde_mm256_set1_ps(SIMDE_FLOAT32_C(1.0)), SIMDE_CMP_EQ_OQ); + mask = simde_mm256_andnot_ps(matched, mask); + matched = simde_mm256_or_ps(matched, mask); + + simde__m256 res = simde_mm256_set1_ps(SIMDE_MATH_INFINITYF); + + retval = simde_mm256_or_ps(retval, simde_mm256_and_ps(mask, res)); + } + + { /* Remaining conditions. + * + * Including the else case in this complicates things a lot, but + * we're using cheap operations to get rid of expensive multiply + * and add functions. This should be a small improvement on SSE + * prior to 4.1. On SSE 4.1 we can use _mm256_blendv_ps which is + * very fast and this becomes a huge win. NEON, AltiVec, and + * WASM also have blend operations, so this should be a big win + * there, too. */ + + /* else if (a < 0.02425) */ + simde__m256 mask_lo = simde_mm256_cmp_ps(a, simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.02425)), SIMDE_CMP_LT_OQ); + /* else if (a > 0.97575) */ + simde__m256 mask_hi = simde_mm256_cmp_ps(a, simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.97575)), SIMDE_CMP_GT_OQ); + + simde__m256 mask = simde_mm256_or_ps(mask_lo, mask_hi); + matched = simde_mm256_or_ps(matched, mask); + + /* else */ + simde__m256 mask_el = simde_x_mm256_not_ps(matched); + mask = simde_mm256_or_ps(mask, mask_el); + + /* r = a - 0.5f */ + simde__m256 r = simde_mm256_sub_ps(a, simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.5))); + + /* lo: q = a + * hi: q = (1.0 - a) */ + simde__m256 q = simde_mm256_and_ps(mask_lo, a); + q = simde_mm256_or_ps(q, simde_mm256_and_ps(mask_hi, simde_mm256_sub_ps(simde_mm256_set1_ps(1.0f), a))); + + /* q = simde_math_sqrtf(-2.0f * simde_math_logf(q)) */ + q = simde_mm256_log_ps(q); + q = simde_mm256_mul_ps(q, simde_mm256_set1_ps(SIMDE_FLOAT32_C(-2.0))); + q = simde_mm256_sqrt_ps(q); + + /* el: q = r * r */ + q = simde_x_mm256_select_ps(q, simde_mm256_mul_ps(r, r), mask_el); + + /* lo: float numerator = ((((((c_c[0] * q + c_c[1]) * q + c_c[2]) * q + c_c[3]) * q + c_c[4]) * q + c_c[5]) * 1.0f); */ + /* hi: float numerator = ((((((c_c[0] * q + c_c[1]) * q + c_c[2]) * q + c_c[3]) * q + c_c[4]) * q + c_c[5]) * -1.0f); */ + /* el: float numerator = ((((((c_a[0] * q + c_a[1]) * q + c_a[2]) * q + c_a[3]) * q + c_a[4]) * q + c_a[5]) * r); */ + simde__m256 numerator = simde_x_mm256_select_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(-7.784894002430293e-03)), simde_mm256_set1_ps(SIMDE_FLOAT32_C(-3.969683028665376e+01)), mask_el); + numerator = simde_mm256_fmadd_ps(numerator, q, simde_x_mm256_select_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(-3.223964580411365e-01)), simde_mm256_set1_ps(SIMDE_FLOAT32_C( 2.209460984245205e+02)), mask_el)); + numerator = simde_mm256_fmadd_ps(numerator, q, simde_x_mm256_select_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(-2.400758277161838e+00)), simde_mm256_set1_ps(SIMDE_FLOAT32_C(-2.759285104469687e+02)), mask_el)); + numerator = simde_mm256_fmadd_ps(numerator, q, simde_x_mm256_select_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(-2.549732539343734e+00)), simde_mm256_set1_ps(SIMDE_FLOAT32_C( 1.383577518672690e+02)), mask_el)); + numerator = simde_mm256_fmadd_ps(numerator, q, simde_x_mm256_select_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C( 4.374664141464968e+00)), simde_mm256_set1_ps(SIMDE_FLOAT32_C(-3.066479806614716e+01)), mask_el)); + numerator = simde_mm256_fmadd_ps(numerator, q, simde_x_mm256_select_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C( 2.938163982698783e+00)), simde_mm256_set1_ps(SIMDE_FLOAT32_C( 2.506628277459239e+00)), mask_el)); + { + simde__m256 multiplier; + multiplier = simde_mm256_and_ps(mask_lo, simde_mm256_set1_ps(SIMDE_FLOAT32_C( 1.0))); + multiplier = simde_mm256_or_ps(multiplier, simde_mm256_and_ps(mask_hi, simde_mm256_set1_ps(SIMDE_FLOAT32_C(-1.0)))); + multiplier = simde_mm256_or_ps(multiplier, simde_mm256_and_ps(mask_el, r)); + numerator = simde_mm256_mul_ps(numerator, multiplier); + } + + /* lo/hi: float denominator = (((((c_d[0] * q + c_d[1]) * q + c_d[2]) * q + c_d[3]) * 1 + 0.0f) * q + 1); */ + /* el: float denominator = (((((c_b[0] * q + c_b[1]) * q + c_b[2]) * q + c_b[3]) * q + c_b[4]) * q + 1); */ + simde__m256 denominator = simde_x_mm256_select_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C( 7.784695709041462e-03)), simde_mm256_set1_ps(SIMDE_FLOAT32_C(-5.447609879822406e+01)), mask_el); + denominator = simde_mm256_fmadd_ps(denominator, q, simde_x_mm256_select_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C( 3.224671290700398e-01)), simde_mm256_set1_ps(SIMDE_FLOAT32_C( 1.615858368580409e+02)), mask_el)); + denominator = simde_mm256_fmadd_ps(denominator, q, simde_x_mm256_select_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C( 2.445134137142996e+00)), simde_mm256_set1_ps(SIMDE_FLOAT32_C(-1.556989798598866e+02)), mask_el)); + denominator = simde_mm256_fmadd_ps(denominator, q, simde_x_mm256_select_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C( 3.754408661907416e+00)), simde_mm256_set1_ps(SIMDE_FLOAT32_C( 6.680131188771972e+01)), mask_el)); + denominator = simde_mm256_fmadd_ps(denominator, simde_x_mm256_select_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C( 1.0)), q, mask_el), + simde_x_mm256_select_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C( 0.0)), simde_mm256_set1_ps(SIMDE_FLOAT32_C(-1.328068155288572e+01)), mask_el)); + denominator = simde_mm256_fmadd_ps(denominator, q, simde_mm256_set1_ps(SIMDE_FLOAT32_C(1.0))); + + /* res = numerator / denominator; */ + simde__m256 res = simde_mm256_div_ps(numerator, denominator); + + retval = simde_mm256_or_ps(retval, simde_mm256_and_ps(mask, res)); + } + + return retval; + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_cdfnorminv_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_cdfnorminvf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_cdfnorminv_ps + #define _mm256_cdfnorminv_ps(a) simde_mm256_cdfnorminv_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_cdfnorminv_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_cdfnorminv_pd(a); + #elif SIMDE_NATURAL_VECTOR_SIZE_GE(256) + simde__m256d matched, retval = simde_mm256_setzero_pd(); + + { /* if (a < 0 || a > 1) */ + matched = simde_mm256_or_pd(simde_mm256_cmp_pd(a, simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.0)), SIMDE_CMP_LT_OQ), simde_mm256_cmp_pd(a, simde_mm256_set1_pd(SIMDE_FLOAT64_C(1.0)), SIMDE_CMP_GT_OQ)); + + /* We don't actually need to do anything here since we initialize + * retval to 0.0. */ + } + + { /* else if (a == 0) */ + simde__m256d mask = simde_mm256_cmp_pd(a, simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.0)), SIMDE_CMP_EQ_OQ); + mask = simde_mm256_andnot_pd(matched, mask); + matched = simde_mm256_or_pd(matched, mask); + + simde__m256d res = simde_mm256_set1_pd(-SIMDE_MATH_INFINITY); + + retval = simde_mm256_or_pd(retval, simde_mm256_and_pd(mask, res)); + } + + { /* else if (a == 1) */ + simde__m256d mask = simde_mm256_cmp_pd(a, simde_mm256_set1_pd(SIMDE_FLOAT64_C(1.0)), SIMDE_CMP_EQ_OQ); + mask = simde_mm256_andnot_pd(matched, mask); + matched = simde_mm256_or_pd(matched, mask); + + simde__m256d res = simde_mm256_set1_pd(SIMDE_MATH_INFINITY); + + retval = simde_mm256_or_pd(retval, simde_mm256_and_pd(mask, res)); + } + + { /* Remaining conditions. + * + * Including the else case in this complicates things a lot, but + * we're using cheap operations to get rid of expensive multiply + * and add functions. This should be a small improvement on SSE + * prior to 4.1. On SSE 4.1 we can use _mm256_blendv_pd which is + * very fast and this becomes a huge win. NEON, AltiVec, and + * WASM also have blend operations, so this should be a big win + * there, too. */ + + /* else if (a < 0.02425) */ + simde__m256d mask_lo = simde_mm256_cmp_pd(a, simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.02425)), SIMDE_CMP_LT_OQ); + /* else if (a > 0.97575) */ + simde__m256d mask_hi = simde_mm256_cmp_pd(a, simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.97575)), SIMDE_CMP_GT_OQ); + + simde__m256d mask = simde_mm256_or_pd(mask_lo, mask_hi); + matched = simde_mm256_or_pd(matched, mask); + + /* else */ + simde__m256d mask_el = simde_x_mm256_not_pd(matched); + mask = simde_mm256_or_pd(mask, mask_el); + + /* r = a - 0.5 */ + simde__m256d r = simde_mm256_sub_pd(a, simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.5))); + + /* lo: q = a + * hi: q = (1.0 - a) */ + simde__m256d q = simde_mm256_and_pd(mask_lo, a); + q = simde_mm256_or_pd(q, simde_mm256_and_pd(mask_hi, simde_mm256_sub_pd(simde_mm256_set1_pd(1.0), a))); + + /* q = simde_math_sqrt(-2.0 * simde_math_log(q)) */ + q = simde_mm256_log_pd(q); + q = simde_mm256_mul_pd(q, simde_mm256_set1_pd(SIMDE_FLOAT64_C(-2.0))); + q = simde_mm256_sqrt_pd(q); + + /* el: q = r * r */ + q = simde_x_mm256_select_pd(q, simde_mm256_mul_pd(r, r), mask_el); + + /* lo: double numerator = ((((((c_c[0] * q + c_c[1]) * q + c_c[2]) * q + c_c[3]) * q + c_c[4]) * q + c_c[5]) * 1.0); */ + /* hi: double numerator = ((((((c_c[0] * q + c_c[1]) * q + c_c[2]) * q + c_c[3]) * q + c_c[4]) * q + c_c[5]) * -1.0); */ + /* el: double numerator = ((((((c_a[0] * q + c_a[1]) * q + c_a[2]) * q + c_a[3]) * q + c_a[4]) * q + c_a[5]) * r); */ + simde__m256d numerator = simde_x_mm256_select_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(-7.784894002430293e-03)), simde_mm256_set1_pd(SIMDE_FLOAT64_C(-3.969683028665376e+01)), mask_el); + numerator = simde_mm256_fmadd_pd(numerator, q, simde_x_mm256_select_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(-3.223964580411365e-01)), simde_mm256_set1_pd(SIMDE_FLOAT64_C( 2.209460984245205e+02)), mask_el)); + numerator = simde_mm256_fmadd_pd(numerator, q, simde_x_mm256_select_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(-2.400758277161838e+00)), simde_mm256_set1_pd(SIMDE_FLOAT64_C(-2.759285104469687e+02)), mask_el)); + numerator = simde_mm256_fmadd_pd(numerator, q, simde_x_mm256_select_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(-2.549732539343734e+00)), simde_mm256_set1_pd(SIMDE_FLOAT64_C( 1.383577518672690e+02)), mask_el)); + numerator = simde_mm256_fmadd_pd(numerator, q, simde_x_mm256_select_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C( 4.374664141464968e+00)), simde_mm256_set1_pd(SIMDE_FLOAT64_C(-3.066479806614716e+01)), mask_el)); + numerator = simde_mm256_fmadd_pd(numerator, q, simde_x_mm256_select_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C( 2.938163982698783e+00)), simde_mm256_set1_pd(SIMDE_FLOAT64_C( 2.506628277459239e+00)), mask_el)); + { + simde__m256d multiplier; + multiplier = simde_mm256_and_pd(mask_lo, simde_mm256_set1_pd(SIMDE_FLOAT64_C( 1.0))); + multiplier = simde_mm256_or_pd(multiplier, simde_mm256_and_pd(mask_hi, simde_mm256_set1_pd(SIMDE_FLOAT64_C(-1.0)))); + multiplier = simde_mm256_or_pd(multiplier, simde_mm256_and_pd(mask_el, r)); + numerator = simde_mm256_mul_pd(numerator, multiplier); + } + + /* lo/hi: double denominator = (((((c_d[0] * q + c_d[1]) * q + c_d[2]) * q + c_d[3]) * 1 + 0.0f) * q + 1); */ + /* el: double denominator = (((((c_b[0] * q + c_b[1]) * q + c_b[2]) * q + c_b[3]) * q + c_b[4]) * q + 1); */ + simde__m256d denominator = simde_x_mm256_select_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C( 7.784695709041462e-03)), simde_mm256_set1_pd(SIMDE_FLOAT64_C(-5.447609879822406e+01)), mask_el); + denominator = simde_mm256_fmadd_pd(denominator, q, simde_x_mm256_select_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C( 3.224671290700398e-01)), simde_mm256_set1_pd(SIMDE_FLOAT64_C( 1.615858368580409e+02)), mask_el)); + denominator = simde_mm256_fmadd_pd(denominator, q, simde_x_mm256_select_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C( 2.445134137142996e+00)), simde_mm256_set1_pd(SIMDE_FLOAT64_C(-1.556989798598866e+02)), mask_el)); + denominator = simde_mm256_fmadd_pd(denominator, q, simde_x_mm256_select_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C( 3.754408661907416e+00)), simde_mm256_set1_pd(SIMDE_FLOAT64_C( 6.680131188771972e+01)), mask_el)); + denominator = simde_mm256_fmadd_pd(denominator, simde_x_mm256_select_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C( 1.0)), q, mask_el), + simde_x_mm256_select_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C( 0.0)), simde_mm256_set1_pd(SIMDE_FLOAT64_C(-1.328068155288572e+01)), mask_el)); + denominator = simde_mm256_fmadd_pd(denominator, q, simde_mm256_set1_pd(SIMDE_FLOAT64_C(1.0))); + + /* res = numerator / denominator; */ + simde__m256d res = simde_mm256_div_pd(numerator, denominator); + + retval = simde_mm256_or_pd(retval, simde_mm256_and_pd(mask, res)); + } + + return retval; + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_cdfnorminv_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_cdfnorminv(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_cdfnorminv_pd + #define _mm256_cdfnorminv_pd(a) simde_mm256_cdfnorminv_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_cdfnorminv_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cdfnorminv_ps(a); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_cdfnorminv_ps(a_.m256[i]); + } + + return simde__m512_from_private(r_); + #else + + simde__m512 retval = simde_mm512_setzero_ps(); + simde__mmask16 matched; + + { /* if (a < 0 || a > 1) */ + matched = simde_mm512_cmp_ps_mask(a, simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.0)), SIMDE_CMP_LT_OQ); + matched |= simde_mm512_cmp_ps_mask(a, simde_mm512_set1_ps(SIMDE_FLOAT32_C(1.0)), SIMDE_CMP_GT_OQ); + + /* We don't actually need to do anything here since we initialize + * retval to 0.0. */ + } + + { /* else if (a == 0) */ + simde__mmask16 mask = simde_mm512_cmp_ps_mask(a, simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.0)), SIMDE_CMP_EQ_OQ); + matched |= mask; + + retval = simde_mm512_mask_mov_ps(retval, mask, simde_mm512_set1_ps(-SIMDE_MATH_INFINITYF)); + } + + { /* else if (a == 1) */ + simde__mmask16 mask = simde_mm512_cmp_ps_mask(a, simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.0)), SIMDE_CMP_EQ_OQ); + matched |= mask; + + retval = simde_mm512_mask_mov_ps(retval, mask, simde_mm512_set1_ps(SIMDE_MATH_INFINITYF)); + } + + { /* else if (a < 0.02425) */ + simde__mmask16 mask_lo = simde_mm512_cmp_ps_mask(a, simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.02425)), SIMDE_CMP_LT_OQ); + /* else if (a > 0.97575) */ + simde__mmask16 mask_hi = simde_mm512_cmp_ps_mask(a, simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.97575)), SIMDE_CMP_GT_OQ); + + simde__mmask16 mask = mask_lo | mask_hi; + matched = matched | mask; + + /* else */ + simde__mmask16 mask_el = ~matched; + + /* r = a - 0.5f */ + simde__m512 r = simde_mm512_sub_ps(a, simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.5))); + + /* lo: q = a + * hi: q = (1.0 - a) */ + simde__m512 q = simde_mm512_maskz_mov_ps(mask_lo, a); + q = simde_mm512_mask_sub_ps(q, mask_hi, simde_mm512_set1_ps(SIMDE_FLOAT32_C(1.0)), a); + + /* q = simde_math_sqrtf(-2.0f * simde_math_logf(q)) */ + q = simde_mm512_log_ps(q); + q = simde_mm512_mul_ps(q, simde_mm512_set1_ps(SIMDE_FLOAT32_C(-2.0))); + q = simde_mm512_sqrt_ps(q); + + /* el: q = r * r */ + q = simde_mm512_mask_mul_ps(q, mask_el, r, r); + + /* lo: float numerator = ((((((c_c[0] * q + c_c[1]) * q + c_c[2]) * q + c_c[3]) * q + c_c[4]) * q + c_c[5]) * 1.0f); */ + /* hi: float numerator = ((((((c_c[0] * q + c_c[1]) * q + c_c[2]) * q + c_c[3]) * q + c_c[4]) * q + c_c[5]) * -1.0f); */ + /* el: float numerator = ((((((c_a[0] * q + c_a[1]) * q + c_a[2]) * q + c_a[3]) * q + c_a[4]) * q + c_a[5]) * r); */ + simde__m512 numerator = simde_mm512_mask_mov_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(-7.784894002430293e-03)), mask_el, simde_mm512_set1_ps(SIMDE_FLOAT32_C(-3.969683028665376e+01))); + numerator = simde_mm512_fmadd_ps(numerator, q, simde_mm512_mask_mov_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(-3.223964580411365e-01)), mask_el, simde_mm512_set1_ps(SIMDE_FLOAT32_C( 2.209460984245205e+02)))); + numerator = simde_mm512_fmadd_ps(numerator, q, simde_mm512_mask_mov_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(-2.400758277161838e+00)), mask_el, simde_mm512_set1_ps(SIMDE_FLOAT32_C(-2.759285104469687e+02)))); + numerator = simde_mm512_fmadd_ps(numerator, q, simde_mm512_mask_mov_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(-2.549732539343734e+00)), mask_el, simde_mm512_set1_ps(SIMDE_FLOAT32_C( 1.383577518672690e+02)))); + numerator = simde_mm512_fmadd_ps(numerator, q, simde_mm512_mask_mov_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C( 4.374664141464968e+00)), mask_el, simde_mm512_set1_ps(SIMDE_FLOAT32_C(-3.066479806614716e+01)))); + numerator = simde_mm512_fmadd_ps(numerator, q, simde_mm512_mask_mov_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C( 2.938163982698783e+00)), mask_el, simde_mm512_set1_ps(SIMDE_FLOAT32_C( 2.506628277459239e+00)))); + { + simde__m512 multiplier; + multiplier = simde_mm512_set1_ps(SIMDE_FLOAT32_C( 1.0)); + multiplier = simde_mm512_mask_mov_ps(multiplier, mask_hi, simde_mm512_set1_ps(SIMDE_FLOAT32_C(-1.0))); + multiplier = simde_mm512_mask_mov_ps(multiplier, mask_el, r); + numerator = simde_mm512_mul_ps(numerator, multiplier); + } + + /* lo/hi: float denominator = (((((c_d[0] * q + c_d[1]) * q + c_d[2]) * q + c_d[3]) * 1 + 0.0f) * q + 1); */ + /* el: float denominator = (((((c_b[0] * q + c_b[1]) * q + c_b[2]) * q + c_b[3]) * q + c_b[4]) * q + 1); */ + simde__m512 denominator = simde_mm512_mask_mov_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C( 7.784695709041462e-03)), mask_el, simde_mm512_set1_ps(SIMDE_FLOAT32_C(-5.447609879822406e+01))); + denominator = simde_mm512_fmadd_ps(denominator, q, simde_mm512_mask_mov_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C( 3.224671290700398e-01)), mask_el, simde_mm512_set1_ps(SIMDE_FLOAT32_C( 1.615858368580409e+02)))); + denominator = simde_mm512_fmadd_ps(denominator, q, simde_mm512_mask_mov_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C( 2.445134137142996e+00)), mask_el, simde_mm512_set1_ps(SIMDE_FLOAT32_C(-1.556989798598866e+02)))); + denominator = simde_mm512_fmadd_ps(denominator, q, simde_mm512_mask_mov_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C( 3.754408661907416e+00)), mask_el, simde_mm512_set1_ps(SIMDE_FLOAT32_C( 6.680131188771972e+01)))); + denominator = simde_mm512_fmadd_ps(denominator, simde_mm512_mask_mov_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C( 1.0)), mask_el, q), + simde_mm512_mask_mov_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C( 0.0)), mask_el, simde_mm512_set1_ps(SIMDE_FLOAT32_C(-1.328068155288572e+01)))); + denominator = simde_mm512_fmadd_ps(denominator, q, simde_mm512_set1_ps(SIMDE_FLOAT32_C(1.0))); + + /* res = numerator / denominator; */ + retval = simde_mm512_mask_div_ps(retval, mask_lo | mask_hi | mask_el, numerator, denominator); + } + + return retval; + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_cdfnorminv_ps + #define _mm512_cdfnorminv_ps(a) simde_mm512_cdfnorminv_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_cdfnorminv_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_cdfnorminv_pd(a); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_cdfnorminv_pd(a_.m256d[i]); + } + + return simde__m512d_from_private(r_); + #else + + simde__m512d retval = simde_mm512_setzero_pd(); + simde__mmask8 matched; + + { /* if (a < 0 || a > 1) */ + matched = simde_mm512_cmp_pd_mask(a, simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.0)), SIMDE_CMP_LT_OQ); + matched |= simde_mm512_cmp_pd_mask(a, simde_mm512_set1_pd(SIMDE_FLOAT64_C(1.0)), SIMDE_CMP_GT_OQ); + + /* We don't actually need to do anything here since we initialize + * retval to 0.0. */ + } + + { /* else if (a == 0) */ + simde__mmask8 mask = simde_mm512_cmp_pd_mask(a, simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.0)), SIMDE_CMP_EQ_OQ); + matched |= mask; + + retval = simde_mm512_mask_mov_pd(retval, mask, simde_mm512_set1_pd(-SIMDE_MATH_INFINITY)); + } + + { /* else if (a == 1) */ + simde__mmask8 mask = simde_mm512_cmp_pd_mask(a, simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.0)), SIMDE_CMP_EQ_OQ); + matched |= mask; + + retval = simde_mm512_mask_mov_pd(retval, mask, simde_mm512_set1_pd(SIMDE_MATH_INFINITY)); + } + + { /* else if (a < 0.02425) */ + simde__mmask8 mask_lo = simde_mm512_cmp_pd_mask(a, simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.02425)), SIMDE_CMP_LT_OQ); + /* else if (a > 0.97575) */ + simde__mmask8 mask_hi = simde_mm512_cmp_pd_mask(a, simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.97575)), SIMDE_CMP_GT_OQ); + + simde__mmask8 mask = mask_lo | mask_hi; + matched = matched | mask; + + /* else */ + simde__mmask8 mask_el = ~matched; + + /* r = a - 0.5f */ + simde__m512d r = simde_mm512_sub_pd(a, simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.5))); + + /* lo: q = a + * hi: q = (1.0 - a) */ + simde__m512d q = a; + q = simde_mm512_mask_sub_pd(q, mask_hi, simde_mm512_set1_pd(SIMDE_FLOAT64_C(1.0)), a); + + /* q = simde_math_sqrtf(-2.0f * simde_math_logf(q)) */ + q = simde_mm512_log_pd(q); + q = simde_mm512_mul_pd(q, simde_mm512_set1_pd(SIMDE_FLOAT64_C(-2.0))); + q = simde_mm512_sqrt_pd(q); + + /* el: q = r * r */ + q = simde_mm512_mask_mul_pd(q, mask_el, r, r); + + /* lo: float numerator = ((((((c_c[0] * q + c_c[1]) * q + c_c[2]) * q + c_c[3]) * q + c_c[4]) * q + c_c[5]) * 1.0f); */ + /* hi: float numerator = ((((((c_c[0] * q + c_c[1]) * q + c_c[2]) * q + c_c[3]) * q + c_c[4]) * q + c_c[5]) * -1.0f); */ + /* el: float numerator = ((((((c_a[0] * q + c_a[1]) * q + c_a[2]) * q + c_a[3]) * q + c_a[4]) * q + c_a[5]) * r); */ + simde__m512d numerator = simde_mm512_mask_mov_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(-7.784894002430293e-03)), mask_el, simde_mm512_set1_pd(SIMDE_FLOAT64_C(-3.969683028665376e+01))); + numerator = simde_mm512_fmadd_pd(numerator, q, simde_mm512_mask_mov_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(-3.223964580411365e-01)), mask_el, simde_mm512_set1_pd(SIMDE_FLOAT64_C( 2.209460984245205e+02)))); + numerator = simde_mm512_fmadd_pd(numerator, q, simde_mm512_mask_mov_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(-2.400758277161838e+00)), mask_el, simde_mm512_set1_pd(SIMDE_FLOAT64_C(-2.759285104469687e+02)))); + numerator = simde_mm512_fmadd_pd(numerator, q, simde_mm512_mask_mov_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(-2.549732539343734e+00)), mask_el, simde_mm512_set1_pd(SIMDE_FLOAT64_C( 1.383577518672690e+02)))); + numerator = simde_mm512_fmadd_pd(numerator, q, simde_mm512_mask_mov_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C( 4.374664141464968e+00)), mask_el, simde_mm512_set1_pd(SIMDE_FLOAT64_C(-3.066479806614716e+01)))); + numerator = simde_mm512_fmadd_pd(numerator, q, simde_mm512_mask_mov_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C( 2.938163982698783e+00)), mask_el, simde_mm512_set1_pd(SIMDE_FLOAT64_C( 2.506628277459239e+00)))); + { + simde__m512d multiplier; + multiplier = simde_mm512_set1_pd(SIMDE_FLOAT64_C( 1.0)); + multiplier = simde_mm512_mask_mov_pd(multiplier, mask_hi, simde_mm512_set1_pd(SIMDE_FLOAT64_C(-1.0))); + multiplier = simde_mm512_mask_mov_pd(multiplier, mask_el, r); + numerator = simde_mm512_mul_pd(numerator, multiplier); + } + + /* lo/hi: float denominator = (((((c_d[0] * q + c_d[1]) * q + c_d[2]) * q + c_d[3]) * 1 + 0.0f) * q + 1); */ + /* el: float denominator = (((((c_b[0] * q + c_b[1]) * q + c_b[2]) * q + c_b[3]) * q + c_b[4]) * q + 1); */ + simde__m512d denominator = simde_mm512_mask_mov_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C( 7.784695709041462e-03)), mask_el, simde_mm512_set1_pd(SIMDE_FLOAT64_C(-5.447609879822406e+01))); + denominator = simde_mm512_fmadd_pd(denominator, q, simde_mm512_mask_mov_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C( 3.224671290700398e-01)), mask_el, simde_mm512_set1_pd(SIMDE_FLOAT64_C( 1.615858368580409e+02)))); + denominator = simde_mm512_fmadd_pd(denominator, q, simde_mm512_mask_mov_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C( 2.445134137142996e+00)), mask_el, simde_mm512_set1_pd(SIMDE_FLOAT64_C(-1.556989798598866e+02)))); + denominator = simde_mm512_fmadd_pd(denominator, q, simde_mm512_mask_mov_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C( 3.754408661907416e+00)), mask_el, simde_mm512_set1_pd(SIMDE_FLOAT64_C( 6.680131188771972e+01)))); + denominator = simde_mm512_fmadd_pd(denominator, simde_mm512_mask_mov_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C( 1.0)), mask_el, q), + simde_mm512_mask_mov_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C( 0.0)), mask_el, simde_mm512_set1_pd(SIMDE_FLOAT64_C(-1.328068155288572e+01)))); + denominator = simde_mm512_fmadd_pd(denominator, q, simde_mm512_set1_pd(SIMDE_FLOAT64_C(1.0))); + + /* res = numerator / denominator; */ + retval = simde_mm512_mask_div_pd(retval, mask_lo | mask_hi | mask_el, numerator, denominator); + } + + return retval; + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_cdfnorminv_pd + #define _mm512_cdfnorminv_pd(a) simde_mm512_cdfnorminv_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_cdfnorminv_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cdfnorminv_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_cdfnorminv_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cdfnorminv_ps + #define _mm512_mask_cdfnorminv_ps(src, k, a) simde_mm512_mask_cdfnorminv_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_cdfnorminv_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_cdfnorminv_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_cdfnorminv_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_cdfnorminv_pd + #define _mm512_mask_cdfnorminv_pd(src, k, a) simde_mm512_mask_cdfnorminv_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_erfinv_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_erfinv_ps(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + /* https://stackoverflow.com/questions/27229371/inverse-error-function-in-c */ + simde__m128 one = simde_mm_set1_ps(SIMDE_FLOAT32_C(1.0)); + + simde__m128 lnx = simde_mm_log_ps(simde_mm_mul_ps(simde_mm_sub_ps(one, a), simde_mm_add_ps(one, a))); + + simde__m128 tt1 = simde_mm_mul_ps(simde_mm_set1_ps(HEDLEY_STATIC_CAST(simde_float32, SIMDE_MATH_PI)), simde_mm_set1_ps(SIMDE_FLOAT32_C(0.147))); + tt1 = simde_mm_div_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C(2.0)), tt1); + tt1 = simde_mm_add_ps(tt1, simde_mm_mul_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C(0.5)), lnx)); + + simde__m128 tt2 = simde_mm_set1_ps(SIMDE_FLOAT32_C(1.0) / SIMDE_FLOAT32_C(0.147)); + tt2 = simde_mm_mul_ps(tt2, lnx); + + simde__m128 r = simde_mm_mul_ps(tt1, tt1); + r = simde_mm_sub_ps(r, tt2); + r = simde_mm_sqrt_ps(r); + r = simde_mm_add_ps(simde_x_mm_negate_ps(tt1), r); + r = simde_mm_sqrt_ps(r); + + return simde_x_mm_xorsign_ps(r, a); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_erfinvf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_erfinv_ps + #define _mm_erfinv_ps(a) simde_mm_erfinv_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_erfinv_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_erfinv_pd(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + simde__m128d one = simde_mm_set1_pd(SIMDE_FLOAT64_C(1.0)); + + simde__m128d lnx = simde_mm_log_pd(simde_mm_mul_pd(simde_mm_sub_pd(one, a), simde_mm_add_pd(one, a))); + + simde__m128d tt1 = simde_mm_mul_pd(simde_mm_set1_pd(SIMDE_MATH_PI), simde_mm_set1_pd(SIMDE_FLOAT64_C(0.147))); + tt1 = simde_mm_div_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(2.0)), tt1); + tt1 = simde_mm_add_pd(tt1, simde_mm_mul_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(0.5)), lnx)); + + simde__m128d tt2 = simde_mm_set1_pd(SIMDE_FLOAT64_C(1.0) / SIMDE_FLOAT64_C(0.147)); + tt2 = simde_mm_mul_pd(tt2, lnx); + + simde__m128d r = simde_mm_mul_pd(tt1, tt1); + r = simde_mm_sub_pd(r, tt2); + r = simde_mm_sqrt_pd(r); + r = simde_mm_add_pd(simde_x_mm_negate_pd(tt1), r); + r = simde_mm_sqrt_pd(r); + + return simde_x_mm_xorsign_pd(r, a); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_erfinv(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_erfinv_pd + #define _mm_erfinv_pd(a) simde_mm_erfinv_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_erfinv_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_erfinv_ps(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + simde__m256 one = simde_mm256_set1_ps(SIMDE_FLOAT32_C(1.0)); + simde__m256 sgn = simde_x_mm256_copysign_ps(one, a); + + a = simde_mm256_mul_ps(simde_mm256_sub_ps(one, a), simde_mm256_add_ps(one, a)); + simde__m256 lnx = simde_mm256_log_ps(a); + + simde__m256 tt1 = simde_mm256_mul_ps(simde_mm256_set1_ps(HEDLEY_STATIC_CAST(simde_float32, SIMDE_MATH_PI)), simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.147))); + tt1 = simde_mm256_div_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(2.0)), tt1); + tt1 = simde_mm256_add_ps(tt1, simde_mm256_mul_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.5)), lnx)); + + simde__m256 tt2 = simde_mm256_set1_ps(SIMDE_FLOAT32_C(1.0) / SIMDE_FLOAT32_C(0.147)); + tt2 = simde_mm256_mul_ps(tt2, lnx); + + simde__m256 r = simde_mm256_mul_ps(tt1, tt1); + r = simde_mm256_sub_ps(r, tt2); + r = simde_mm256_sqrt_ps(r); + r = simde_mm256_add_ps(simde_x_mm256_negate_ps(tt1), r); + r = simde_mm256_sqrt_ps(r); + + return simde_mm256_mul_ps(sgn, r); + #else + simde__m256_private + a_ = simde__m256_to_private(a), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_erfinvf(a_.f32[i]); + } + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_erfinv_ps + #define _mm256_erfinv_ps(a) simde_mm256_erfinv_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_erfinv_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_erfinv_pd(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + simde__m256d one = simde_mm256_set1_pd(SIMDE_FLOAT64_C(1.0)); + simde__m256d sgn = simde_x_mm256_copysign_pd(one, a); + + a = simde_mm256_mul_pd(simde_mm256_sub_pd(one, a), simde_mm256_add_pd(one, a)); + simde__m256d lnx = simde_mm256_log_pd(a); + + simde__m256d tt1 = simde_mm256_mul_pd(simde_mm256_set1_pd(SIMDE_MATH_PI), simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.147))); + tt1 = simde_mm256_div_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(2.0)), tt1); + tt1 = simde_mm256_add_pd(tt1, simde_mm256_mul_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.5)), lnx)); + + simde__m256d tt2 = simde_mm256_set1_pd(SIMDE_FLOAT64_C(1.0) / SIMDE_FLOAT64_C(0.147)); + tt2 = simde_mm256_mul_pd(tt2, lnx); + + simde__m256d r = simde_mm256_mul_pd(tt1, tt1); + r = simde_mm256_sub_pd(r, tt2); + r = simde_mm256_sqrt_pd(r); + r = simde_mm256_add_pd(simde_x_mm256_negate_pd(tt1), r); + r = simde_mm256_sqrt_pd(r); + + return simde_mm256_mul_pd(sgn, r); + #else + simde__m256d_private + a_ = simde__m256d_to_private(a), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_erfinv(a_.f64[i]); + } + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_erfinv_pd + #define _mm256_erfinv_pd(a) simde_mm256_erfinv_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_erfinv_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_erfinv_ps(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + simde__m512 one = simde_mm512_set1_ps(SIMDE_FLOAT32_C(1.0)); + simde__m512 sgn = simde_x_mm512_copysign_ps(one, a); + + a = simde_mm512_mul_ps(simde_mm512_sub_ps(one, a), simde_mm512_add_ps(one, a)); + simde__m512 lnx = simde_mm512_log_ps(a); + + simde__m512 tt1 = simde_mm512_mul_ps(simde_mm512_set1_ps(HEDLEY_STATIC_CAST(simde_float32, SIMDE_MATH_PI)), simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.147))); + tt1 = simde_mm512_div_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(2.0)), tt1); + tt1 = simde_mm512_add_ps(tt1, simde_mm512_mul_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.5)), lnx)); + + simde__m512 tt2 = simde_mm512_set1_ps(SIMDE_FLOAT32_C(1.0) / SIMDE_FLOAT32_C(0.147)); + tt2 = simde_mm512_mul_ps(tt2, lnx); + + simde__m512 r = simde_mm512_mul_ps(tt1, tt1); + r = simde_mm512_sub_ps(r, tt2); + r = simde_mm512_sqrt_ps(r); + r = simde_mm512_add_ps(simde_x_mm512_negate_ps(tt1), r); + r = simde_mm512_sqrt_ps(r); + + return simde_mm512_mul_ps(sgn, r); + #else + simde__m512_private + a_ = simde__m512_to_private(a), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_erfinvf(a_.f32[i]); + } + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_erfinv_ps + #define _mm512_erfinv_ps(a) simde_mm512_erfinv_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_erfinv_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_erfinv_pd(a); + #elif (SIMDE_NATURAL_VECTOR_SIZE > 0) + simde__m512d one = simde_mm512_set1_pd(SIMDE_FLOAT64_C(1.0)); + simde__m512d sgn = simde_x_mm512_copysign_pd(one, a); + + a = simde_mm512_mul_pd(simde_mm512_sub_pd(one, a), simde_mm512_add_pd(one, a)); + simde__m512d lnx = simde_mm512_log_pd(a); + + simde__m512d tt1 = simde_mm512_mul_pd(simde_mm512_set1_pd(SIMDE_MATH_PI), simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.147))); + tt1 = simde_mm512_div_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(2.0)), tt1); + tt1 = simde_mm512_add_pd(tt1, simde_mm512_mul_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.5)), lnx)); + + simde__m512d tt2 = simde_mm512_set1_pd(SIMDE_FLOAT64_C(1.0) / SIMDE_FLOAT64_C(0.147)); + tt2 = simde_mm512_mul_pd(tt2, lnx); + + simde__m512d r = simde_mm512_mul_pd(tt1, tt1); + r = simde_mm512_sub_pd(r, tt2); + r = simde_mm512_sqrt_pd(r); + r = simde_mm512_add_pd(simde_x_mm512_negate_pd(tt1), r); + r = simde_mm512_sqrt_pd(r); + + return simde_mm512_mul_pd(sgn, r); + #else + simde__m512d_private + a_ = simde__m512d_to_private(a), + r_; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_erfinv(a_.f64[i]); + } + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_erfinv_pd + #define _mm512_erfinv_pd(a) simde_mm512_erfinv_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_erfinv_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_erfinv_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_erfinv_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_erfinv_ps + #define _mm512_mask_erfinv_ps(src, k, a) simde_mm512_mask_erfinv_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_erfinv_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_erfinv_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_erfinv_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_erfinv_pd + #define _mm512_mask_erfinv_pd(src, k, a) simde_mm512_mask_erfinv_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_erfcinv_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_erfcinv_ps(a); + #elif SIMDE_NATURAL_VECTOR_SIZE_GE(128) + simde__m128 matched, retval = simde_mm_setzero_ps(); + + { /* if (a < 2.0f && a > 0.0625f) */ + matched = simde_mm_cmplt_ps(a, simde_mm_set1_ps(SIMDE_FLOAT32_C(2.0))); + matched = simde_mm_and_ps(matched, simde_mm_cmpgt_ps(a, simde_mm_set1_ps(SIMDE_FLOAT32_C(0.0625)))); + + if (!simde_mm_test_all_zeros(simde_mm_castps_si128(matched), simde_x_mm_setone_si128())) { + retval = simde_mm_erfinv_ps(simde_mm_sub_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C(1.0)), a)); + } + + if (simde_mm_test_all_ones(simde_mm_castps_si128(matched))) { + return retval; + } + } + + { /* else if (a < 0.0625f && a > 0.0f) */ + simde__m128 mask = simde_mm_cmplt_ps(a, simde_mm_set1_ps(SIMDE_FLOAT32_C(0.0625))); + mask = simde_mm_and_ps(mask, simde_mm_cmpgt_ps(a, simde_mm_set1_ps(SIMDE_FLOAT32_C(0.0)))); + mask = simde_mm_andnot_ps(matched, mask); + + if (!simde_mm_test_all_zeros(simde_mm_castps_si128(mask), simde_x_mm_setone_si128())) { + matched = simde_mm_or_ps(matched, mask); + + /* t = 1/(sqrt(-log(a))) */ + simde__m128 t = simde_x_mm_negate_ps(simde_mm_log_ps(a)); + t = simde_mm_sqrt_ps(t); + t = simde_mm_div_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C(1.0)), t); + + const simde__m128 p[] = { + simde_mm_set1_ps(SIMDE_FLOAT32_C( 0.1550470003116)), + simde_mm_set1_ps(SIMDE_FLOAT32_C( 1.382719649631)), + simde_mm_set1_ps(SIMDE_FLOAT32_C( 0.690969348887)), + simde_mm_set1_ps(SIMDE_FLOAT32_C(-1.128081391617)), + simde_mm_set1_ps(SIMDE_FLOAT32_C( 0.680544246825)), + simde_mm_set1_ps(SIMDE_FLOAT32_C(-0.164441567910)) + }; + + const simde__m128 q[] = { + simde_mm_set1_ps(SIMDE_FLOAT32_C( 0.155024849822)), + simde_mm_set1_ps(SIMDE_FLOAT32_C( 1.385228141995)), + simde_mm_set1_ps(SIMDE_FLOAT32_C( 1.000000000000)) + }; + + /* float numerator = p[0] / t + p[1] + t * (p[2] + t * (p[3] + t * (p[4] + t * p[5])))) */ + simde__m128 numerator = simde_mm_fmadd_ps(p[5], t, p[4]); + numerator = simde_mm_fmadd_ps(numerator, t, p[3]); + numerator = simde_mm_fmadd_ps(numerator, t, p[2]); + numerator = simde_mm_fmadd_ps(numerator, t, p[1]); + numerator = simde_mm_add_ps(numerator, simde_mm_div_ps(p[0], t)); + + /* float denominator = (q[0] + t * (q[1] + t * (q[2]))) */ + simde__m128 denominator = simde_mm_fmadd_ps(q[2], t, q[1]); + denominator = simde_mm_fmadd_ps(denominator, t, q[0]); + + simde__m128 res = simde_mm_div_ps(numerator, denominator); + + retval = simde_mm_or_ps(retval, simde_mm_and_ps(mask, res)); + } + } + + { /* else if (a < 0.0f) */ + simde__m128 mask = simde_mm_cmplt_ps(a, simde_mm_set1_ps(SIMDE_FLOAT32_C(0.0))); + mask = simde_mm_andnot_ps(matched, mask); + + if (!simde_mm_test_all_zeros(simde_mm_castps_si128(mask), simde_x_mm_setone_si128())) { + matched = simde_mm_or_ps(matched, mask); + + /* t = 1/(sqrt(-log(a))) */ + simde__m128 t = simde_x_mm_negate_ps(simde_mm_log_ps(a)); + t = simde_mm_sqrt_ps(t); + t = simde_mm_div_ps(simde_mm_set1_ps(SIMDE_FLOAT32_C(1.0)), t); + + const simde__m128 p[] = { + simde_mm_set1_ps(SIMDE_FLOAT32_C( 0.00980456202915)), + simde_mm_set1_ps(SIMDE_FLOAT32_C( 0.36366788917100)), + simde_mm_set1_ps(SIMDE_FLOAT32_C( 0.97302949837000)), + simde_mm_set1_ps(SIMDE_FLOAT32_C( -0.5374947401000)) + }; + + const simde__m128 q[] = { + simde_mm_set1_ps(SIMDE_FLOAT32_C( 0.00980451277802)), + simde_mm_set1_ps(SIMDE_FLOAT32_C( 0.36369997154400)), + simde_mm_set1_ps(SIMDE_FLOAT32_C( 1.00000000000000)) + }; + + /* float numerator = (p[0] / t + p[1] + t * (p[2] + t * p[3])) */ + simde__m128 numerator = simde_mm_fmadd_ps(p[3], t, p[2]); + numerator = simde_mm_fmadd_ps(numerator, t, p[1]); + numerator = simde_mm_add_ps(numerator, simde_mm_div_ps(p[0], t)); + + /* float denominator = (q[0] + t * (q[1] + t * (q[2]))) */ + simde__m128 denominator = simde_mm_fmadd_ps(q[2], t, q[1]); + denominator = simde_mm_fmadd_ps(denominator, t, q[0]); + + simde__m128 res = simde_mm_div_ps(numerator, denominator); + + retval = simde_mm_or_ps(retval, simde_mm_and_ps(mask, res)); + + if (simde_mm_test_all_ones(simde_mm_castps_si128(matched))) { + return retval; + } + } + } + + { /* else if (a == 0.0f) */ + simde__m128 mask = simde_mm_cmpeq_ps(a, simde_mm_set1_ps(SIMDE_FLOAT32_C(0.0))); + mask = simde_mm_andnot_ps(matched, mask); + matched = simde_mm_or_ps(matched, mask); + + simde__m128 res = simde_mm_set1_ps(SIMDE_MATH_INFINITYF); + + retval = simde_mm_or_ps(retval, simde_mm_and_ps(mask, res)); + } + + { /* else */ + /* (a >= 2.0f) */ + retval = simde_mm_or_ps(retval, simde_mm_andnot_ps(matched, simde_mm_set1_ps(-SIMDE_MATH_INFINITYF))); + } + + return retval; + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_erfcinvf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_erfcinv_ps + #define _mm_erfcinv_ps(a) simde_mm_erfcinv_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_erfcinv_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_erfcinv_pd(a); + #elif SIMDE_NATURAL_VECTOR_SIZE_GE(128) + simde__m128d matched, retval = simde_mm_setzero_pd(); + + { /* if (a < 2.0 && a > 0.0625) */ + matched = simde_mm_cmplt_pd(a, simde_mm_set1_pd(SIMDE_FLOAT64_C(2.0))); + matched = simde_mm_and_pd(matched, simde_mm_cmpgt_pd(a, simde_mm_set1_pd(SIMDE_FLOAT64_C(0.0625)))); + + if (!simde_mm_test_all_zeros(simde_mm_castpd_si128(matched), simde_x_mm_setone_si128())) { + retval = simde_mm_erfinv_pd(simde_mm_sub_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(1.0)), a)); + } + + if (simde_mm_test_all_ones(simde_mm_castpd_si128(matched))) { + return retval; + } + } + + { /* else if (a < 0.0625 && a > 0.0) */ + simde__m128d mask = simde_mm_cmplt_pd(a, simde_mm_set1_pd(SIMDE_FLOAT64_C(0.0625))); + mask = simde_mm_and_pd(mask, simde_mm_cmpgt_pd(a, simde_mm_set1_pd(SIMDE_FLOAT64_C(0.0)))); + mask = simde_mm_andnot_pd(matched, mask); + + if (!simde_mm_test_all_zeros(simde_mm_castpd_si128(mask), simde_x_mm_setone_si128())) { + matched = simde_mm_or_pd(matched, mask); + + /* t = 1/(sqrt(-log(a))) */ + simde__m128d t = simde_x_mm_negate_pd(simde_mm_log_pd(a)); + t = simde_mm_sqrt_pd(t); + t = simde_mm_div_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(1.0)), t); + + const simde__m128d p[] = { + simde_mm_set1_pd(SIMDE_FLOAT64_C( 0.1550470003116)), + simde_mm_set1_pd(SIMDE_FLOAT64_C( 1.382719649631)), + simde_mm_set1_pd(SIMDE_FLOAT64_C( 0.690969348887)), + simde_mm_set1_pd(SIMDE_FLOAT64_C(-1.128081391617)), + simde_mm_set1_pd(SIMDE_FLOAT64_C( 0.680544246825)), + simde_mm_set1_pd(SIMDE_FLOAT64_C(-0.164441567910)) + }; + + const simde__m128d q[] = { + simde_mm_set1_pd(SIMDE_FLOAT64_C( 0.155024849822)), + simde_mm_set1_pd(SIMDE_FLOAT64_C( 1.385228141995)), + simde_mm_set1_pd(SIMDE_FLOAT64_C( 1.000000000000)) + }; + + /* float numerator = p[0] / t + p[1] + t * (p[2] + t * (p[3] + t * (p[4] + t * p[5])))) */ + simde__m128d numerator = simde_mm_fmadd_pd(p[5], t, p[4]); + numerator = simde_mm_fmadd_pd(numerator, t, p[3]); + numerator = simde_mm_fmadd_pd(numerator, t, p[2]); + numerator = simde_mm_fmadd_pd(numerator, t, p[1]); + numerator = simde_mm_add_pd(numerator, simde_mm_div_pd(p[0], t)); + + /* float denominator = (q[0] + t * (q[1] + t * (q[2]))) */ + simde__m128d denominator = simde_mm_fmadd_pd(q[2], t, q[1]); + denominator = simde_mm_fmadd_pd(denominator, t, q[0]); + + simde__m128d res = simde_mm_div_pd(numerator, denominator); + + retval = simde_mm_or_pd(retval, simde_mm_and_pd(mask, res)); + } + } + + { /* else if (a < 0.0) */ + simde__m128d mask = simde_mm_cmplt_pd(a, simde_mm_set1_pd(SIMDE_FLOAT64_C(0.0))); + mask = simde_mm_andnot_pd(matched, mask); + + if (!simde_mm_test_all_zeros(simde_mm_castpd_si128(mask), simde_x_mm_setone_si128())) { + matched = simde_mm_or_pd(matched, mask); + + /* t = 1/(sqrt(-log(a))) */ + simde__m128d t = simde_x_mm_negate_pd(simde_mm_log_pd(a)); + t = simde_mm_sqrt_pd(t); + t = simde_mm_div_pd(simde_mm_set1_pd(SIMDE_FLOAT64_C(1.0)), t); + + const simde__m128d p[] = { + simde_mm_set1_pd(SIMDE_FLOAT64_C( 0.00980456202915)), + simde_mm_set1_pd(SIMDE_FLOAT64_C( 0.36366788917100)), + simde_mm_set1_pd(SIMDE_FLOAT64_C( 0.97302949837000)), + simde_mm_set1_pd(SIMDE_FLOAT64_C( -0.5374947401000)) + }; + + const simde__m128d q[] = { + simde_mm_set1_pd(SIMDE_FLOAT64_C( 0.00980451277802)), + simde_mm_set1_pd(SIMDE_FLOAT64_C( 0.36369997154400)), + simde_mm_set1_pd(SIMDE_FLOAT64_C( 1.00000000000000)) + }; + + /* float numerator = (p[0] / t + p[1] + t * (p[2] + t * p[3])) */ + simde__m128d numerator = simde_mm_fmadd_pd(p[3], t, p[2]); + numerator = simde_mm_fmadd_pd(numerator, t, p[1]); + numerator = simde_mm_add_pd(numerator, simde_mm_div_pd(p[0], t)); + + /* float denominator = (q[0] + t * (q[1] + t * (q[2]))) */ + simde__m128d denominator = simde_mm_fmadd_pd(q[2], t, q[1]); + denominator = simde_mm_fmadd_pd(denominator, t, q[0]); + + simde__m128d res = simde_mm_div_pd(numerator, denominator); + + retval = simde_mm_or_pd(retval, simde_mm_and_pd(mask, res)); + + if (simde_mm_test_all_ones(simde_mm_castpd_si128(matched))) { + return retval; + } + } + } + + { /* else if (a == 0.0) */ + simde__m128d mask = simde_mm_cmpeq_pd(a, simde_mm_set1_pd(SIMDE_FLOAT64_C(0.0))); + mask = simde_mm_andnot_pd(matched, mask); + matched = simde_mm_or_pd(matched, mask); + + simde__m128d res = simde_mm_set1_pd(SIMDE_MATH_INFINITY); + + retval = simde_mm_or_pd(retval, simde_mm_and_pd(mask, res)); + } + + { /* else */ + /* (a >= 2.0) */ + retval = simde_mm_or_pd(retval, simde_mm_andnot_pd(matched, simde_mm_set1_pd(-SIMDE_MATH_INFINITY))); + } + + return retval; + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_erfcinv(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_erfcinv_pd + #define _mm_erfcinv_pd(a) simde_mm_erfcinv_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_erfcinv_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_erfcinv_ps(a); + #elif SIMDE_NATURAL_VECTOR_SIZE_GE(256) + simde__m256 matched, retval = simde_mm256_setzero_ps(); + + { /* if (a < 2.0f && a > 0.0625f) */ + matched = simde_mm256_cmp_ps(a, simde_mm256_set1_ps(SIMDE_FLOAT32_C(2.0)), SIMDE_CMP_LT_OQ); + matched = simde_mm256_and_ps(matched, simde_mm256_cmp_ps(a, simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.0625)), SIMDE_CMP_GT_OQ)); + + if (!simde_mm256_testz_ps(matched, matched)) { + retval = simde_mm256_erfinv_ps(simde_mm256_sub_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(1.0)), a)); + } + + if (simde_x_mm256_test_all_ones(simde_mm256_castps_si256(matched))) { + return retval; + } + } + + { /* else if (a < 0.0625f && a > 0.0f) */ + simde__m256 mask = simde_mm256_cmp_ps(a, simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.0625)), SIMDE_CMP_LT_OQ); + mask = simde_mm256_and_ps(mask, simde_mm256_cmp_ps(a, simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.0)), SIMDE_CMP_GT_OQ)); + mask = simde_mm256_andnot_ps(matched, mask); + + if (!simde_mm256_testz_ps(mask, mask)) { + matched = simde_mm256_or_ps(matched, mask); + + /* t = 1/(sqrt(-log(a))) */ + simde__m256 t = simde_x_mm256_negate_ps(simde_mm256_log_ps(a)); + t = simde_mm256_sqrt_ps(t); + t = simde_mm256_div_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(1.0)), t); + + const simde__m256 p[] = { + simde_mm256_set1_ps(SIMDE_FLOAT32_C( 0.1550470003116)), + simde_mm256_set1_ps(SIMDE_FLOAT32_C( 1.382719649631)), + simde_mm256_set1_ps(SIMDE_FLOAT32_C( 0.690969348887)), + simde_mm256_set1_ps(SIMDE_FLOAT32_C(-1.128081391617)), + simde_mm256_set1_ps(SIMDE_FLOAT32_C( 0.680544246825)), + simde_mm256_set1_ps(SIMDE_FLOAT32_C(-0.16444156791)) + }; + + const simde__m256 q[] = { + simde_mm256_set1_ps(SIMDE_FLOAT32_C( 0.155024849822)), + simde_mm256_set1_ps(SIMDE_FLOAT32_C( 1.385228141995)), + simde_mm256_set1_ps(SIMDE_FLOAT32_C( 1.000000000000)) + }; + + /* float numerator = p[0] / t + p[1] + t * (p[2] + t * (p[3] + t * (p[4] + t * p[5])))) */ + simde__m256 numerator = simde_mm256_fmadd_ps(p[5], t, p[4]); + numerator = simde_mm256_fmadd_ps(numerator, t, p[3]); + numerator = simde_mm256_fmadd_ps(numerator, t, p[2]); + numerator = simde_mm256_fmadd_ps(numerator, t, p[1]); + numerator = simde_mm256_add_ps(numerator, simde_mm256_div_ps(p[0], t)); + + /* float denominator = (q[0] + t * (q[1] + t * (q[2]))) */ + simde__m256 denominator = simde_mm256_fmadd_ps(q[2], t, q[1]); + denominator = simde_mm256_fmadd_ps(denominator, t, q[0]); + + simde__m256 res = simde_mm256_div_ps(numerator, denominator); + + retval = simde_mm256_or_ps(retval, simde_mm256_and_ps(mask, res)); + } + } + + { /* else if (a < 0.0f) */ + simde__m256 mask = simde_mm256_cmp_ps(a, simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.0)), SIMDE_CMP_LT_OQ); + mask = simde_mm256_andnot_ps(matched, mask); + + if (!simde_mm256_testz_ps(mask, mask)) { + matched = simde_mm256_or_ps(matched, mask); + + /* t = 1/(sqrt(-log(a))) */ + simde__m256 t = simde_x_mm256_negate_ps(simde_mm256_log_ps(a)); + t = simde_mm256_sqrt_ps(t); + t = simde_mm256_div_ps(simde_mm256_set1_ps(SIMDE_FLOAT32_C(1.0)), t); + + const simde__m256 p[] = { + simde_mm256_set1_ps(SIMDE_FLOAT32_C( 0.00980456202915)), + simde_mm256_set1_ps(SIMDE_FLOAT32_C( 0.36366788917100)), + simde_mm256_set1_ps(SIMDE_FLOAT32_C( 0.97302949837000)), + simde_mm256_set1_ps(SIMDE_FLOAT32_C(-0.5374947401000)) + }; + + const simde__m256 q[] = { + simde_mm256_set1_ps(SIMDE_FLOAT32_C( 0.00980451277802)), + simde_mm256_set1_ps(SIMDE_FLOAT32_C( 0.36369997154400)), + simde_mm256_set1_ps(SIMDE_FLOAT32_C( 1.00000000000000)) + }; + + /* float numerator = (p[0] / t + p[1] + t * (p[2] + t * p[3])) */ + simde__m256 numerator = simde_mm256_fmadd_ps(p[3], t, p[2]); + numerator = simde_mm256_fmadd_ps(numerator, t, p[1]); + numerator = simde_mm256_add_ps(numerator, simde_mm256_div_ps(p[0], t)); + + /* float denominator = (q[0] + t * (q[1] + t * (q[2]))) */ + simde__m256 denominator = simde_mm256_fmadd_ps(q[2], t, q[1]); + denominator = simde_mm256_fmadd_ps(denominator, t, q[0]); + + simde__m256 res = simde_mm256_div_ps(numerator, denominator); + + retval = simde_mm256_or_ps(retval, simde_mm256_and_ps(mask, res)); + + if (simde_x_mm256_test_all_ones(simde_mm256_castps_si256(matched))) { + return retval; + } + } + } + + { /* else if (a == 0.0f) */ + simde__m256 mask = simde_mm256_cmp_ps(a, simde_mm256_set1_ps(SIMDE_FLOAT32_C(0.0)), SIMDE_CMP_EQ_OQ); + mask = simde_mm256_andnot_ps(matched, mask); + matched = simde_mm256_or_ps(matched, mask); + + simde__m256 res = simde_mm256_set1_ps(SIMDE_MATH_INFINITYF); + + retval = simde_mm256_or_ps(retval, simde_mm256_and_ps(mask, res)); + } + + { /* else */ + /* (a >= 2.0f) */ + retval = simde_mm256_or_ps(retval, simde_mm256_andnot_ps(matched, simde_mm256_set1_ps(-SIMDE_MATH_INFINITYF))); + } + + return retval; + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_erfcinv_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_erfcinvf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_erfcinv_ps + #define _mm256_erfcinv_ps(a) simde_mm256_erfcinv_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_erfcinv_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_erfcinv_pd(a); + #elif SIMDE_NATURAL_VECTOR_SIZE_GE(256) + simde__m256d matched, retval = simde_mm256_setzero_pd(); + + { /* if (a < 2.0 && a > 0.0625) */ + matched = simde_mm256_cmp_pd(a, simde_mm256_set1_pd(SIMDE_FLOAT64_C(2.0)), SIMDE_CMP_LT_OQ); + matched = simde_mm256_and_pd(matched, simde_mm256_cmp_pd(a, simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.0625)), SIMDE_CMP_GT_OQ)); + + if (!simde_mm256_testz_pd(matched, matched)) { + retval = simde_mm256_erfinv_pd(simde_mm256_sub_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(1.0)), a)); + } + + if (simde_x_mm256_test_all_ones(simde_mm256_castpd_si256(matched))) { + return retval; + } + } + + { /* else if (a < 0.0625 && a > 0.0) */ + simde__m256d mask = simde_mm256_cmp_pd(a, simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.0625)), SIMDE_CMP_LT_OQ); + mask = simde_mm256_and_pd(mask, simde_mm256_cmp_pd(a, simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.0)), SIMDE_CMP_GT_OQ)); + mask = simde_mm256_andnot_pd(matched, mask); + + if (!simde_mm256_testz_pd(mask, mask)) { + matched = simde_mm256_or_pd(matched, mask); + + /* t = 1/(sqrt(-log(a))) */ + simde__m256d t = simde_x_mm256_negate_pd(simde_mm256_log_pd(a)); + t = simde_mm256_sqrt_pd(t); + t = simde_mm256_div_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(1.0)), t); + + const simde__m256d p[] = { + simde_mm256_set1_pd(SIMDE_FLOAT64_C( 0.1550470003116)), + simde_mm256_set1_pd(SIMDE_FLOAT64_C( 1.382719649631)), + simde_mm256_set1_pd(SIMDE_FLOAT64_C( 0.690969348887)), + simde_mm256_set1_pd(SIMDE_FLOAT64_C(-1.128081391617)), + simde_mm256_set1_pd(SIMDE_FLOAT64_C( 0.680544246825)), + simde_mm256_set1_pd(SIMDE_FLOAT64_C(-0.16444156791)) + }; + + const simde__m256d q[] = { + simde_mm256_set1_pd(SIMDE_FLOAT64_C( 0.155024849822)), + simde_mm256_set1_pd(SIMDE_FLOAT64_C( 1.385228141995)), + simde_mm256_set1_pd(SIMDE_FLOAT64_C( 1.000000000000)) + }; + + /* float numerator = p[0] / t + p[1] + t * (p[2] + t * (p[3] + t * (p[4] + t * p[5])))) */ + simde__m256d numerator = simde_mm256_fmadd_pd(p[5], t, p[4]); + numerator = simde_mm256_fmadd_pd(numerator, t, p[3]); + numerator = simde_mm256_fmadd_pd(numerator, t, p[2]); + numerator = simde_mm256_fmadd_pd(numerator, t, p[1]); + numerator = simde_mm256_add_pd(numerator, simde_mm256_div_pd(p[0], t)); + + /* float denominator = (q[0] + t * (q[1] + t * (q[2]))) */ + simde__m256d denominator = simde_mm256_fmadd_pd(q[2], t, q[1]); + denominator = simde_mm256_fmadd_pd(denominator, t, q[0]); + + simde__m256d res = simde_mm256_div_pd(numerator, denominator); + + retval = simde_mm256_or_pd(retval, simde_mm256_and_pd(mask, res)); + } + } + + { /* else if (a < 0.0) */ + simde__m256d mask = simde_mm256_cmp_pd(a, simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.0)), SIMDE_CMP_LT_OQ); + mask = simde_mm256_andnot_pd(matched, mask); + + if (!simde_mm256_testz_pd(mask, mask)) { + matched = simde_mm256_or_pd(matched, mask); + + /* t = 1/(sqrt(-log(a))) */ + simde__m256d t = simde_x_mm256_negate_pd(simde_mm256_log_pd(a)); + t = simde_mm256_sqrt_pd(t); + t = simde_mm256_div_pd(simde_mm256_set1_pd(SIMDE_FLOAT64_C(1.0)), t); + + const simde__m256d p[] = { + simde_mm256_set1_pd(SIMDE_FLOAT64_C( 0.00980456202915)), + simde_mm256_set1_pd(SIMDE_FLOAT64_C( 0.36366788917100)), + simde_mm256_set1_pd(SIMDE_FLOAT64_C( 0.97302949837000)), + simde_mm256_set1_pd(SIMDE_FLOAT64_C(-0.5374947401000)) + }; + + const simde__m256d q[] = { + simde_mm256_set1_pd(SIMDE_FLOAT64_C( 0.00980451277802)), + simde_mm256_set1_pd(SIMDE_FLOAT64_C( 0.36369997154400)), + simde_mm256_set1_pd(SIMDE_FLOAT64_C( 1.00000000000000)) + }; + + /* float numerator = (p[0] / t + p[1] + t * (p[2] + t * p[3])) */ + simde__m256d numerator = simde_mm256_fmadd_pd(p[3], t, p[2]); + numerator = simde_mm256_fmadd_pd(numerator, t, p[1]); + numerator = simde_mm256_add_pd(numerator, simde_mm256_div_pd(p[0], t)); + + /* float denominator = (q[0] + t * (q[1] + t * (q[2]))) */ + simde__m256d denominator = simde_mm256_fmadd_pd(q[2], t, q[1]); + denominator = simde_mm256_fmadd_pd(denominator, t, q[0]); + + simde__m256d res = simde_mm256_div_pd(numerator, denominator); + + retval = simde_mm256_or_pd(retval, simde_mm256_and_pd(mask, res)); + + if (simde_x_mm256_test_all_ones(simde_mm256_castpd_si256(matched))) { + return retval; + } + } + } + + { /* else if (a == 0.0) */ + simde__m256d mask = simde_mm256_cmp_pd(a, simde_mm256_set1_pd(SIMDE_FLOAT64_C(0.0)), SIMDE_CMP_EQ_OQ); + mask = simde_mm256_andnot_pd(matched, mask); + matched = simde_mm256_or_pd(matched, mask); + + simde__m256d res = simde_mm256_set1_pd(SIMDE_MATH_INFINITY); + + retval = simde_mm256_or_pd(retval, simde_mm256_and_pd(mask, res)); + } + + { /* else */ + /* (a >= 2.0) */ + retval = simde_mm256_or_pd(retval, simde_mm256_andnot_pd(matched, simde_mm256_set1_pd(-SIMDE_MATH_INFINITY))); + } + + return retval; + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_erfcinv_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_erfcinv(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_erfcinv_pd + #define _mm256_erfcinv_pd(a) simde_mm256_erfcinv_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_erfcinv_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_erfcinv_ps(a); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) && (!defined(SIMDE_ARCH_ARM) || defined(SIMDE_ARCH_AARCH64)) + /* The results on Arm are *slightly* off, which causes problems for + * the edge cases; for example, if you pass 2.0 sqrt will be called + * with a value of -0.0 instead of 0.0, resulting in a NaN. */ + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_erfcinv_ps(a_.m256[i]); + } + return simde__m512_from_private(r_); + #else + simde__m512 retval = simde_mm512_setzero_ps(); + simde__mmask16 matched; + + { /* if (a < 2.0f && a > 0.0625f) */ + matched = simde_mm512_cmp_ps_mask(a, simde_mm512_set1_ps(SIMDE_FLOAT32_C(2.0)), SIMDE_CMP_LT_OQ); + matched &= simde_mm512_cmp_ps_mask(a, simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.0625)), SIMDE_CMP_GT_OQ); + + if (matched != 0) { + retval = simde_mm512_erfinv_ps(simde_mm512_sub_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(1.0)), a)); + } + + if (matched == 1) { + return retval; + } + } + + { /* else if (a < 0.0625f && a > 0.0f) */ + simde__mmask16 mask = simde_mm512_cmp_ps_mask(a, simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.0625)), SIMDE_CMP_LT_OQ); + mask &= simde_mm512_cmp_ps_mask(a, simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.0)), SIMDE_CMP_GT_OQ); + mask = ~matched & mask; + + if (mask != 0) { + matched = matched | mask; + + /* t = 1/(sqrt(-log(a))) */ + simde__m512 t = simde_x_mm512_negate_ps(simde_mm512_log_ps(a)); + t = simde_mm512_sqrt_ps(t); + t = simde_mm512_div_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(1.0)), t); + + const simde__m512 p[] = { + simde_mm512_set1_ps(SIMDE_FLOAT32_C( 0.1550470003116)), + simde_mm512_set1_ps(SIMDE_FLOAT32_C( 1.382719649631)), + simde_mm512_set1_ps(SIMDE_FLOAT32_C( 0.690969348887)), + simde_mm512_set1_ps(SIMDE_FLOAT32_C(-1.128081391617)), + simde_mm512_set1_ps(SIMDE_FLOAT32_C( 0.680544246825)), + simde_mm512_set1_ps(SIMDE_FLOAT32_C(-0.16444156791)) + }; + + const simde__m512 q[] = { + simde_mm512_set1_ps(SIMDE_FLOAT32_C( 0.155024849822)), + simde_mm512_set1_ps(SIMDE_FLOAT32_C( 1.385228141995)), + simde_mm512_set1_ps(SIMDE_FLOAT32_C( 1.000000000000)) + }; + + /* float numerator = p[0] / t + p[1] + t * (p[2] + t * (p[3] + t * (p[4] + t * p[5])))) */ + simde__m512 numerator = simde_mm512_fmadd_ps(p[5], t, p[4]); + numerator = simde_mm512_fmadd_ps(numerator, t, p[3]); + numerator = simde_mm512_fmadd_ps(numerator, t, p[2]); + numerator = simde_mm512_fmadd_ps(numerator, t, p[1]); + numerator = simde_mm512_add_ps(numerator, simde_mm512_div_ps(p[0], t)); + + /* float denominator = (q[0] + t * (q[1] + t * (q[2]))) */ + simde__m512 denominator = simde_mm512_fmadd_ps(q[2], t, q[1]); + denominator = simde_mm512_fmadd_ps(denominator, t, q[0]); + + simde__m512 res = simde_mm512_div_ps(numerator, denominator); + + retval = simde_mm512_or_ps(retval, simde_mm512_maskz_mov_ps(mask, res)); + } + } + + { /* else if (a < 0.0f) */ + simde__mmask16 mask = simde_mm512_cmp_ps_mask(a, simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.0)), SIMDE_CMP_LT_OQ); + mask = ~matched & mask; + + if (mask != 0) { + matched = matched | mask; + + /* t = 1/(sqrt(-log(a))) */ + simde__m512 t = simde_x_mm512_negate_ps(simde_mm512_log_ps(a)); + t = simde_mm512_sqrt_ps(t); + t = simde_mm512_div_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(1.0)), t); + + const simde__m512 p[] = { + simde_mm512_set1_ps(SIMDE_FLOAT32_C( 0.00980456202915)), + simde_mm512_set1_ps(SIMDE_FLOAT32_C( 0.36366788917100)), + simde_mm512_set1_ps(SIMDE_FLOAT32_C( 0.97302949837000)), + simde_mm512_set1_ps(SIMDE_FLOAT32_C( -0.5374947401000)) + }; + + const simde__m512 q[] = { + simde_mm512_set1_ps(SIMDE_FLOAT32_C( 0.00980451277802)), + simde_mm512_set1_ps(SIMDE_FLOAT32_C( 0.36369997154400)), + simde_mm512_set1_ps(SIMDE_FLOAT32_C( 1.00000000000000)) + }; + + /* float numerator = (p[0] / t + p[1] + t * (p[2] + t * p[3])) */ + simde__m512 numerator = simde_mm512_fmadd_ps(p[3], t, p[2]); + numerator = simde_mm512_fmadd_ps(numerator, t, p[1]); + numerator = simde_mm512_add_ps(numerator, simde_mm512_div_ps(p[0], t)); + + /* float denominator = (q[0] + t * (q[1] + t * (q[2]))) */ + simde__m512 denominator = simde_mm512_fmadd_ps(q[2], t, q[1]); + denominator = simde_mm512_fmadd_ps(denominator, t, q[0]); + + simde__m512 res = simde_mm512_div_ps(numerator, denominator); + + retval = simde_mm512_or_ps(retval, simde_mm512_maskz_mov_ps(mask, res)); + + if (matched == 1) { + return retval; + } + } + } + + { /* else if (a == 0.0f) */ + simde__mmask16 mask = simde_mm512_cmp_ps_mask(a, simde_mm512_set1_ps(SIMDE_FLOAT32_C(0.0)), SIMDE_CMP_EQ_OQ); + mask = ~matched & mask; + matched = matched | mask; + + simde__m512 res = simde_mm512_set1_ps(SIMDE_MATH_INFINITYF); + + retval = simde_mm512_or_ps(retval, simde_mm512_maskz_mov_ps(mask, res)); + } + + { /* else */ + /* (a >= 2.0f) */ + retval = simde_mm512_or_ps(retval, simde_mm512_maskz_mov_ps(~matched, simde_mm512_set1_ps(-SIMDE_MATH_INFINITYF))); + } + + return retval; + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_erfcinv_ps + #define _mm512_erfcinv_ps(a) simde_mm512_erfcinv_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_erfcinv_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_erfcinv_pd(a); + #elif SIMDE_NATURAL_VECTOR_SIZE_LE(256) + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_erfcinv_pd(a_.m256d[i]); + } + return simde__m512d_from_private(r_); + #else + simde__m512d retval = simde_mm512_setzero_pd(); + simde__mmask8 matched; + + { /* if (a < 2.0f && a > 0.0625f) */ + matched = simde_mm512_cmp_pd_mask(a, simde_mm512_set1_pd(SIMDE_FLOAT64_C(2.0)), SIMDE_CMP_LT_OQ); + matched &= simde_mm512_cmp_pd_mask(a, simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.0625)), SIMDE_CMP_GT_OQ); + + if (matched != 0) { + retval = simde_mm512_erfinv_pd(simde_mm512_sub_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(1.0)), a)); + } + + if (matched == 1) { + return retval; + } + } + + { /* else if (a < 0.0625f && a > 0.0f) */ + simde__mmask8 mask = simde_mm512_cmp_pd_mask(a, simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.0625)), SIMDE_CMP_LT_OQ); + mask &= simde_mm512_cmp_pd_mask(a, simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.0)), SIMDE_CMP_GT_OQ); + mask = ~matched & mask; + + if (mask != 0) { + matched = matched | mask; + + /* t = 1/(sqrt(-log(a))) */ + simde__m512d t = simde_x_mm512_negate_pd(simde_mm512_log_pd(a)); + t = simde_mm512_sqrt_pd(t); + t = simde_mm512_div_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(1.0)), t); + + const simde__m512d p[] = { + simde_mm512_set1_pd(SIMDE_FLOAT64_C( 0.1550470003116)), + simde_mm512_set1_pd(SIMDE_FLOAT64_C( 1.382719649631)), + simde_mm512_set1_pd(SIMDE_FLOAT64_C( 0.690969348887)), + simde_mm512_set1_pd(SIMDE_FLOAT64_C(-1.128081391617)), + simde_mm512_set1_pd(SIMDE_FLOAT64_C( 0.680544246825)), + simde_mm512_set1_pd(SIMDE_FLOAT64_C(-0.16444156791)) + }; + + const simde__m512d q[] = { + simde_mm512_set1_pd(SIMDE_FLOAT64_C( 0.155024849822)), + simde_mm512_set1_pd(SIMDE_FLOAT64_C( 1.385228141995)), + simde_mm512_set1_pd(SIMDE_FLOAT64_C( 1.000000000000)) + }; + + /* float numerator = p[0] / t + p[1] + t * (p[2] + t * (p[3] + t * (p[4] + t * p[5])))) */ + simde__m512d numerator = simde_mm512_fmadd_pd(p[5], t, p[4]); + numerator = simde_mm512_fmadd_pd(numerator, t, p[3]); + numerator = simde_mm512_fmadd_pd(numerator, t, p[2]); + numerator = simde_mm512_fmadd_pd(numerator, t, p[1]); + numerator = simde_mm512_add_pd(numerator, simde_mm512_div_pd(p[0], t)); + + /* float denominator = (q[0] + t * (q[1] + t * (q[2]))) */ + simde__m512d denominator = simde_mm512_fmadd_pd(q[2], t, q[1]); + denominator = simde_mm512_fmadd_pd(denominator, t, q[0]); + + simde__m512d res = simde_mm512_div_pd(numerator, denominator); + + retval = simde_mm512_or_pd(retval, simde_mm512_maskz_mov_pd(mask, res)); + } + } + + { /* else if (a < 0.0f) */ + simde__mmask8 mask = simde_mm512_cmp_pd_mask(a, simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.0)), SIMDE_CMP_LT_OQ); + mask = ~matched & mask; + + if (mask != 0) { + matched = matched | mask; + + /* t = 1/(sqrt(-log(a))) */ + simde__m512d t = simde_x_mm512_negate_pd(simde_mm512_log_pd(a)); + t = simde_mm512_sqrt_pd(t); + t = simde_mm512_div_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(1.0)), t); + + const simde__m512d p[] = { + simde_mm512_set1_pd(SIMDE_FLOAT64_C( 0.00980456202915)), + simde_mm512_set1_pd(SIMDE_FLOAT64_C( 0.36366788917100)), + simde_mm512_set1_pd(SIMDE_FLOAT64_C( 0.97302949837000)), + simde_mm512_set1_pd(SIMDE_FLOAT64_C( -0.5374947401000)) + }; + + const simde__m512d q[] = { + simde_mm512_set1_pd(SIMDE_FLOAT64_C( 0.00980451277802)), + simde_mm512_set1_pd(SIMDE_FLOAT64_C( 0.36369997154400)), + simde_mm512_set1_pd(SIMDE_FLOAT64_C( 1.00000000000000)) + }; + + /* float numerator = (p[0] / t + p[1] + t * (p[2] + t * p[3])) */ + simde__m512d numerator = simde_mm512_fmadd_pd(p[3], t, p[2]); + numerator = simde_mm512_fmadd_pd(numerator, t, p[1]); + numerator = simde_mm512_add_pd(numerator, simde_mm512_div_pd(p[0], t)); + + /* float denominator = (q[0] + t * (q[1] + t * (q[2]))) */ + simde__m512d denominator = simde_mm512_fmadd_pd(q[2], t, q[1]); + denominator = simde_mm512_fmadd_pd(denominator, t, q[0]); + + simde__m512d res = simde_mm512_div_pd(numerator, denominator); + + retval = simde_mm512_or_pd(retval, simde_mm512_maskz_mov_pd(mask, res)); + + if (matched == 1) { + return retval; + } + } + } + + { /* else if (a == 0.0f) */ + simde__mmask8 mask = simde_mm512_cmp_pd_mask(a, simde_mm512_set1_pd(SIMDE_FLOAT64_C(0.0)), SIMDE_CMP_EQ_OQ); + mask = ~matched & mask; + matched = matched | mask; + + simde__m512d res = simde_mm512_set1_pd(SIMDE_MATH_INFINITY); + + retval = simde_mm512_or_pd(retval, simde_mm512_maskz_mov_pd(mask, res)); + } + + { /* else */ + /* (a >= 2.0f) */ + retval = simde_mm512_or_pd(retval, simde_mm512_maskz_mov_pd(~matched, simde_mm512_set1_pd(-SIMDE_MATH_INFINITY))); + } + + return retval; + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_erfcinv_pd + #define _mm512_erfcinv_pd(a) simde_mm512_erfcinv_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_erfcinv_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_erfcinv_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_erfcinv_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_erfcinv_ps + #define _mm512_mask_erfcinv_ps(src, k, a) simde_mm512_mask_erfcinv_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_erfcinv_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_erfcinv_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_erfcinv_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_erfcinv_pd + #define _mm512_mask_erfcinv_pd(src, k, a) simde_mm512_mask_erfcinv_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_logb_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_logb_ps(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_logbf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_logb_ps + #define _mm_logb_ps(a) simde_mm_logb_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_logb_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_logb_pd(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_logb(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_logb_pd + #define _mm_logb_pd(a) simde_mm_logb_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_logb_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_logb_ps(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_logb_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_logbf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_logb_ps + #define _mm256_logb_ps(a) simde_mm256_logb_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_logb_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_logb_pd(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_logb_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_logb(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_logb_pd + #define _mm256_logb_pd(a) simde_mm256_logb_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_logb_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_logb_ps(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_logb_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_logbf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_logb_ps + #define _mm512_logb_ps(a) simde_mm512_logb_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_logb_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_logb_pd(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_logb_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_logb(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_logb_pd + #define _mm512_logb_pd(a) simde_mm512_logb_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_logb_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_logb_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_logb_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_logb_ps + #define _mm512_mask_logb_ps(src, k, a) simde_mm512_mask_logb_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_logb_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_logb_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_logb_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_logb_pd + #define _mm512_mask_logb_pd(src, k, a) simde_mm512_mask_logb_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_log2_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_log2_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_MATH_SLEEF_VERSION_CHECK(3,4,0) && (SIMDE_ACCURACY_PREFERENCE <= 1) + return Sleef_log2f4_u35(a); + #else + return Sleef_log2f4_u10(a); + #endif + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_log2f(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_log2_ps + #define _mm_log2_ps(a) simde_mm_log2_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_log2_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_log2_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_MATH_SLEEF_VERSION_CHECK(3,4,0) && (SIMDE_ACCURACY_PREFERENCE <= 1) + return Sleef_log2d2_u35(a); + #else + return Sleef_log2d2_u10(a); + #endif + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_log2(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_log2_pd + #define _mm_log2_pd(a) simde_mm_log2_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_log2_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_log2_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_MATH_SLEEF_VERSION_CHECK(3,4,0) && (SIMDE_ACCURACY_PREFERENCE <= 1) + return Sleef_log2f8_u35(a); + #else + return Sleef_log2f8_u10(a); + #endif + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_log2_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_log2f(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_log2_ps + #define _mm256_log2_ps(a) simde_mm256_log2_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_log2_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_log2_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_MATH_SLEEF_VERSION_CHECK(3,4,0) && (SIMDE_ACCURACY_PREFERENCE <= 1) + return Sleef_log2d4_u35(a); + #else + return Sleef_log2d4_u10(a); + #endif + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_log2_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_log2(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_log2_pd + #define _mm256_log2_pd(a) simde_mm256_log2_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_log2_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_log2_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_MATH_SLEEF_VERSION_CHECK(3,4,0) && (SIMDE_ACCURACY_PREFERENCE <= 1) + return Sleef_log2f16_u35(a); + #else + return Sleef_log2f16_u10(a); + #endif + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_log2_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_log2f(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_log2_ps + #define _mm512_log2_ps(a) simde_mm512_log2_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_log2_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_log2_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_MATH_SLEEF_VERSION_CHECK(3,4,0) && (SIMDE_ACCURACY_PREFERENCE <= 1) + return Sleef_log2d8_u35(a); + #else + return Sleef_log2d8_u10(a); + #endif + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_log2_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_log2(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_log2_pd + #define _mm512_log2_pd(a) simde_mm512_log2_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_log2_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_log2_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_log2_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_log2_ps + #define _mm512_mask_log2_ps(src, k, a) simde_mm512_mask_log2_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_log2_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_log2_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_log2_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_log2_pd + #define _mm512_mask_log2_pd(src, k, a) simde_mm512_mask_log2_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_log1p_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_log1p_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_log1pf4_u10(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_log1pf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_log1p_ps + #define _mm_log1p_ps(a) simde_mm_log1p_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_log1p_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_log1p_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_log1pd2_u10(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_log1p(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_log1p_pd + #define _mm_log1p_pd(a) simde_mm_log1p_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_log1p_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_log1p_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_log1pf8_u10(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_log1p_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_log1pf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_log1p_ps + #define _mm256_log1p_ps(a) simde_mm256_log1p_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_log1p_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_log1p_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_log1pd4_u10(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_log1p_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_log1p(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_log1p_pd + #define _mm256_log1p_pd(a) simde_mm256_log1p_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_log1p_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_log1p_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_log1pf16_u10(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_log1p_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_log1pf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_log1p_ps + #define _mm512_log1p_ps(a) simde_mm512_log1p_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_log1p_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_log1p_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_log1pd8_u10(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_log1p_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_log1p(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_log1p_pd + #define _mm512_log1p_pd(a) simde_mm512_log1p_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_log1p_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_log1p_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_log1p_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_log1p_ps + #define _mm512_mask_log1p_ps(src, k, a) simde_mm512_mask_log1p_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_log1p_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_log1p_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_log1p_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_log1p_pd + #define _mm512_mask_log1p_pd(src, k, a) simde_mm512_mask_log1p_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_log10_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_log10_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_log10f4_u10(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_log10f(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_log10_ps + #define _mm_log10_ps(a) simde_mm_log10_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_log10_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_log10_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_log10d2_u10(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_log10(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_log10_pd + #define _mm_log10_pd(a) simde_mm_log10_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_log10_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_log10_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_log10f8_u10(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_log10_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_log10f(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_log10_ps + #define _mm256_log10_ps(a) simde_mm256_log10_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_log10_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_log10_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_log10d4_u10(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_log10_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_log10(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_log10_pd + #define _mm256_log10_pd(a) simde_mm256_log10_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_log10_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_log10_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_log10f16_u10(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_log10_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_log10f(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_log10_ps + #define _mm512_log10_ps(a) simde_mm512_log10_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_log10_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_log10_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_log10d8_u10(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_log10_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_log10(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_log10_pd + #define _mm512_log10_pd(a) simde_mm512_log10_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_log10_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_log10_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_log10_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_log10_ps + #define _mm512_mask_log10_ps(src, k, a) simde_mm512_mask_log10_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_log10_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_log10_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_log10_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_log10_pd + #define _mm512_mask_log10_pd(src, k, a) simde_mm512_mask_log10_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_nearbyint_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_nearbyint_ps(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_nearbyintf(a_.f32[i]); + } + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_nearbyint_ps + #define _mm512_nearbyint_ps(a) simde_mm512_nearbyint_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_nearbyint_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_nearbyint_pd(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_nearbyint(a_.f64[i]); + } + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_nearbyint_pd + #define _mm512_nearbyint_pd(a) simde_mm512_nearbyint_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_nearbyint_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_nearbyint_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_nearbyint_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_nearbyint_ps + #define _mm512_mask_nearbyint_ps(src, k, a) simde_mm512_mask_nearbyint_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_nearbyint_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_nearbyint_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_nearbyint_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_nearbyint_pd + #define _mm512_mask_nearbyint_pd(src, k, a) simde_mm512_mask_nearbyint_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_pow_ps (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_pow_ps(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_powf4_u10(a, b); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_powf(a_.f32[i], b_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_pow_ps + #define _mm_pow_ps(a, b) simde_mm_pow_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_pow_pd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_pow_pd(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_powd2_u10(a, b); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_pow(a_.f64[i], b_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_pow_pd + #define _mm_pow_pd(a, b) simde_mm_pow_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_pow_ps (simde__m256 a, simde__m256 b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_pow_ps(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_powf8_u10(a, b); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_powf(a_.f32[i], b_.f32[i]); + } + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_pow_ps + #define _mm256_pow_ps(a, b) simde_mm256_pow_ps(a, b) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_pow_pd (simde__m256d a, simde__m256d b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_pow_pd(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_powd4_u10(a, b); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_pow(a_.f64[i], b_.f64[i]); + } + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_pow_pd + #define _mm256_pow_pd(a, b) simde_mm256_pow_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_pow_ps (simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_pow_ps(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_powf16_u10(a, b); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a), + b_ = simde__m512_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_powf(a_.f32[i], b_.f32[i]); + } + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_pow_ps + #define _mm512_pow_ps(a, b) simde_mm512_pow_ps(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_pow_pd (simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_pow_pd(a, b); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_powd8_u10(a, b); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a), + b_ = simde__m512d_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_pow(a_.f64[i], b_.f64[i]); + } + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_pow_pd + #define _mm512_pow_pd(a, b) simde_mm512_pow_pd(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_pow_ps(simde__m512 src, simde__mmask16 k, simde__m512 a, simde__m512 b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_pow_ps(src, k, a, b); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_pow_ps(a, b)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_pow_ps + #define _mm512_mask_pow_ps(src, k, a, b) simde_mm512_mask_pow_ps(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_pow_pd(simde__m512d src, simde__mmask8 k, simde__m512d a, simde__m512d b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_pow_pd(src, k, a, b); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_pow_pd(a, b)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_pow_pd + #define _mm512_mask_pow_pd(src, k, a, b) simde_mm512_mask_pow_pd(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_clog_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_clog_ps(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + simde__m128_private pow_res_ = simde__m128_to_private(simde_mm_pow_ps(a, simde_mm_set1_ps(SIMDE_FLOAT32_C(2.0)))); + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i += 2) { + r_.f32[ i ] = simde_math_logf(simde_math_sqrtf(pow_res_.f32[i] + pow_res_.f32[i+1])); + r_.f32[i + 1] = simde_math_atan2f(a_.f32[i + 1], a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_clog_ps + #define _mm_clog_ps(a) simde_mm_clog_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_clog_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_clog_ps(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + simde__m256_private pow_res_ = simde__m256_to_private(simde_mm256_pow_ps(a, simde_mm256_set1_ps(SIMDE_FLOAT32_C(2.0)))); + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i += 2) { + r_.f32[ i ] = simde_math_logf(simde_math_sqrtf(pow_res_.f32[i] + pow_res_.f32[i + 1])); + r_.f32[i + 1] = simde_math_atan2f(a_.f32[i + 1], a_.f32[i]); + } + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_clog_ps + #define _mm256_clog_ps(a) simde_mm256_clog_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_csqrt_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_csqrt_ps(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + simde__m128 pow_res= simde_mm_pow_ps(a,simde_mm_set1_ps(SIMDE_FLOAT32_C(2.0))); + simde__m128_private pow_res_=simde__m128_to_private(pow_res); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i+=2) { + simde_float32 sign = simde_math_copysignf(SIMDE_FLOAT32_C(1.0), a_.f32[i + 1]); + simde_float32 temp = simde_math_sqrtf(pow_res_.f32[i] + pow_res_.f32[i+1]); + + r_.f32[ i ] = simde_math_sqrtf(( a_.f32[i] + temp) / SIMDE_FLOAT32_C(2.0)); + r_.f32[i + 1] = sign * simde_math_sqrtf((-a_.f32[i] + temp) / SIMDE_FLOAT32_C(2.0)); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_csqrt_ps + #define _mm_csqrt_ps(a) simde_mm_csqrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_csqrt_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_csqrt_ps(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + simde__m256 pow_res= simde_mm256_pow_ps(a,simde_mm256_set1_ps(SIMDE_FLOAT32_C(2.0))); + simde__m256_private pow_res_=simde__m256_to_private(pow_res); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i+=2) { + simde_float32 sign = simde_math_copysignf(SIMDE_FLOAT32_C(1.0), a_.f32[i + 1]); + simde_float32 temp = simde_math_sqrtf(pow_res_.f32[i] + pow_res_.f32[i+1]); + + r_.f32[ i ] = simde_math_sqrtf(( a_.f32[i] + temp) / SIMDE_FLOAT32_C(2.0)); + r_.f32[i + 1] = sign * simde_math_sqrtf((-a_.f32[i] + temp) / SIMDE_FLOAT32_C(2.0)); + } + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_csqrt_ps + #define _mm256_csqrt_ps(a) simde_mm256_csqrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rem_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_rem_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.i8 = a_.i8 % b_.i8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] % b_.i8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_rem_epi8 + #define _mm_rem_epi8(a, b) simde_mm_rem_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rem_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_rem_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.i16 = a_.i16 % b_.i16; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] % b_.i16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_rem_epi16 + #define _mm_rem_epi16(a, b) simde_mm_rem_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rem_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_rem_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.i32 = a_.i32 % b_.i32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] % b_.i32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#define simde_mm_irem_epi32(a, b) simde_mm_rem_epi32(a, b) +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_rem_epi32 + #define _mm_rem_epi32(a, b) simde_mm_rem_epi32(a, b) + #undef _mm_irem_epi32 + #define _mm_irem_epi32(a, b) simde_mm_rem_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rem_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_rem_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.i64 = a_.i64 % b_.i64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] % b_.i64[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_rem_epi64 + #define _mm_rem_epi64(a, b) simde_mm_rem_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rem_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_rem_epu8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.u8 = a_.u8 % b_.u8; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = a_.u8[i] % b_.u8[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_rem_epu8 + #define _mm_rem_epu8(a, b) simde_mm_rem_epu8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rem_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_rem_epu16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.u16 = a_.u16 % b_.u16; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = a_.u16[i] % b_.u16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_rem_epu16 + #define _mm_rem_epu16(a, b) simde_mm_rem_epu16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rem_epu32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_rem_epu32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.u32 = a_.u32 % b_.u32; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] % b_.u32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#define simde_mm_urem_epi32(a, b) simde_mm_rem_epu32(a, b) +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_rem_epu32 + #define _mm_rem_epu32(a, b) simde_mm_rem_epu32(a, b) + #undef _mm_urem_epi32 + #define _mm_urem_epi32(a, b) simde_mm_rem_epu32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rem_epu64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_rem_epu64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.u64 = a_.u64 % b_.u64; + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = a_.u64[i] % b_.u64[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_rem_epu64 + #define _mm_rem_epu64(a, b) simde_mm_rem_epu64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_rem_epi8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_rem_epi8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.i8 = a_.i8 % b_.i8; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_rem_epi8(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] % b_.i8[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_rem_epi8 + #define _mm256_rem_epi8(a, b) simde_mm256_rem_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_rem_epi16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_rem_epi16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.i16 = a_.i16 % b_.i16; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_rem_epi16(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] % b_.i16[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_rem_epi16 + #define _mm256_rem_epi16(a, b) simde_mm256_rem_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_rem_epi32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_rem_epi32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.i32 = a_.i32 % b_.i32; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_rem_epi32(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] % b_.i32[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#define simde_mm256_irem_epi32(a, b) simde_mm256_rem_epi32(a, b) +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_rem_epi32 + #define _mm256_rem_epi32(a, b) simde_mm256_rem_epi32(a, b) + #undef _mm256_irem_epi32 + #define _mm256_irem_epi32(a, b) simde_mm256_rem_epi32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_rem_epi64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_rem_epi64(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.i64 = a_.i64 % b_.i64; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_rem_epi64(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] % b_.i64[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_rem_epi64 + #define _mm256_rem_epi64(a, b) simde_mm256_rem_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_rem_epu8 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_rem_epu8(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.u8 = a_.u8 % b_.u8; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_rem_epu8(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = a_.u8[i] % b_.u8[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_rem_epu8 + #define _mm256_rem_epu8(a, b) simde_mm256_rem_epu8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_rem_epu16 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_rem_epu16(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.u16 = a_.u16 % b_.u16; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_rem_epu16(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = a_.u16[i] % b_.u16[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_rem_epu16 + #define _mm256_rem_epu16(a, b) simde_mm256_rem_epu16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_rem_epu32 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_rem_epu32(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.u32 = a_.u32 % b_.u32; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_rem_epu32(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] % b_.u32[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#define simde_mm256_urem_epi32(a, b) simde_mm256_rem_epu32(a, b) +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_rem_epu32 + #define _mm256_rem_epu32(a, b) simde_mm256_rem_epu32(a, b) + #undef _mm256_urem_epi32 + #define _mm256_urem_epi32(a, b) simde_mm256_rem_epu32(a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_rem_epu64 (simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_rem_epu64(a, b); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.u64 = a_.u64 % b_.u64; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_rem_epu64(a_.m128i[i], b_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = a_.u64[i] % b_.u64[i]; + } + #endif + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_rem_epu64 + #define _mm256_rem_epu64(a, b) simde_mm256_rem_epu64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_rem_epi8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rem_epi8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.i8 = a_.i8 % b_.i8; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_rem_epi8(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = a_.i8[i] % b_.i8[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_rem_epi8 + #define _mm512_rem_epi8(a, b) simde_mm512_rem_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_rem_epi16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rem_epi16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.i16 = a_.i16 % b_.i16; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_rem_epi16(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = a_.i16[i] % b_.i16[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_rem_epi16 + #define _mm512_rem_epi16(a, b) simde_mm512_rem_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_rem_epi32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rem_epi32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.i32 = a_.i32 % b_.i32; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_rem_epi32(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = a_.i32[i] % b_.i32[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_rem_epi32 + #define _mm512_rem_epi32(a, b) simde_mm512_rem_epi32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_rem_epi32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_rem_epi32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_rem_epi32(a, b)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_rem_epi32 + #define _mm512_mask_rem_epi32(src, k, a, b) simde_mm512_mask_rem_epi32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_rem_epi64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rem_epi64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.i64 = a_.i64 % b_.i64; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_rem_epi64(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = a_.i64[i] % b_.i64[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_rem_epi64 + #define _mm512_rem_epi64(a, b) simde_mm512_rem_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_rem_epu8 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rem_epu8(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.u8 = a_.u8 % b_.u8; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_rem_epu8(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = a_.u8[i] % b_.u8[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_rem_epu8 + #define _mm512_rem_epu8(a, b) simde_mm512_rem_epu8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_rem_epu16 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rem_epu16(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.u16 = a_.u16 % b_.u16; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_rem_epu16(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = a_.u16[i] % b_.u16[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_rem_epu16 + #define _mm512_rem_epu16(a, b) simde_mm512_rem_epu16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_rem_epu32 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rem_epu32(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.u32 = a_.u32 % b_.u32; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_rem_epu32(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = a_.u32[i] % b_.u32[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_rem_epu32 + #define _mm512_rem_epu32(a, b) simde_mm512_rem_epu32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_mask_rem_epu32(simde__m512i src, simde__mmask16 k, simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_rem_epu32(src, k, a, b); + #else + return simde_mm512_mask_mov_epi32(src, k, simde_mm512_rem_epu32(a, b)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_rem_epu32 + #define _mm512_mask_rem_epu32(src, k, a, b) simde_mm512_mask_rem_epu32(src, k, a, b) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512i +simde_mm512_rem_epu64 (simde__m512i a, simde__m512i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rem_epu64(a, b); + #else + simde__m512i_private + r_, + a_ = simde__m512i_to_private(a), + b_ = simde__m512i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) && !defined(SIMDE_BUG_PGI_30104) + r_.u64 = a_.u64 % b_.u64; + #else + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256i) / sizeof(r_.m256i[0])) ; i++) { + r_.m256i[i] = simde_mm256_rem_epu64(a_.m256i[i], b_.m256i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = a_.u64[i] % b_.u64[i]; + } + #endif + #endif + + return simde__m512i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_rem_epu64 + #define _mm512_rem_epu64(a, b) simde_mm512_rem_epu64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_recip_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_recip_ps(a); + #else + return simde_mm512_div_ps(simde_mm512_set1_ps(SIMDE_FLOAT32_C(1.0)), a); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_recip_ps + #define _mm512_recip_ps(a) simde_mm512_recip_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_recip_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_recip_pd(a); + #else + return simde_mm512_div_pd(simde_mm512_set1_pd(SIMDE_FLOAT64_C(1.0)), a); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_recip_pd + #define _mm512_recip_pd(a) simde_mm512_recip_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_recip_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_recip_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_recip_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_recip_ps + #define _mm512_mask_recip_ps(src, k, a) simde_mm512_mask_recip_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_recip_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_recip_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_recip_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_recip_pd + #define _mm512_mask_recip_pd(src, k, a) simde_mm512_mask_recip_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_rint_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rint_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_rintf16(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_rintf(a_.f32[i]); + } + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_rint_ps + #define _mm512_rint_ps(a) simde_mm512_rint_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_rint_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_rint_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_rintd8(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_rint(a_.f64[i]); + } + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_rint_pd + #define _mm512_rint_pd(a) simde_mm512_rint_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_rint_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_rint_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_rint_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_rint_ps + #define _mm512_mask_rint_ps(src, k, a) simde_mm512_mask_rint_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_rint_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_rint_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_rint_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_rint_pd + #define _mm512_mask_rint_pd(src, k, a) simde_mm512_mask_rint_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_sin_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_sin_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_sinf4_u10(a); + #else + return Sleef_sinf4_u35(a); + #endif + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_sinf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_sin_ps + #define _mm_sin_ps(a) simde_mm_sin_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_sin_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_sin_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_sind2_u10(a); + #else + return Sleef_sind2_u35(a); + #endif + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_sin(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_sin_pd + #define _mm_sin_pd(a) simde_mm_sin_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_sin_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_sin_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_sinf8_u10(a); + #else + return Sleef_sinf8_u35(a); + #endif + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_sin_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_sinf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_sin_ps + #define _mm256_sin_ps(a) simde_mm256_sin_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_sin_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_sin_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_sind4_u10(a); + #else + return Sleef_sind4_u35(a); + #endif + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_sin_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_sin(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_sin_pd + #define _mm256_sin_pd(a) simde_mm256_sin_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_sin_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sin_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_sinf16_u10(a); + #else + return Sleef_sinf16_u35(a); + #endif + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_sin_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_sinf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_sin_ps + #define _mm512_sin_ps(a) simde_mm512_sin_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_sin_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sin_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_sind8_u10(a); + #else + return Sleef_sind8_u35(a); + #endif + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_sin_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_sin(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_sin_pd + #define _mm512_sin_pd(a) simde_mm512_sin_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_sin_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sin_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_sin_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sin_ps + #define _mm512_mask_sin_ps(src, k, a) simde_mm512_mask_sin_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_sin_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sin_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_sin_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sin_pd + #define _mm512_mask_sin_pd(src, k, a) simde_mm512_mask_sin_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_sincos_ps (simde__m128* mem_addr, simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_sincos_ps(HEDLEY_REINTERPRET_CAST(__m128*, mem_addr), a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + Sleef___m128_2 temp; + + #if SIMDE_ACCURACY_PREFERENCE > 1 + temp = Sleef_sincosf4_u10(a); + #else + temp = Sleef_sincosf4_u35(a); + #endif + + *mem_addr = temp.y; + return temp.x; + #else + simde__m128 r; + + r = simde_mm_sin_ps(a); + *mem_addr = simde_mm_cos_ps(a); + + return r; + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_sincos_ps + #define _mm_sincos_ps(mem_addr, a) simde_mm_sincos_ps((mem_addr),(a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_sincos_pd (simde__m128d* mem_addr, simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_sincos_pd(HEDLEY_REINTERPRET_CAST(__m128d*, mem_addr), a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + Sleef___m128d_2 temp; + + #if SIMDE_ACCURACY_PREFERENCE > 1 + temp = Sleef_sincosd2_u10(a); + #else + temp = Sleef_sincosd2_u35(a); + #endif + + *mem_addr = temp.y; + return temp.x; + #else + simde__m128d r; + + r = simde_mm_sin_pd(a); + *mem_addr = simde_mm_cos_pd(a); + + return r; + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_sincos_pd + #define _mm_sincos_pd(mem_addr, a) simde_mm_sincos_pd((mem_addr),(a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_sincos_ps (simde__m256* mem_addr, simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_sincos_ps(HEDLEY_REINTERPRET_CAST(__m256*, mem_addr), a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + Sleef___m256_2 temp; + + #if SIMDE_ACCURACY_PREFERENCE > 1 + temp = Sleef_sincosf8_u10(a); + #else + temp = Sleef_sincosf8_u35(a); + #endif + + *mem_addr = temp.y; + return temp.x; + #else + simde__m256 r; + + r = simde_mm256_sin_ps(a); + *mem_addr = simde_mm256_cos_ps(a); + + return r; + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_sincos_ps + #define _mm256_sincos_ps(mem_addr, a) simde_mm256_sincos_ps((mem_addr),(a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_sincos_pd (simde__m256d* mem_addr, simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_sincos_pd(HEDLEY_REINTERPRET_CAST(__m256d*, mem_addr), a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + Sleef___m256d_2 temp; + + #if SIMDE_ACCURACY_PREFERENCE > 1 + temp = Sleef_sincosd4_u10(a); + #else + temp = Sleef_sincosd4_u35(a); + #endif + + *mem_addr = temp.y; + return temp.x; + #else + simde__m256d r; + + r = simde_mm256_sin_pd(a); + *mem_addr = simde_mm256_cos_pd(a); + + return r; + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_sincos_pd + #define _mm256_sincos_pd(mem_addr, a) simde_mm256_sincos_pd((mem_addr),(a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_sincos_ps (simde__m512* mem_addr, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sincos_ps(HEDLEY_REINTERPRET_CAST(__m512*, mem_addr), a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + Sleef___m512_2 temp; + + #if SIMDE_ACCURACY_PREFERENCE > 1 + temp = Sleef_sincosf16_u10(a); + #else + temp = Sleef_sincosf16_u35(a); + #endif + + *mem_addr = temp.y; + return temp.x; + #else + simde__m512 r; + + r = simde_mm512_sin_ps(a); + *mem_addr = simde_mm512_cos_ps(a); + + return r; + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_sincos_ps + #define _mm512_sincos_ps(mem_addr, a) simde_mm512_sincos_ps((mem_addr),(a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_sincos_pd (simde__m512d* mem_addr, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sincos_pd(HEDLEY_REINTERPRET_CAST(__m512d*, mem_addr), a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + Sleef___m512d_2 temp; + + #if SIMDE_ACCURACY_PREFERENCE > 1 + temp = Sleef_sincosd8_u10(a); + #else + temp = Sleef_sincosd8_u35(a); + #endif + + *mem_addr = temp.y; + return temp.x; + #else + simde__m512d r; + + r = simde_mm512_sin_pd(a); + *mem_addr = simde_mm512_cos_pd(a); + + return r; + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_sincos_pd + #define _mm512_sincos_pd(mem_addr, a) simde_mm512_sincos_pd((mem_addr),(a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_sincos_ps(simde__m512* mem_addr, simde__m512 sin_src, simde__m512 cos_src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sincos_ps(mem_addr, sin_src, cos_src, k, a); + #else + simde__m512 cos_res, sin_res; + sin_res = simde_mm512_sincos_ps(&cos_res, a); + *mem_addr = simde_mm512_mask_mov_ps(cos_src, k, cos_res); + return simde_mm512_mask_mov_ps(sin_src, k, sin_res); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sincos_ps + #define _mm512_mask_sincos_ps(mem_addr, sin_src, cos_src, k, a) simde_mm512_mask_sincos_ps(mem_addr, sin_src, cos_src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_sincos_pd(simde__m512d* mem_addr, simde__m512d sin_src, simde__m512d cos_src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sincos_pd(mem_addr, sin_src, cos_src, k, a); + #else + simde__m512d cos_res, sin_res; + sin_res = simde_mm512_sincos_pd(&cos_res, a); + *mem_addr = simde_mm512_mask_mov_pd(cos_src, k, cos_res); + return simde_mm512_mask_mov_pd(sin_src, k, sin_res); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sincos_pd + #define _mm512_mask_sincos_pd(mem_addr, sin_src, cos_src, k, a) simde_mm512_mask_sincos_pd(mem_addr, sin_src, cos_src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_sind_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_sind_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_sinf4_u10(simde_x_mm_deg2rad_ps(a)); + #else + return Sleef_sinf4_u35(simde_x_mm_deg2rad_ps(a)); + #endif + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_sinf(simde_math_deg2radf(a_.f32[i])); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_sind_ps + #define _mm_sind_ps(a) simde_mm_sind_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_sind_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_sind_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_sind2_u10(simde_x_mm_deg2rad_pd(a)); + #else + return Sleef_sind2_u35(simde_x_mm_deg2rad_pd(a)); + #endif + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_sin(simde_math_deg2rad(a_.f64[i])); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_sind_pd + #define _mm_sind_pd(a) simde_mm_sind_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_sind_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_sind_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_sinf8_u10(simde_x_mm256_deg2rad_ps(a)); + #else + return Sleef_sinf8_u35(simde_x_mm256_deg2rad_ps(a)); + #endif + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_sind_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_sinf(simde_math_deg2radf(a_.f32[i])); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_sind_ps + #define _mm256_sind_ps(a) simde_mm256_sind_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_sind_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_sind_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_sind4_u10(simde_x_mm256_deg2rad_pd(a)); + #else + return Sleef_sind4_u35(simde_x_mm256_deg2rad_pd(a)); + #endif + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_sind_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_sin(simde_math_deg2rad(a_.f64[i])); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_sind_pd + #define _mm256_sind_pd(a) simde_mm256_sind_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_sind_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sind_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_sinf16_u10(simde_x_mm512_deg2rad_ps(a)); + #else + return Sleef_sinf16_u35(simde_x_mm512_deg2rad_ps(a)); + #endif + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_sind_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_sinf(simde_math_deg2radf(a_.f32[i])); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_sind_ps + #define _mm512_sind_ps(a) simde_mm512_sind_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_sind_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sind_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_sind8_u10(simde_x_mm512_deg2rad_pd(a)); + #else + return Sleef_sind8_u35(simde_x_mm512_deg2rad_pd(a)); + #endif + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_sind_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_sin(simde_math_deg2rad(a_.f64[i])); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_sind_pd + #define _mm512_sind_pd(a) simde_mm512_sind_pd(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_sind_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sind_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_sind_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sind_ps + #define _mm512_mask_sind_ps(src, k, a) simde_mm512_mask_sind_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_sind_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sind_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_sind_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sind_pd + #define _mm512_mask_sind_pd(src, k, a) simde_mm512_mask_sind_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_sinh_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_sinh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_sinhf4_u10(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_sinhf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_sinh_ps + #define _mm_sinh_ps(a) simde_mm_sinh_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_sinh_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_sinh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_sinhd2_u10(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_sinh(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_sinh_pd + #define _mm_sinh_pd(a) simde_mm_sinh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_sinh_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_sinh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_sinhf8_u10(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_sinh_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_sinhf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_sinh_ps + #define _mm256_sinh_ps(a) simde_mm256_sinh_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_sinh_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_sinh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_sinhd4_u10(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_sinh_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_sinh(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_sinh_pd + #define _mm256_sinh_pd(a) simde_mm256_sinh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_sinh_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sinh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_sinhf16_u10(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_sinh_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_sinhf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_sinh_ps + #define _mm512_sinh_ps(a) simde_mm512_sinh_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_sinh_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_sinh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_sinhd8_u10(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_sinh_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_sinh(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_sinh_pd + #define _mm512_sinh_pd(a) simde_mm512_sinh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_sinh_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sinh_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_sinh_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sinh_ps + #define _mm512_mask_sinh_ps(src, k, a) simde_mm512_mask_sinh_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_sinh_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_sinh_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_sinh_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_sinh_pd + #define _mm512_mask_sinh_pd(src, k, a) simde_mm512_mask_sinh_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_svml_ceil_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_svml_ceil_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_ceilf4(a); + #else + return simde_mm_round_ps(a, SIMDE_MM_FROUND_TO_POS_INF); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_svml_ceil_ps + #define _mm_svml_ceil_ps(a) simde_mm_svml_ceil_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_svml_ceil_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_svml_ceil_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_ceild2(a); + #else + return simde_mm_round_pd(a, SIMDE_MM_FROUND_TO_POS_INF); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_svml_ceil_pd + #define _mm_svml_ceil_pd(a) simde_mm_svml_ceil_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_svml_ceil_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_svml_ceil_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_ceilf8(a); + #else + return simde_mm256_round_ps(a, SIMDE_MM_FROUND_TO_POS_INF); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_svml_ceil_ps + #define _mm256_svml_ceil_ps(a) simde_mm256_svml_ceil_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_svml_ceil_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_svml_ceil_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_ceild4(a); + #else + return simde_mm256_round_pd(a, SIMDE_MM_FROUND_TO_POS_INF); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_svml_ceil_pd + #define _mm256_svml_ceil_pd(a) simde_mm256_svml_ceil_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_ceil_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_ceil_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_ceilf16(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_ceil_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_ceilf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_ceil_ps + #define _mm512_ceil_ps(a) simde_mm512_ceil_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_ceil_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_ceil_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_ceild8(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_ceil_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_ceil(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_ceil_pd + #define _mm512_ceil_pd(a) simde_mm512_ceil_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_ceil_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_ceil_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_ceil_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_ceil_ps + #define _mm512_mask_ceil_ps(src, k, a) simde_mm512_mask_ceil_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_ceil_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_ceil_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_ceil_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_ceil_pd + #define _mm512_mask_ceil_pd(src, k, a) simde_mm512_mask_ceil_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_svml_floor_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_svml_floor_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_floorf4(a); + #else + return simde_mm_round_ps(a, SIMDE_MM_FROUND_TO_NEG_INF); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_svml_floor_ps + #define _mm_svml_floor_ps(a) simde_mm_svml_floor_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_svml_floor_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_svml_floor_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_floord2(a); + #else + return simde_mm_round_pd(a, SIMDE_MM_FROUND_TO_NEG_INF); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_svml_floor_pd + #define _mm_svml_floor_pd(a) simde_mm_svml_floor_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_svml_floor_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_svml_floor_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_floorf8(a); + #else + return simde_mm256_round_ps(a, SIMDE_MM_FROUND_TO_NEG_INF); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_svml_floor_ps + #define _mm256_svml_floor_ps(a) simde_mm256_svml_floor_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_svml_floor_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_svml_floor_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_floord4(a); + #else + return simde_mm256_round_pd(a, SIMDE_MM_FROUND_TO_NEG_INF); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_svml_floor_pd + #define _mm256_svml_floor_pd(a) simde_mm256_svml_floor_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_floor_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_floor_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_floorf16(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_floor_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_floorf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_floor_ps + #define _mm512_floor_ps(a) simde_mm512_floor_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_floor_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_floor_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_floord8(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_floor_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_floor(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_floor_pd + #define _mm512_floor_pd(a) simde_mm512_floor_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_floor_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_floor_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_floor_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_floor_ps + #define _mm512_mask_floor_ps(src, k, a) simde_mm512_mask_floor_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_floor_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_floor_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_floor_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_floor_pd + #define _mm512_mask_floor_pd(src, k, a) simde_mm512_mask_floor_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_svml_round_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_svml_round_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_roundf4(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_roundf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_svml_round_ps + #define _mm_svml_round_ps(a) simde_mm_svml_round_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_svml_round_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_svml_round_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_roundd2(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_round(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_svml_round_pd + #define _mm_svml_round_pd(a) simde_mm_svml_round_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_svml_round_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_svml_round_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_roundf8(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_svml_round_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_roundf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_svml_round_ps + #define _mm256_svml_round_ps(a) simde_mm256_svml_round_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_svml_round_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_svml_round_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_roundd4(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_svml_round_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_round(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_svml_round_pd + #define _mm256_svml_round_pd(a) simde_mm256_svml_round_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_svml_round_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_svml_round_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_roundd8(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_svml_round_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_round(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_svml_round_pd + #define _mm512_svml_round_pd(a) simde_mm512_svml_round_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_svml_round_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_svml_round_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_svml_round_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_svml_round_pd + #define _mm512_mask_svml_round_pd(src, k, a) simde_mm512_mask_svml_round_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_svml_sqrt_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_svml_sqrt_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_sqrtf4(a); + #else + return simde_mm_sqrt_ps(a); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_svml_sqrt_ps + #define _mm_svml_sqrt_ps(a) simde_mm_svml_sqrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_svml_sqrt_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_svml_sqrt_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_sqrtd2(a); + #else + return simde_mm_sqrt_pd(a); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_svml_sqrt_pd + #define _mm_svml_sqrt_pd(a) simde_mm_svml_sqrt_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_svml_sqrt_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_svml_sqrt_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_sqrtf8(a); + #else + return simde_mm256_sqrt_ps(a); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_svml_sqrt_ps + #define _mm256_svml_sqrt_ps(a) simde_mm256_svml_sqrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_svml_sqrt_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_svml_sqrt_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_sqrtd4(a); + #else + return simde_mm256_sqrt_pd(a); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_svml_sqrt_pd + #define _mm256_svml_sqrt_pd(a) simde_mm256_svml_sqrt_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_svml_sqrt_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_svml_sqrt_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_sqrtf16(a); + #else + return simde_mm512_sqrt_ps(a); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_svml_sqrt_ps + #define _mm512_svml_sqrt_ps(a) simde_mm512_svml_sqrt_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_svml_sqrt_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_svml_sqrt_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_sqrtd8(a); + #else + return simde_mm512_sqrt_pd(a); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_svml_sqrt_pd + #define _mm512_svml_sqrt_pd(a) simde_mm512_svml_sqrt_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_tan_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_tan_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_tanf4_u10(a); + #else + return Sleef_tanf4_u35(a); + #endif + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_tanf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_tan_ps + #define _mm_tan_ps(a) simde_mm_tan_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_tan_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_tan_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_tand2_u10(a); + #else + return Sleef_tand2_u35(a); + #endif + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_tan(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_tan_pd + #define _mm_tan_pd(a) simde_mm_tan_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_tan_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_tan_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_tanf8_u10(a); + #else + return Sleef_tanf8_u35(a); + #endif + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_tan_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_tanf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_tan_ps + #define _mm256_tan_ps(a) simde_mm256_tan_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_tan_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_tan_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_tand4_u10(a); + #else + return Sleef_tand4_u35(a); + #endif + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_tan_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_tan(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_tan_pd + #define _mm256_tan_pd(a) simde_mm256_tan_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_tan_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_tan_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_tanf16_u10(a); + #else + return Sleef_tanf16_u35(a); + #endif + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_tan_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_tanf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_tan_ps + #define _mm512_tan_ps(a) simde_mm512_tan_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_tan_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_tan_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_tand8_u10(a); + #else + return Sleef_tand8_u35(a); + #endif + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_tan_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_tan(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_tan_pd + #define _mm512_tan_pd(a) simde_mm512_tan_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_tan_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_tan_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_tan_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_tan_ps + #define _mm512_mask_tan_ps(src, k, a) simde_mm512_mask_tan_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_tan_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_tan_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_tan_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_tan_pd + #define _mm512_mask_tan_pd(src, k, a) simde_mm512_mask_tan_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_tand_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_tand_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_tanf4_u10(simde_x_mm_deg2rad_ps(a)); + #else + return Sleef_tanf4_u35(simde_x_mm_deg2rad_ps(a)); + #endif + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_tanf(simde_math_deg2radf(a_.f32[i])); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_tand_ps + #define _mm_tand_ps(a) simde_mm_tand_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_tand_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_tand_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_tand2_u10(simde_x_mm_deg2rad_pd(a)); + #else + return Sleef_tand2_u35(simde_x_mm_deg2rad_pd(a)); + #endif + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_tan(simde_math_deg2rad(a_.f64[i])); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_tand_pd + #define _mm_tand_pd(a) simde_mm_tand_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_tand_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_tand_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_tanf8_u10(simde_x_mm256_deg2rad_ps(a)); + #else + return Sleef_tanf8_u35(simde_x_mm256_deg2rad_ps(a)); + #endif + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_tand_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_tanf(simde_math_deg2radf(a_.f32[i])); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_tand_ps + #define _mm256_tand_ps(a) simde_mm256_tand_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_tand_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_tand_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_tand4_u10(simde_x_mm256_deg2rad_pd(a)); + #else + return Sleef_tand4_u35(simde_x_mm256_deg2rad_pd(a)); + #endif + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_tand_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_tan(simde_math_deg2rad(a_.f64[i])); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_tand_pd + #define _mm256_tand_pd(a) simde_mm256_tand_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_tand_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_tand_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_tanf16_u10(simde_x_mm512_deg2rad_ps(a)); + #else + return Sleef_tanf16_u35(simde_x_mm512_deg2rad_ps(a)); + #endif + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_tand_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_tanf(simde_math_deg2radf(a_.f32[i])); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_tand_ps + #define _mm512_tand_ps(a) simde_mm512_tand_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_tand_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_tand_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + #if SIMDE_ACCURACY_PREFERENCE > 1 + return Sleef_tand8_u10(simde_x_mm512_deg2rad_pd(a)); + #else + return Sleef_tand8_u35(simde_x_mm512_deg2rad_pd(a)); + #endif + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_tand_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_tan(simde_math_deg2rad(a_.f64[i])); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_tand_pd + #define _mm512_tand_pd(a) simde_mm512_tand_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_tand_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_tand_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_tand_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_tand_ps + #define _mm512_mask_tand_ps(src, k, a) simde_mm512_mask_tand_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_tand_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_tand_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_tand_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_tand_pd + #define _mm512_mask_tand_pd(src, k, a) simde_mm512_mask_tand_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_tanh_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_tanh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_tanhf4_u10(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_tanhf(a_.f32[i]); + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_tanh_ps + #define _mm_tanh_ps(a) simde_mm_tanh_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_tanh_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_tanh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_tanhd2_u10(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_tanh(a_.f64[i]); + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_tanh_pd + #define _mm_tanh_pd(a) simde_mm_tanh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_tanh_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_tanh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_tanhf8_u10(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_tanh_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_tanhf(a_.f32[i]); + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_tanh_ps + #define _mm256_tanh_ps(a) simde_mm256_tanh_ps(a) +#endif + + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_tanh_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_tanh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_tanhd4_u10(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_tanh_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_tanh(a_.f64[i]); + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_tanh_pd + #define _mm256_tanh_pd(a) simde_mm256_tanh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_tanh_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_tanh_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_tanhf16_u10(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_tanh_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_tanhf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_tanh_ps + #define _mm512_tanh_ps(a) simde_mm512_tanh_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_tanh_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_tanh_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_tanhd8_u10(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_tanh_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_tanh(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_tanh_pd + #define _mm512_tanh_pd(a) simde_mm512_tanh_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_tanh_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_tanh_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_tanh_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_tanh_ps + #define _mm512_mask_tanh_ps(src, k, a) simde_mm512_mask_tanh_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_tanh_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_tanh_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_tanh_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_tanh_pd + #define _mm512_mask_tanh_pd(src, k, a) simde_mm512_mask_tanh_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_trunc_ps (simde__m128 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_trunc_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_truncf4(a); + #else + return simde_mm_round_ps(a, SIMDE_MM_FROUND_TO_ZERO); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_trunc_ps + #define _mm_trunc_ps(a) simde_mm_trunc_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_trunc_pd (simde__m128d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_trunc_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_SSE_NATIVE) + return Sleef_truncd2(a); + #else + return simde_mm_round_pd(a, SIMDE_MM_FROUND_TO_ZERO); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_trunc_pd + #define _mm_trunc_pd(a) simde_mm_trunc_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_trunc_ps (simde__m256 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_trunc_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_truncf8(a); + #else + return simde_mm256_round_ps(a, SIMDE_MM_FROUND_TO_ZERO); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_trunc_ps + #define _mm256_trunc_ps(a) simde_mm256_trunc_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_trunc_pd (simde__m256d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_trunc_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX_NATIVE) + return Sleef_truncd4(a); + #else + return simde_mm256_round_pd(a, SIMDE_MM_FROUND_TO_ZERO); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_trunc_pd + #define _mm256_trunc_pd(a) simde_mm256_trunc_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_trunc_ps (simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_trunc_ps(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_truncf16(a); + #else + simde__m512_private + r_, + a_ = simde__m512_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256) / sizeof(r_.m256[0])) ; i++) { + r_.m256[i] = simde_mm256_trunc_ps(a_.m256[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + r_.f32[i] = simde_math_truncf(a_.f32[i]); + } + #endif + + return simde__m512_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_trunc_ps + #define _mm512_trunc_ps(a) simde_mm512_trunc_ps(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_trunc_pd (simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_trunc_pd(a); + #elif defined(SIMDE_MATH_SLEEF_ENABLE) && defined(SIMDE_X86_AVX512F_NATIVE) + return Sleef_truncd8(a); + #else + simde__m512d_private + r_, + a_ = simde__m512d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(256) + for (size_t i = 0 ; i < (sizeof(r_.m256d) / sizeof(r_.m256d[0])) ; i++) { + r_.m256d[i] = simde_mm256_trunc_pd(a_.m256d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + r_.f64[i] = simde_math_trunc(a_.f64[i]); + } + #endif + + return simde__m512d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_trunc_pd + #define _mm512_trunc_pd(a) simde_mm512_trunc_pd(a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512 +simde_mm512_mask_trunc_ps(simde__m512 src, simde__mmask16 k, simde__m512 a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_trunc_ps(src, k, a); + #else + return simde_mm512_mask_mov_ps(src, k, simde_mm512_trunc_ps(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_trunc_ps + #define _mm512_mask_trunc_ps(src, k, a) simde_mm512_mask_trunc_ps(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m512d +simde_mm512_mask_trunc_pd(simde__m512d src, simde__mmask8 k, simde__m512d a) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX512F_NATIVE) + return _mm512_mask_trunc_pd(src, k, a); + #else + return simde_mm512_mask_mov_pd(src, k, simde_mm512_trunc_pd(a)); + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm512_mask_trunc_pd + #define _mm512_mask_trunc_pd(src, k, a) simde_mm512_mask_trunc_pd(src, k, a) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_udivrem_epi32 (simde__m128i * mem_addr, simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_SSE_NATIVE) + return _mm_udivrem_epi32(mem_addr, a, b); + #else + simde__m128i r; + + r = simde_mm_div_epu32(a, b); + *mem_addr = simde_x_mm_sub_epu32(a, simde_x_mm_mullo_epu32(r, b)); + + return r; + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm_udivrem_epi32 + #define _mm_udivrem_epi32(mem_addr, a, b) simde_mm_udivrem_epi32((mem_addr),(a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_udivrem_epi32 (simde__m256i* mem_addr, simde__m256i a, simde__m256i b) { + #if defined(SIMDE_X86_SVML_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) + return _mm256_udivrem_epi32(HEDLEY_REINTERPRET_CAST(__m256i*, mem_addr), a, b); + #else + simde__m256i r; + + r = simde_mm256_div_epu32(a, b); + *mem_addr = simde_x_mm256_sub_epu32(a, simde_x_mm256_mullo_epu32(r, b)); + + return r; + #endif +} +#if defined(SIMDE_X86_SVML_ENABLE_NATIVE_ALIASES) + #undef _mm256_udivrem_epi32 + #define _mm256_udivrem_epi32(mem_addr, a, b) simde_mm256_udivrem_epi32((mem_addr),(a), (b)) +#endif + +SIMDE_END_DECLS_ + +HEDLEY_DIAGNOSTIC_POP + +#endif /* !defined(SIMDE_X86_SVML_H) */ diff --git a/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/xop.h b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/xop.h new file mode 100644 index 00000000..5249f06d --- /dev/null +++ b/thirdparty/rexglue-sdk/thirdparty/simde/simde/x86/xop.h @@ -0,0 +1,3740 @@ +/* SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Copyright: + * 2020 Evan Nemerson + */ + +#if !defined(SIMDE_X86_XOP_H) +#define SIMDE_X86_XOP_H + +#include "avx2.h" + +#if !defined(SIMDE_X86_XOP_NATIVE) && defined(SIMDE_ENABLE_NATIVE_ALIASES) +# define SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES +#endif + +HEDLEY_DIAGNOSTIC_PUSH +SIMDE_DISABLE_UNWANTED_DIAGNOSTICS +SIMDE_BEGIN_DECLS_ + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_cmov_si128 (simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_cmov_si128(a, b, c); + #elif defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm_ternarylogic_epi32(a, b, c, 0xe4); + #elif defined(SIMDE_X86_SSE2_NATIVE) + return _mm_or_si128(_mm_and_si128(c, a), _mm_andnot_si128(c, b)); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vbslq_s8(c_.neon_u8, a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_v128_bitselect(a_.wasm_v128, b_.wasm_v128, c_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P7_NATIVE) + r_.altivec_i32 = vec_sel(b_.altivec_i32, a_.altivec_i32, c_.altivec_u32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32f = (c_.i32f & a_.i32f) | (~c_.i32f & b_.i32f); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = (c_.i32f[i] & a_.i32f[i]) | (~c_.i32f[i] & b_.i32f[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_cmov_si128(a, b, c) simde_mm_cmov_si128((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256i +simde_mm256_cmov_si256 (simde__m256i a, simde__m256i b, simde__m256i c) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(SIMDE_X86_AVX_NATIVE) && !defined(SIMDE_BUG_GCC_98521) && !defined(SIMDE_BUG_MCST_LCC_MISSING_CMOV_M256) + return _mm256_cmov_si256(a, b, c); + #elif defined(SIMDE_X86_AVX512VL_NATIVE) + return _mm256_ternarylogic_epi32(a, b, c, 0xe4); + #elif defined(SIMDE_X86_AVX2_NATIVE) + return _mm256_or_si256(_mm256_and_si256(c, a), _mm256_andnot_si256(c, b)); + #else + simde__m256i_private + r_, + a_ = simde__m256i_to_private(a), + b_ = simde__m256i_to_private(b), + c_ = simde__m256i_to_private(c); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128i) / sizeof(r_.m128i[0])) ; i++) { + r_.m128i[i] = simde_mm_cmov_si128(a_.m128i[i], b_.m128i[i], c_.m128i[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32f) / sizeof(r_.i32f[0])) ; i++) { + r_.i32f[i] = (c_.i32f[i] & a_.i32f[i]) | (~c_.i32f[i] & b_.i32f[i]); + } + #endif + + return simde__m256i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm256_cmov_si256(a, b, c) simde_mm256_cmov_si256((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comeq_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_EQ) + return _mm_com_epi8(a, b, _MM_PCOMCTRL_EQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comeq_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vceqq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), a_.i8 == b_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] == b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comeq_epi8(a, b) simde_mm_comeq_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comeq_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_EQ) + return _mm_com_epi16(a, b, _MM_PCOMCTRL_EQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comeq_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vceqq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), a_.i16 == b_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] == b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comeq_epi16(a, b) simde_mm_comeq_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comeq_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_EQ) + return _mm_com_epi32(a, b, _MM_PCOMCTRL_EQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comeq_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vceqq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.i32 == b_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] == b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comeq_epi32(a, b) simde_mm_comeq_epi32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comeq_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_EQ) + return _mm_com_epi64(a, b, _MM_PCOMCTRL_EQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comeq_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vceqq_s64(a_.neon_i64, b_.neon_i64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 == b_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] == b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comeq_epi64(a, b) simde_mm_comeq_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comeq_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_EQ) + return _mm_com_epu8(a, b, _MM_PCOMCTRL_EQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comeq_epu8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vceqq_u8(a_.neon_u8, b_.neon_u8); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u8), a_.u8 == b_.u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] == b_.u8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comeq_epu8(a, b) simde_mm_comeq_epu8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comeq_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_EQ) + return _mm_com_epu16(a, b, _MM_PCOMCTRL_EQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comeq_epu16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vceqq_u16(a_.neon_u16, b_.neon_u16); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), a_.u16 == b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] == b_.u16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comeq_epu16(a, b) simde_mm_comeq_epu16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comeq_epu32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_EQ) + return _mm_com_epu32(a, b, _MM_PCOMCTRL_EQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comeq_epu32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vceqq_u32(a_.neon_u32, b_.neon_u32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), a_.u32 == b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] == b_.u32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comeq_epu32(a, b) simde_mm_comeq_epu32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comeq_epu64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_EQ) + return _mm_com_epu64(a, b, _MM_PCOMCTRL_EQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comeq_epu64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vceqq_u64(a_.neon_u64, b_.neon_u64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), a_.u64 == b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] == b_.u64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comeq_epu64(a, b) simde_mm_comeq_epu64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comge_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GE) + return _mm_com_epi8(a, b, _MM_PCOMCTRL_GE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comge_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcgeq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), a_.i8 >= b_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] >= b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comge_epi8(a, b) simde_mm_comge_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comge_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GE) + return _mm_com_epi16(a, b, _MM_PCOMCTRL_GE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comge_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vcgeq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), a_.i16 >= b_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] >= b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comge_epi16(a, b) simde_mm_comge_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comge_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GE) + return _mm_com_epi32(a, b, _MM_PCOMCTRL_GE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comge_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcgeq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.i32 >= b_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] >= b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comge_epi32(a, b) simde_mm_comge_epi32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comge_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GE) + return _mm_com_epi64(a, b, _MM_PCOMCTRL_GE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comge_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcgeq_s64(a_.neon_i64, b_.neon_i64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 >= b_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] >= b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comge_epi64(a, b) simde_mm_comge_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comge_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GE) + return _mm_com_epu8(a, b, _MM_PCOMCTRL_GE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comge_epu8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcgeq_u8(a_.neon_u8, b_.neon_u8); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u8), a_.u8 >= b_.u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] >= b_.u8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comge_epu8(a, b) simde_mm_comge_epu8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comge_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GE) + return _mm_com_epu16(a, b, _MM_PCOMCTRL_GE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comge_epu16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vcgeq_u16(a_.neon_u16, b_.neon_u16); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), a_.u16 >= b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] >= b_.u16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comge_epu16(a, b) simde_mm_comge_epu16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comge_epu32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GE) + return _mm_com_epu32(a, b, _MM_PCOMCTRL_GE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comge_epu32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcgeq_u32(a_.neon_u32, b_.neon_u32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), a_.u32 >= b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] >= b_.u32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comge_epu32(a, b) simde_mm_comge_epu32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comge_epu64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GE) + return _mm_com_epu64(a, b, _MM_PCOMCTRL_GE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comge_epu64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcgeq_u64(a_.neon_u64, b_.neon_u64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), a_.u64 >= b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] >= b_.u64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comge_epu64(a, b) simde_mm_comge_epu64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comgt_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GT) + return _mm_com_epi8(a, b, _MM_PCOMCTRL_GT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comgt_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcgtq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), a_.i8 > b_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] > b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comgt_epi8(a, b) simde_mm_comgt_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comgt_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GT) + return _mm_com_epi16(a, b, _MM_PCOMCTRL_GT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comgt_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vcgtq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), a_.i16 > b_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] > b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comgt_epi16(a, b) simde_mm_comgt_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comgt_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GT) + return _mm_com_epi32(a, b, _MM_PCOMCTRL_GT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comgt_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcgtq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.i32 > b_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] > b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comgt_epi32(a, b) simde_mm_comgt_epi32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comgt_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GT) + return _mm_com_epi64(a, b, _MM_PCOMCTRL_GT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comgt_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcgtq_s64(a_.neon_i64, b_.neon_i64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 > b_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] > b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comgt_epi64(a, b) simde_mm_comgt_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comgt_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GT) + return _mm_com_epu8(a, b, _MM_PCOMCTRL_GT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comgt_epu8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcgtq_u8(a_.neon_u8, b_.neon_u8); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u8), a_.u8 > b_.u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] > b_.u8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comgt_epu8(a, b) simde_mm_comgt_epu8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comgt_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GT) + return _mm_com_epu16(a, b, _MM_PCOMCTRL_GT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comgt_epu16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vcgtq_u16(a_.neon_u16, b_.neon_u16); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), a_.u16 > b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] > b_.u16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comgt_epu16(a, b) simde_mm_comgt_epu16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comgt_epu32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GT) + return _mm_com_epu32(a, b, _MM_PCOMCTRL_GT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comgt_epu32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcgtq_u32(a_.neon_u32, b_.neon_u32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), a_.u32 > b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] > b_.u32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comgt_epu32(a, b) simde_mm_comgt_epu32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comgt_epu64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_GT) + return _mm_com_epu64(a, b, _MM_PCOMCTRL_GT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comgt_epu64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcgtq_u64(a_.neon_u64, b_.neon_u64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), a_.u64 > b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] > b_.u64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comgt_epu64(a, b) simde_mm_comgt_epu64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comle_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LE) + return _mm_com_epi8(a, b, _MM_PCOMCTRL_LE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comle_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcleq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), a_.i8 <= b_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] <= b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comle_epi8(a, b) simde_mm_comle_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comle_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LE) + return _mm_com_epi16(a, b, _MM_PCOMCTRL_LE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comle_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), a_.i16 <= b_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] <= b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comle_epi16(a, b) simde_mm_comle_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comle_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LE) + return _mm_com_epi32(a, b, _MM_PCOMCTRL_LE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comle_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcleq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.i32 <= b_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] <= b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comle_epi32(a, b) simde_mm_comle_epi32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comle_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LE) + return _mm_com_epi64(a, b, _MM_PCOMCTRL_LE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comle_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcleq_s64(a_.neon_i64, b_.neon_i64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 <= b_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] <= b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comle_epi64(a, b) simde_mm_comle_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comle_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LE) + return _mm_com_epu8(a, b, _MM_PCOMCTRL_LE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comle_epu8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcleq_u8(a_.neon_u8, b_.neon_u8); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u8), a_.u8 <= b_.u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] <= b_.u8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comle_epu8(a, b) simde_mm_comle_epu8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comle_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LE) + return _mm_com_epu16(a, b, _MM_PCOMCTRL_LE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comle_epu16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vcleq_u16(a_.neon_u16, b_.neon_u16); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), a_.u16 <= b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] <= b_.u16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comle_epu16(a, b) simde_mm_comle_epu16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comle_epu32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LE) + return _mm_com_epu32(a, b, _MM_PCOMCTRL_LE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comle_epu32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcleq_u32(a_.neon_u32, b_.neon_u32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), a_.u32 <= b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] <= b_.u32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comle_epu32(a, b) simde_mm_comle_epu32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comle_epu64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LE) + return _mm_com_epu64(a, b, _MM_PCOMCTRL_LE); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comle_epu64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcleq_u64(a_.neon_u64, b_.neon_u64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), a_.u64 <= b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] <= b_.u64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comle_epu64(a, b) simde_mm_comle_epu64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comlt_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LT) + return _mm_com_epi8(a, b, _MM_PCOMCTRL_LT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comlt_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcltq_s8(a_.neon_i8, b_.neon_i8); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), a_.i8 < b_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] < b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comlt_epi8(a, b) simde_mm_comlt_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comlt_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LT) + return _mm_com_epi16(a, b, _MM_PCOMCTRL_LT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comlt_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vcltq_s16(a_.neon_i16, b_.neon_i16); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), a_.i16 < b_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] < b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comlt_epi16(a, b) simde_mm_comlt_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comlt_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LT) + return _mm_com_epi32(a, b, _MM_PCOMCTRL_LT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comlt_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcltq_s32(a_.neon_i32, b_.neon_i32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.i32 < b_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] < b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comlt_epi32(a, b) simde_mm_comlt_epi32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comlt_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LT) + return _mm_com_epi64(a, b, _MM_PCOMCTRL_LT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comlt_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcltq_s64(a_.neon_i64, b_.neon_i64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 < b_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] < b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comlt_epi64(a, b) simde_mm_comlt_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comlt_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LT) + return _mm_com_epu8(a, b, _MM_PCOMCTRL_LT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comlt_epu8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vcltq_u8(a_.neon_u8, b_.neon_u8); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u8), a_.u8 < b_.u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] < b_.u8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comlt_epu8(a, b) simde_mm_comlt_epu8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comlt_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LT) + return _mm_com_epu16(a, b, _MM_PCOMCTRL_LT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comlt_epu16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vcltq_u16(a_.neon_u16, b_.neon_u16); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), a_.u16 < b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] < b_.u16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comlt_epu16(a, b) simde_mm_comlt_epu16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comlt_epu32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LT) + return _mm_com_epu32(a, b, _MM_PCOMCTRL_LT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comlt_epu32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vcltq_u32(a_.neon_u32, b_.neon_u32); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), a_.u32 < b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] < b_.u32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comlt_epu32(a, b) simde_mm_comlt_epu32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comlt_epu64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LT) + return _mm_com_epu64(a, b, _MM_PCOMCTRL_LT); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comlt_epu64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u64 = vcltq_u64(a_.neon_u64, b_.neon_u64); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), a_.u64 < b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] < b_.u64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comlt_epu64(a, b) simde_mm_comlt_epu64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comneq_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_NEQ) + return _mm_com_epi8(a, b, _MM_PCOMCTRL_NEQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comneq_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vmvnq_u8(vceqq_s8(a_.neon_i8, b_.neon_i8)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i8), a_.i8 != b_.i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + r_.i8[i] = (a_.i8[i] != b_.i8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comneq_epi8(a, b) simde_mm_comneq_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comneq_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_NEQ) + return _mm_com_epi16(a, b, _MM_PCOMCTRL_NEQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comneq_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vmvnq_u16(vceqq_s16(a_.neon_i16, b_.neon_i16)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i16), a_.i16 != b_.i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] != b_.i16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comneq_epi16(a, b) simde_mm_comneq_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comneq_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_NEQ) + return _mm_com_epi32(a, b, _MM_PCOMCTRL_NEQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comneq_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vmvnq_u32(vceqq_s32(a_.neon_i32, b_.neon_i32)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i32), a_.i32 != b_.i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] != b_.i32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comneq_epi32(a, b) simde_mm_comneq_epi32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comneq_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_NEQ) + return _mm_com_epi64(a, b, _MM_PCOMCTRL_NEQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comneq_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u32 = vmvnq_u32(vreinterpretq_u32_u64(vceqq_s64(a_.neon_i64, b_.neon_i64))); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.i64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.i64), a_.i64 != b_.i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (a_.i64[i] != b_.i64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comneq_epi64(a, b) simde_mm_comneq_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comneq_epu8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_NEQ) + return _mm_com_epu8(a, b, _MM_PCOMCTRL_NEQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comneq_epu8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vmvnq_u8(vceqq_u8(a_.neon_u8, b_.neon_u8)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u8 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u8), a_.u8 != b_.u8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (a_.u8[i] != b_.u8[i]) ? ~INT8_C(0) : INT8_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comneq_epu8(a, b) simde_mm_comneq_epu8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comneq_epu16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_NEQ) + return _mm_com_epu16(a, b, _MM_PCOMCTRL_NEQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comneq_epu16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vmvnq_u16(vceqq_u16(a_.neon_u16, b_.neon_u16)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u16 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u16), a_.u16 != b_.u16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (a_.u16[i] != b_.u16[i]) ? ~INT16_C(0) : INT16_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comneq_epu16(a, b) simde_mm_comneq_epu16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comneq_epu32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_NEQ) + return _mm_com_epu32(a, b, _MM_PCOMCTRL_NEQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comneq_epu32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vmvnq_u32(vceqq_u32(a_.neon_u32, b_.neon_u32)); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u32 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u32), a_.u32 != b_.u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (a_.u32[i] != b_.u32[i]) ? ~INT32_C(0) : INT32_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comneq_epu32(a, b) simde_mm_comneq_epu32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comneq_epu64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_NEQ) + return _mm_com_epu64(a, b, _MM_PCOMCTRL_NEQ); + #elif defined(SIMDE_X86_XOP_NATIVE) + return _mm_comneq_epu64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + r_.neon_u32 = vmvnq_u32(vreinterpretq_u32_u64(vceqq_u64(a_.neon_u64, b_.neon_u64))); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_OPS) + r_.u64 = HEDLEY_REINTERPRET_CAST(__typeof__(r_.u64), a_.u64 != b_.u64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (a_.u64[i] != b_.u64[i]) ? ~INT64_C(0) : INT64_C(0); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comneq_epu64(a, b) simde_mm_comneq_epu64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comfalse_epi8 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_mm_setzero_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comfalse_epi8(a, b) simde_mm_comfalse_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comfalse_epi16 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_mm_setzero_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comfalse_epi16(a, b) simde_mm_comfalse_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comfalse_epi32 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_mm_setzero_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comfalse_epi32(a, b) simde_mm_comfalse_epi32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comfalse_epi64 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_mm_setzero_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comfalse_epi64(a, b) simde_mm_comfalse_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comfalse_epu8 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_mm_setzero_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comfalse_epu8(a, b) simde_mm_comfalse_epu8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comfalse_epu16 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_mm_setzero_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comfalse_epu16(a, b) simde_mm_comfalse_epu16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comfalse_epu32 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_mm_setzero_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comfalse_epu32(a, b) simde_mm_comfalse_epu32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comfalse_epu64 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_mm_setzero_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comfalse_epu64(a, b) simde_mm_comfalse_epu64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comtrue_epi8 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_x_mm_setone_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comtrue_epi8(a, b) simde_mm_comtrue_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comtrue_epi16 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_x_mm_setone_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comtrue_epi16(a, b) simde_mm_comtrue_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comtrue_epi32 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_x_mm_setone_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comtrue_epi32(a, b) simde_mm_comtrue_epi32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comtrue_epi64 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_x_mm_setone_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comtrue_epi64(a, b) simde_mm_comtrue_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comtrue_epu8 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_x_mm_setone_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comtrue_epu8(a, b) simde_mm_comtrue_epu8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comtrue_epu16 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_x_mm_setone_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comtrue_epu16(a, b) simde_mm_comtrue_epu16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comtrue_epu32 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_x_mm_setone_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comtrue_epu32(a, b) simde_mm_comtrue_epu32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_comtrue_epu64 (simde__m128i a, simde__m128i b) { + (void) a; + (void) b; + return simde_x_mm_setone_si128(); +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_comtrue_epu64(a, b) simde_mm_comtrue_epu64((a), (b)) +#endif + +#if defined(SIMDE_X86_XOP_NATIVE) && defined(_MM_PCOMCTRL_LT) + #define SIMDE_X86_XOP_HAVE_COM_ 1 + #define SIMDE_MM_PCOMCTRL_LT _MM_PCOMCTRL_LT + #define SIMDE_MM_PCOMCTRL_LE _MM_PCOMCTRL_LE + #define SIMDE_MM_PCOMCTRL_GT _MM_PCOMCTRL_GT + #define SIMDE_MM_PCOMCTRL_GE _MM_PCOMCTRL_GE + #define SIMDE_MM_PCOMCTRL_EQ _MM_PCOMCTRL_EQ + #define SIMDE_MM_PCOMCTRL_NEQ _MM_PCOMCTRL_NEQ + #define SIMDE_MM_PCOMCTRL_FALSE _MM_PCOMCTRL_FALSE + #define SIMDE_MM_PCOMCTRL_TRUE _MM_PCOMCTRL_TRUE +#else + #define SIMDE_MM_PCOMCTRL_LT 0 + #define SIMDE_MM_PCOMCTRL_LE 1 + #define SIMDE_MM_PCOMCTRL_GT 2 + #define SIMDE_MM_PCOMCTRL_GE 3 + #define SIMDE_MM_PCOMCTRL_EQ 4 + #define SIMDE_MM_PCOMCTRL_NEQ 5 + #define SIMDE_MM_PCOMCTRL_FALSE 6 + #define SIMDE_MM_PCOMCTRL_TRUE 7 + + #if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _MM_PCOMCTRL_LT SIMDE_MM_PCOMCTRL_LT + #define _MM_PCOMCTRL_LE SIMDE_MM_PCOMCTRL_LE + #define _MM_PCOMCTRL_GT SIMDE_MM_PCOMCTRL_GT + #define _MM_PCOMCTRL_GE SIMDE_MM_PCOMCTRL_GE + #define _MM_PCOMCTRL_EQ SIMDE_MM_PCOMCTRL_EQ + #define _MM_PCOMCTRL_NEQ SIMDE_MM_PCOMCTRL_NEQ + #define _MM_PCOMCTRL_FALSE SIMDE_MM_PCOMCTRL_FALSE + #define _MM_PCOMCTRL_TRUE SIMDE_MM_PCOMCTRL_TRUE + #endif +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_com_epi8 (simde__m128i a, simde__m128i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + switch (imm8) { + case SIMDE_MM_PCOMCTRL_LT: + return simde_mm_comlt_epi8(a, b); + case SIMDE_MM_PCOMCTRL_LE: + return simde_mm_comle_epi8(a, b); + case SIMDE_MM_PCOMCTRL_GT: + return simde_mm_comgt_epi8(a, b); + case SIMDE_MM_PCOMCTRL_GE: + return simde_mm_comge_epi8(a, b); + case SIMDE_MM_PCOMCTRL_EQ: + return simde_mm_comeq_epi8(a, b); + case SIMDE_MM_PCOMCTRL_NEQ: + return simde_mm_comneq_epi8(a, b); + case SIMDE_MM_PCOMCTRL_FALSE: + return simde_mm_comfalse_epi8(a, b); + case SIMDE_MM_PCOMCTRL_TRUE: + return simde_mm_comtrue_epi8(a, b); + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm_setzero_si128()); + } +} +#if defined(SIMDE_X86_XOP_NATIVE) && defined(SIMDE_X86_XOP_HAVE_COM_) + #define simde_mm_com_epi8(a, b, imm8) _mm_com_epi8((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_com_epi8(a, b, imm8) simde_mm_com_epi8((a), (b), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_com_epi16 (simde__m128i a, simde__m128i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + switch (imm8) { + case SIMDE_MM_PCOMCTRL_LT: + return simde_mm_comlt_epi16(a, b); + case SIMDE_MM_PCOMCTRL_LE: + return simde_mm_comle_epi16(a, b); + case SIMDE_MM_PCOMCTRL_GT: + return simde_mm_comgt_epi16(a, b); + case SIMDE_MM_PCOMCTRL_GE: + return simde_mm_comge_epi16(a, b); + case SIMDE_MM_PCOMCTRL_EQ: + return simde_mm_comeq_epi16(a, b); + case SIMDE_MM_PCOMCTRL_NEQ: + return simde_mm_comneq_epi16(a, b); + case SIMDE_MM_PCOMCTRL_FALSE: + return simde_mm_comfalse_epi16(a, b); + case SIMDE_MM_PCOMCTRL_TRUE: + return simde_mm_comtrue_epi16(a, b); + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm_setzero_si128()); + } +} +#if defined(SIMDE_X86_XOP_NATIVE) && defined(SIMDE_X86_XOP_HAVE_COM_) + #define simde_mm_com_epi16(a, b, imm8) _mm_com_epi16((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_com_epi16(a, b, imm8) simde_mm_com_epi16((a), (b), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_com_epi32 (simde__m128i a, simde__m128i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + switch (imm8) { + case SIMDE_MM_PCOMCTRL_LT: + return simde_mm_comlt_epi32(a, b); + case SIMDE_MM_PCOMCTRL_LE: + return simde_mm_comle_epi32(a, b); + case SIMDE_MM_PCOMCTRL_GT: + return simde_mm_comgt_epi32(a, b); + case SIMDE_MM_PCOMCTRL_GE: + return simde_mm_comge_epi32(a, b); + case SIMDE_MM_PCOMCTRL_EQ: + return simde_mm_comeq_epi32(a, b); + case SIMDE_MM_PCOMCTRL_NEQ: + return simde_mm_comneq_epi32(a, b); + case SIMDE_MM_PCOMCTRL_FALSE: + return simde_mm_comfalse_epi32(a, b); + case SIMDE_MM_PCOMCTRL_TRUE: + return simde_mm_comtrue_epi32(a, b); + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm_setzero_si128()); + } +} +#if defined(SIMDE_X86_XOP_NATIVE) && defined(SIMDE_X86_XOP_HAVE_COM_) + #define simde_mm_com_epi32(a, b, imm8) _mm_com_epi32((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_com_epi32(a, b, imm8) simde_mm_com_epi32((a), (b), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_com_epi64 (simde__m128i a, simde__m128i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + switch (imm8) { + case SIMDE_MM_PCOMCTRL_LT: + return simde_mm_comlt_epi64(a, b); + case SIMDE_MM_PCOMCTRL_LE: + return simde_mm_comle_epi64(a, b); + case SIMDE_MM_PCOMCTRL_GT: + return simde_mm_comgt_epi64(a, b); + case SIMDE_MM_PCOMCTRL_GE: + return simde_mm_comge_epi64(a, b); + case SIMDE_MM_PCOMCTRL_EQ: + return simde_mm_comeq_epi64(a, b); + case SIMDE_MM_PCOMCTRL_NEQ: + return simde_mm_comneq_epi64(a, b); + case SIMDE_MM_PCOMCTRL_FALSE: + return simde_mm_comfalse_epi64(a, b); + case SIMDE_MM_PCOMCTRL_TRUE: + return simde_mm_comtrue_epi64(a, b); + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm_setzero_si128()); + } +} +#if defined(SIMDE_X86_XOP_NATIVE) && defined(SIMDE_X86_XOP_HAVE_COM_) + #define simde_mm_com_epi64(a, b, imm8) _mm_com_epi64((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_com_epi64(a, b, imm8) simde_mm_com_epi64((a), (b), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_com_epu8 (simde__m128i a, simde__m128i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + switch (imm8) { + case SIMDE_MM_PCOMCTRL_LT: + return simde_mm_comlt_epu8(a, b); + case SIMDE_MM_PCOMCTRL_LE: + return simde_mm_comle_epu8(a, b); + case SIMDE_MM_PCOMCTRL_GT: + return simde_mm_comgt_epu8(a, b); + case SIMDE_MM_PCOMCTRL_GE: + return simde_mm_comge_epu8(a, b); + case SIMDE_MM_PCOMCTRL_EQ: + return simde_mm_comeq_epu8(a, b); + case SIMDE_MM_PCOMCTRL_NEQ: + return simde_mm_comneq_epu8(a, b); + case SIMDE_MM_PCOMCTRL_FALSE: + return simde_mm_comfalse_epu8(a, b); + case SIMDE_MM_PCOMCTRL_TRUE: + return simde_mm_comtrue_epu8(a, b); + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm_setzero_si128()); + } +} +#if defined(SIMDE_X86_XOP_NATIVE) && defined(SIMDE_X86_XOP_HAVE_COM_) + #define simde_mm_com_epu8(a, b, imm8) _mm_com_epu8((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_com_epu8(a, b, imm8) simde_mm_com_epu8((a), (b), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_com_epu16 (simde__m128i a, simde__m128i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + switch (imm8) { + case SIMDE_MM_PCOMCTRL_LT: + return simde_mm_comlt_epu16(a, b); + case SIMDE_MM_PCOMCTRL_LE: + return simde_mm_comle_epu16(a, b); + case SIMDE_MM_PCOMCTRL_GT: + return simde_mm_comgt_epu16(a, b); + case SIMDE_MM_PCOMCTRL_GE: + return simde_mm_comge_epu16(a, b); + case SIMDE_MM_PCOMCTRL_EQ: + return simde_mm_comeq_epu16(a, b); + case SIMDE_MM_PCOMCTRL_NEQ: + return simde_mm_comneq_epu16(a, b); + case SIMDE_MM_PCOMCTRL_FALSE: + return simde_mm_comfalse_epu16(a, b); + case SIMDE_MM_PCOMCTRL_TRUE: + return simde_mm_comtrue_epu16(a, b); + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm_setzero_si128()); + } +} +#if defined(SIMDE_X86_XOP_NATIVE) && defined(SIMDE_X86_XOP_HAVE_COM_) + #define simde_mm_com_epu16(a, b, imm8) _mm_com_epu16((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_com_epu16(a, b, imm8) simde_mm_com_epu16((a), (b), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_com_epu32 (simde__m128i a, simde__m128i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + switch (imm8) { + case SIMDE_MM_PCOMCTRL_LT: + return simde_mm_comlt_epu32(a, b); + case SIMDE_MM_PCOMCTRL_LE: + return simde_mm_comle_epu32(a, b); + case SIMDE_MM_PCOMCTRL_GT: + return simde_mm_comgt_epu32(a, b); + case SIMDE_MM_PCOMCTRL_GE: + return simde_mm_comge_epu32(a, b); + case SIMDE_MM_PCOMCTRL_EQ: + return simde_mm_comeq_epu32(a, b); + case SIMDE_MM_PCOMCTRL_NEQ: + return simde_mm_comneq_epu32(a, b); + case SIMDE_MM_PCOMCTRL_FALSE: + return simde_mm_comfalse_epu32(a, b); + case SIMDE_MM_PCOMCTRL_TRUE: + return simde_mm_comtrue_epu32(a, b); + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm_setzero_si128()); + } +} +#if defined(SIMDE_X86_XOP_NATIVE) && defined(SIMDE_X86_XOP_HAVE_COM_) + #define simde_mm_com_epu32(a, b, imm8) _mm_com_epu32((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_com_epu32(a, b, imm8) simde_mm_com_epu32((a), (b), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_com_epu64 (simde__m128i a, simde__m128i b, const int imm8) + SIMDE_REQUIRE_CONSTANT_RANGE(imm8, 0, 7) { + switch (imm8) { + case SIMDE_MM_PCOMCTRL_LT: + return simde_mm_comlt_epu64(a, b); + case SIMDE_MM_PCOMCTRL_LE: + return simde_mm_comle_epu64(a, b); + case SIMDE_MM_PCOMCTRL_GT: + return simde_mm_comgt_epu64(a, b); + case SIMDE_MM_PCOMCTRL_GE: + return simde_mm_comge_epu64(a, b); + case SIMDE_MM_PCOMCTRL_EQ: + return simde_mm_comeq_epu64(a, b); + case SIMDE_MM_PCOMCTRL_NEQ: + return simde_mm_comneq_epu64(a, b); + case SIMDE_MM_PCOMCTRL_FALSE: + return simde_mm_comfalse_epu64(a, b); + case SIMDE_MM_PCOMCTRL_TRUE: + return simde_mm_comtrue_epu64(a, b); + default: + HEDLEY_UNREACHABLE_RETURN(simde_mm_setzero_si128()); + } +} +#if defined(SIMDE_X86_XOP_NATIVE) && defined(SIMDE_X86_XOP_HAVE_COM_) + #define simde_mm_com_epu64(a, b, imm8) _mm_com_epu64((a), (b), (imm8)) +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_com_epu64(a, b, imm8) simde_mm_com_epu64((a), (b), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_frcz_ps (simde__m128 a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_frcz_ps(a); + #else + simde__m128_private + r_, + a_ = simde__m128_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + #if defined(simde_math_modff) + simde_float32 integral; + r_.f32[i] = simde_math_modff(a_.f32[i], &integral); + #else + r_.f32[i] = (a_.f32[i] / 1.0f); + #endif + } + + return simde__m128_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_frcz_ps(a) simde_mm_frcz_ps((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_frcz_pd (simde__m128d a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_frcz_pd(a); + #else + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + #if defined(simde_math_modf) + simde_float64 integral; + r_.f64[i] = simde_math_modf(a_.f64[i], &integral); + #else + r_.f64[i] = (a_.f64[i] / 1.0f); + #endif + } + + return simde__m128d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_frcz_ps(a) simde_mm_frcz_ps((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_frcz_ss (simde__m128 a, simde__m128 b) { + #if defined(SIMDE_X86_XOP_NATIVE) && !defined(SIMDE_BUG_CLANG_48673) + return _mm_frcz_ss(a, b); + #else + simde__m128_private + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + + #if defined(simde_math_modff) + simde_float32 integral; + a_.f32[0] = simde_math_modff(b_.f32[0], &integral); + #else + a_.f32[0] = (b_.f32[0] / 1.0f); + #endif + + return simde__m128_from_private(a_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_frcz_ss(a, b) simde_mm_frcz_ss((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_frcz_sd (simde__m128d a, simde__m128d b) { + #if defined(SIMDE_X86_XOP_NATIVE) && !defined(SIMDE_BUG_CLANG_48673) + return _mm_frcz_sd(a, b); + #else + simde__m128d_private + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + + #if defined(simde_math_modf) + simde_float64 integral; + a_.f64[0] = simde_math_modf(b_.f64[0], &integral); + #else + a_.f64[0] = (b_.f64[0] / 1.0f); + #endif + + return simde__m128d_from_private(a_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_frcz_sd(a, b) simde_mm_frcz_sd((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_frcz_ps (simde__m256 a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm256_frcz_ps(a); + #else + simde__m256_private + r_, + a_ = simde__m256_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_frcz_ps(a_.m128[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + #if defined(simde_math_modff) + simde_float32 integral; + r_.f32[i] = simde_math_modff(a_.f32[i], &integral); + #else + r_.f32[i] = (a_.f32[i] / 1.0f); + #endif + } + #endif + + return simde__m256_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm256_frcz_ps(a) simde_mm256_frcz_ps((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_frcz_pd (simde__m256d a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm256_frcz_pd(a); + #else + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_frcz_pd(a_.m128d[i]); + } + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + #if defined(simde_math_modf) + simde_float64 integral; + r_.f64[i] = simde_math_modf(a_.f64[i], &integral); + #else + r_.f64[i] = (a_.f64[i] / 1.0f); + #endif + } + #endif + + return simde__m256d_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm256_frcz_ps(a) simde_mm256_frcz_ps((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_haddw_epi8 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_haddw_epi8(a); + #elif defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_maddubs_epi16(_mm_set1_epi8(INT8_C(1)), a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vpaddlq_s8(a_.neon_i8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i16x8_extadd_pairwise_i8x16(a_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(signed char) one = vec_splat_s8(1); + r_.altivec_i16 = + vec_add( + vec_mule(a_.altivec_i8, one), + vec_mulo(a_.altivec_i8, one) + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i16 = + ((a_.i16 << 8) >> 8) + + ((a_.i16 >> 8) ); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, a_.i8[(i * 2)]) + HEDLEY_STATIC_CAST(int16_t, a_.i8[(i * 2) + 1]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_haddw_epi8(a) simde_mm_haddw_epi8((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_haddw_epu8 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_haddw_epu8(a); + #elif defined(SIMDE_X86_SSSE3_NATIVE) + return _mm_maddubs_epi16(a, _mm_set1_epi8(INT8_C(1))); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vpaddlq_u8(a_.neon_u8); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u16x8_extadd_pairwise_u8x16(a_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned char) one = vec_splat_u8(1); + r_.altivec_u16 = + vec_add( + vec_mule(a_.altivec_u8, one), + vec_mulo(a_.altivec_u8, one) + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u16 = + ((a_.u16 << 8) >> 8) + + ((a_.u16 >> 8) ); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, a_.u8[(i * 2)]) + HEDLEY_STATIC_CAST(uint16_t, a_.u8[(i * 2) + 1]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_haddw_epu8(a) simde_mm_haddw_epu8((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_haddd_epi8 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_haddd_epi8(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vpaddlq_s16(vpaddlq_s8(a_.neon_i8)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = + HEDLEY_STATIC_CAST(int32_t, a_.i8[(i * 4) ]) + HEDLEY_STATIC_CAST(int32_t, a_.i8[(i * 4) + 1]) + + HEDLEY_STATIC_CAST(int32_t, a_.i8[(i * 4) + 2]) + HEDLEY_STATIC_CAST(int32_t, a_.i8[(i * 4) + 3]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_haddd_epi8(a) simde_mm_haddd_epi8((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_haddd_epi16 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_haddd_epi16(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + return _mm_madd_epi16(a, _mm_set1_epi16(INT8_C(1))); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vpaddlq_s16(a_.neon_i16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_i32x4_extadd_pairwise_i16x8(a_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(signed short) one = vec_splat_s16(1); + r_.altivec_i32 = + vec_add( + vec_mule(a_.altivec_i16, one), + vec_mulo(a_.altivec_i16, one) + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.i32 = + ((a_.i32 << 16) >> 16) + + ((a_.i32 >> 16) ); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = HEDLEY_STATIC_CAST(int32_t, a_.i16[(i * 2)]) + HEDLEY_STATIC_CAST(int32_t, a_.i16[(i * 2) + 1]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_haddd_epi8(a) simde_mm_haddd_epi8((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_haddd_epu8 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_haddd_epu8(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vpaddlq_u16(vpaddlq_u8(a_.neon_u8)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = + HEDLEY_STATIC_CAST(uint32_t, a_.u8[(i * 4) ]) + HEDLEY_STATIC_CAST(uint32_t, a_.u8[(i * 4) + 1]) + + HEDLEY_STATIC_CAST(uint32_t, a_.u8[(i * 4) + 2]) + HEDLEY_STATIC_CAST(uint32_t, a_.u8[(i * 4) + 3]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_haddd_epu8(a) simde_mm_haddd_epu8((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_haddd_epu16 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_haddd_epu16(a); + #elif defined(SIMDE_X86_SSE2_NATIVE) + return + _mm_add_epi32( + _mm_srli_epi32(a, 16), + _mm_and_si128(a, _mm_set1_epi32(INT32_C(0x0000ffff))) + ); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vpaddlq_u16(a_.neon_u16); + #elif defined(SIMDE_WASM_SIMD128_NATIVE) + r_.wasm_v128 = wasm_u32x4_extadd_pairwise_u16x8(a_.wasm_v128); + #elif defined(SIMDE_POWER_ALTIVEC_P6_NATIVE) + SIMDE_POWER_ALTIVEC_VECTOR(unsigned short) one = vec_splat_u16(1); + r_.altivec_u32 = + vec_add( + vec_mule(a_.altivec_u16, one), + vec_mulo(a_.altivec_u16, one) + ); + #elif defined(SIMDE_VECTOR_SUBSCRIPT_SCALAR) + r_.u32 = + ((a_.u32 << 16) >> 16) + + ((a_.u32 >> 16) ); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = HEDLEY_STATIC_CAST(uint32_t, a_.u16[(i * 2)]) + HEDLEY_STATIC_CAST(uint32_t, a_.u16[(i * 2) + 1]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_haddd_epu8(a) simde_mm_haddd_epu8((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_haddq_epi8 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_haddq_epi8(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vpaddlq_s32(vpaddlq_s16(vpaddlq_s8(a_.neon_i8))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = + HEDLEY_STATIC_CAST(int64_t, a_.i8[(i * 8) ]) + HEDLEY_STATIC_CAST(int64_t, a_.i8[(i * 8) + 1]) + + HEDLEY_STATIC_CAST(int64_t, a_.i8[(i * 8) + 2]) + HEDLEY_STATIC_CAST(int64_t, a_.i8[(i * 8) + 3]) + + HEDLEY_STATIC_CAST(int64_t, a_.i8[(i * 8) + 4]) + HEDLEY_STATIC_CAST(int64_t, a_.i8[(i * 8) + 5]) + + HEDLEY_STATIC_CAST(int64_t, a_.i8[(i * 8) + 6]) + HEDLEY_STATIC_CAST(int64_t, a_.i8[(i * 8) + 7]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_haddq_epi8(a) simde_mm_haddq_epi8((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_haddq_epi16 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_haddq_epi16(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vpaddlq_s32(vpaddlq_s16(a_.neon_i16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = + HEDLEY_STATIC_CAST(int64_t, a_.i16[(i * 4) ]) + HEDLEY_STATIC_CAST(int64_t, a_.i16[(i * 4) + 1]) + + HEDLEY_STATIC_CAST(int64_t, a_.i16[(i * 4) + 2]) + HEDLEY_STATIC_CAST(int64_t, a_.i16[(i * 4) + 3]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_haddq_epi16(a) simde_mm_haddq_epi16((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_haddq_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_haddq_epi32(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vpaddlq_s32(a_.neon_i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = HEDLEY_STATIC_CAST(int64_t, a_.i32[(i * 2) ]) + HEDLEY_STATIC_CAST(int64_t, a_.i32[(i * 2) + 1]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_haddq_epi32(a) simde_mm_haddq_epi32((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_haddq_epu8 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_haddq_epu8(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u64 = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(a_.neon_u8))); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.u64[i] = + HEDLEY_STATIC_CAST(uint64_t, a_.u8[(i * 8) ]) + HEDLEY_STATIC_CAST(uint64_t, a_.u8[(i * 8) + 1]) + + HEDLEY_STATIC_CAST(uint64_t, a_.u8[(i * 8) + 2]) + HEDLEY_STATIC_CAST(uint64_t, a_.u8[(i * 8) + 3]) + + HEDLEY_STATIC_CAST(uint64_t, a_.u8[(i * 8) + 4]) + HEDLEY_STATIC_CAST(uint64_t, a_.u8[(i * 8) + 5]) + + HEDLEY_STATIC_CAST(uint64_t, a_.u8[(i * 8) + 6]) + HEDLEY_STATIC_CAST(uint64_t, a_.u8[(i * 8) + 7]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_haddq_epu8(a) simde_mm_haddq_epu8((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_haddq_epu16 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_haddq_epu16(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u64 = vpaddlq_u32(vpaddlq_u16(a_.neon_u16)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.u64[i] = + HEDLEY_STATIC_CAST(uint64_t, a_.u16[(i * 4) ]) + HEDLEY_STATIC_CAST(uint64_t, a_.u16[(i * 4) + 1]) + + HEDLEY_STATIC_CAST(uint64_t, a_.u16[(i * 4) + 2]) + HEDLEY_STATIC_CAST(uint64_t, a_.u16[(i * 4) + 3]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_haddq_epu16(a) simde_mm_haddq_epu16((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_haddq_epu32 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_haddq_epu32(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u64 = vpaddlq_u32(a_.neon_u32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.u64[i] = HEDLEY_STATIC_CAST(uint64_t, a_.u32[(i * 2) ]) + HEDLEY_STATIC_CAST(uint64_t, a_.u32[(i * 2) + 1]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_haddq_epu32(a) simde_mm_haddq_epu32((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_hsubw_epi8 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_hsubw_epi8(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, a_.i8[i * 2]) - HEDLEY_STATIC_CAST(int16_t, a_.i8[(i * 2) + 1]); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_hsubw_epi8(a) simde_mm_hsubw_epi8((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_hsubd_epi16 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_hsubd_epi16(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = + HEDLEY_STATIC_CAST(int32_t, a_.i16[(i * 2) ]) - HEDLEY_STATIC_CAST(int32_t, a_.i16[(i * 2) + 1]); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_hsubd_epi8(a) simde_mm_hsubd_epi8((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_hsubq_epi32 (simde__m128i a) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_hsubq_epi32(a); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = HEDLEY_STATIC_CAST(int64_t, a_.i32[(i * 2) ]) - HEDLEY_STATIC_CAST(int64_t, a_.i32[(i * 2) + 1]); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_hsubq_epi32(a) simde_mm_hsubq_epi32((a)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_macc_epi16 (simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_macc_epi16(a, b, c); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vmlaq_s16(c_.neon_i16, a_.neon_i16, b_.neon_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + r_.i16[i] = (a_.i16[i] * b_.i16[i]) + c_.i16[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_macc_epi16(a, b, c) simde_mm_macc_epi16((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_macc_epi32 (simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_macc_epi32(a, b, c); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vmlaq_s32(c_.neon_i32, a_.neon_i32, b_.neon_i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (a_.i32[i] * b_.i32[i]) + c_.i32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_macc_epi32(a, b, c) simde_mm_macc_epi32((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maccd_epi16 (simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_maccd_epi16(a, b, c); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + int16x8_t even = vuzp1q_s16(a_.neon_i16, b_.neon_i16); + int32x4_t a_even = vmovl_s16(vget_low_s16(even)); + int32x4_t b_even = vmovl_high_s16(even); + r_.neon_i32 = vmlaq_s32(c_.neon_i32, a_even, b_even); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = (HEDLEY_STATIC_CAST(int32_t, a_.i16[i * 2]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[i * 2])) + c_.i32[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_maccd_epi16(a, b, c) simde_mm_maccd_epi16((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_macclo_epi32 (simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_macclo_epi32(a, b, c); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + int32x4_t even = vuzp1q_s32(a_.neon_i32, b_.neon_i32); + r_.neon_i64 = vaddq_s64(vmull_s32(vget_low_s32(even), vget_high_s32(even)), c_.neon_i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (HEDLEY_STATIC_CAST(int64_t, a_.i32[(i * 2) + 0]) * HEDLEY_STATIC_CAST(int64_t, b_.i32[(i * 2) + 0])) + c_.i64[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_macclo_epi16(a, b, c) simde_mm_macclo_epi16((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_macchi_epi32 (simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_macchi_epi32(a, b, c); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + int32x4_t even = vuzp2q_s32(a_.neon_i32, b_.neon_i32); + r_.neon_i64 = vaddq_s64(vmull_s32(vget_low_s32(even), vget_high_s32(even)), c_.neon_i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + r_.i64[i] = (HEDLEY_STATIC_CAST(int64_t, a_.i32[(i * 2) + 1]) * HEDLEY_STATIC_CAST(int64_t, b_.i32[(i * 2) + 1])) + c_.i64[i]; + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_macchi_epi16(a, b, c) simde_mm_macchi_epi16((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maccs_epi16 (simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_maccs_epi16(a, b, c); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + int32x4_t c_lo = vmovl_s16(vget_low_s16(c_.neon_i16)); + int32x4_t c_hi = vmovl_high_s16(c_.neon_i16); + int32x4_t lo = vmlal_s16(c_lo, vget_low_s16(a_.neon_i16), vget_low_s16(b_.neon_i16)); + int32x4_t hi = vmlal_high_s16(c_hi, a_.neon_i16, b_.neon_i16); + r_.neon_i16 = vcombine_s16(vqmovn_s32(lo), vqmovn_s32(hi)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + int32_t tmp = HEDLEY_STATIC_CAST(int32_t, a_.i16[i]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[i]); + tmp += c_.i16[i]; + if (tmp > INT16_MAX) + r_.i16[i] = INT16_MAX; + else if (tmp < INT16_MIN) + r_.i16[i] = INT16_MIN; + else + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, tmp); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_maccs_epi16(a, b, c) simde_mm_maccs_epi16((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maccs_epi32 (simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_maccs_epi32(a, b, c); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + int64x2_t c_lo = vmovl_s32(vget_low_s32(c_.neon_i32)); + int64x2_t c_hi = vmovl_high_s32(c_.neon_i32); + int64x2_t lo = vmlal_s32(c_lo, vget_low_s32(a_.neon_i32), vget_low_s32(b_.neon_i32)); + int64x2_t hi = vmlal_high_s32(c_hi, a_.neon_i32, b_.neon_i32); + r_.neon_i32 = vcombine_s32(vqmovn_s64(lo), vqmovn_s64(hi)); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + int64_t tmp = HEDLEY_STATIC_CAST(int64_t, a_.i32[i]) * HEDLEY_STATIC_CAST(int64_t, b_.i32[i]); + tmp += HEDLEY_STATIC_CAST(int64_t, c_.i32[i]); + if (tmp > INT32_MAX) + r_.i32[i] = INT32_MAX; + else if (tmp < INT32_MIN) + r_.i32[i] = INT32_MIN; + else + r_.i32[i] = HEDLEY_STATIC_CAST(int32_t, tmp); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_maccs_epi32(a, b, c) simde_mm_maccs_epi32((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maccsd_epi16 (simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_maccsd_epi16(a, b, c); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + #if defined(SIMDE_ARM_NEON_A64V8_NATIVE) + int16x8_t even = vuzp1q_s16(a_.neon_i16, b_.neon_i16); + r_.neon_i32 = vqaddq_s32(vmull_s16(vget_low_s16(even), vget_high_s16(even)), c_.neon_i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + int32_t prod = HEDLEY_STATIC_CAST(int32_t, a_.i16[i * 2]) * HEDLEY_STATIC_CAST(int32_t, b_.i16[i * 2]); + r_.i32[i] = simde_math_adds_i32(prod, c_.i32[i]); + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_maccsd_epi16(a, b, c) simde_mm_maccsd_epi16((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maccslo_epi32 (simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_maccslo_epi32(a, b, c); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + int64_t tmp = HEDLEY_STATIC_CAST(int64_t, a_.i32[(i * 2) + 0]) * HEDLEY_STATIC_CAST(int64_t, b_.i32[(i * 2) + 0]); + r_.i64[i] = simde_math_adds_i64(tmp, c_.i64[i]); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_maccslo_epi16(a, b, c) simde_mm_maccslo_epi16((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maccshi_epi32 (simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_maccshi_epi32(a, b, c); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + int64_t tmp = HEDLEY_STATIC_CAST(int64_t, a_.i32[(i * 2) + 1]) * HEDLEY_STATIC_CAST(int64_t, b_.i32[(i * 2) + 1]); + r_.i64[i] = simde_math_adds_i64(tmp, c_.i64[i]); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_maccshi_epi16(a, b, c) simde_mm_maccshi_epi16((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maddd_epi16 (simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_maddd_epi16(a, b, c); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + r_.i32[i] = + (a_.i16[(i * 2) + 0] * b_.i16[(i * 2) + 0]) + + (a_.i16[(i * 2) + 1] * b_.i16[(i * 2) + 1]); + r_.i32[i] += c_.i32[i]; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_maddd_epi16(a, b, c) simde_mm_maddd_epi16((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_maddsd_epi16 (simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_maddsd_epi16(a, b, c); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + /* The AMD64 Architecture Programmer's Manual says that "the" + * addition is saturated; I'm not sure whether that means + * the pairwise addition or the accumulate, or both. */ + r_.i32[i] = + (a_.i16[(i * 2) + 0] * b_.i16[(i * 2) + 0]) + + (a_.i16[(i * 2) + 1] * b_.i16[(i * 2) + 1]); + r_.i32[i] = simde_math_adds_i32(r_.i32[i], c_.i32[i]); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_maddsd_epi16(a, b, c) simde_mm_maddsd_epi16((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sha_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_sha_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i8 = vshlq_s8(a_.neon_i8, b_.neon_i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + if (b_.i8[i] < 0) { + r_.i8[i] = HEDLEY_STATIC_CAST(int8_t, a_.i8[i] >> -b_.i8[i]); + } else { + r_.i8[i] = HEDLEY_STATIC_CAST(int8_t, a_.i8[i] << b_.i8[i]); + } + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_sha_epi8(a, b) simde_mm_sha_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sha_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_sha_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i16 = vshlq_s16(a_.neon_i16, b_.neon_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i16) / sizeof(r_.i16[0])) ; i++) { + if (b_.i16[i] < 0) { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, a_.i16[i] >> -b_.i16[i]); + } else { + r_.i16[i] = HEDLEY_STATIC_CAST(int16_t, a_.i16[i] << b_.i16[i]); + } + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_sha_epi16(a, b) simde_mm_sha_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sha_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_sha_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i32 = vshlq_s32(a_.neon_i32, b_.neon_i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i32) / sizeof(r_.i32[0])) ; i++) { + if (b_.i32[i] < 0) { + r_.i32[i] = HEDLEY_STATIC_CAST(int32_t, a_.i32[i] >> -b_.i32[i]); + } else { + r_.i32[i] = HEDLEY_STATIC_CAST(int32_t, a_.i32[i] << b_.i32[i]); + } + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_sha_epi32(a, b) simde_mm_sha_epi32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_sha_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_sha_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_i64 = vshlq_s64(a_.neon_i64, b_.neon_i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i64) / sizeof(r_.i64[0])) ; i++) { + if (b_.i64[i] < 0) { + r_.i64[i] = HEDLEY_STATIC_CAST(int64_t, a_.i64[i] >> -b_.i64[i]); + } else { + r_.i64[i] = HEDLEY_STATIC_CAST(int64_t, a_.i64[i] << b_.i64[i]); + } + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_sha_epi64(a, b) simde_mm_sha_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_shl_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_shl_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u8 = vshlq_u8(a_.neon_u8, b_.neon_i8); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + if (HEDLEY_UNLIKELY(b_.i8[i] < -7 || b_.i8[i] > 7)) { + r_.u8[i] = 0; + } else { + if (b_.i8[i] < 0) { + r_.u8[i] = HEDLEY_STATIC_CAST(uint8_t, a_.u8[i] >> -b_.i8[i]); + } else { + r_.u8[i] = HEDLEY_STATIC_CAST(uint8_t, a_.u8[i] << b_.i8[i]); + } + } + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_shl_epi8(a, b) simde_mm_shl_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_shl_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_shl_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u16 = vshlq_u16(a_.neon_u16, b_.neon_i16); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + if (HEDLEY_UNLIKELY(b_.i16[i] < -15 || b_.i16[i] > 15)) { + r_.u16[i] = 0; + } else { + if (b_.i16[i] < 0) { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, a_.u16[i] >> -b_.i16[i]); + } else { + r_.u16[i] = HEDLEY_STATIC_CAST(uint16_t, a_.u16[i] << b_.i16[i]); + } + } + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_shl_epi16(a, b) simde_mm_shl_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_shl_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_shl_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u32 = vshlq_u32(a_.neon_u32, b_.neon_i32); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + if (HEDLEY_UNLIKELY(b_.i32[i] < -31 || b_.i32[i] > 31)) { + r_.u32[i] = 0; + } else { + if (b_.i32[i] < 0) { + r_.u32[i] = HEDLEY_STATIC_CAST(uint32_t, a_.u32[i] >> -b_.i32[i]); + } else { + r_.u32[i] = HEDLEY_STATIC_CAST(uint32_t, a_.u32[i] << b_.i32[i]); + } + } + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_shl_epi32(a, b) simde_mm_shl_epi32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_shl_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_shl_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + #if defined(SIMDE_ARM_NEON_A32V7_NATIVE) + r_.neon_u64 = vshlq_u64(a_.neon_u64, b_.neon_i64); + #else + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + if (HEDLEY_UNLIKELY(b_.i64[i] < -63 || b_.i64[i] > 63)) { + r_.u64[i] = 0; + } else { + if (b_.i64[i] < 0) { + r_.u64[i] = HEDLEY_STATIC_CAST(uint64_t, a_.u64[i] >> -b_.i64[i]); + } else { + r_.u64[i] = HEDLEY_STATIC_CAST(uint64_t, a_.u64[i] << b_.i64[i]); + } + } + } + #endif + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_shl_epi64(a, b) simde_mm_shl_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rot_epi8 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_rot_epi8(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (b_.i8[i] < 0) ? + HEDLEY_STATIC_CAST(uint8_t, ((a_.u8[i] >> -b_.i8[i]) | (a_.u8[i] << ( b_.i8[i] & 7)))) : + HEDLEY_STATIC_CAST(uint8_t, ((a_.u8[i] << b_.i8[i]) | (a_.u8[i] >> (-b_.i8[i] & 7)))); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_rot_epi8(a, b) simde_mm_rot_epi8((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rot_epi16 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_rot_epi16(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (b_.i16[i] < 0) ? + HEDLEY_STATIC_CAST(uint16_t, ((a_.u16[i] >> -b_.i16[i]) | (a_.u16[i] << ( b_.i16[i] & 15)))) : + HEDLEY_STATIC_CAST(uint16_t, ((a_.u16[i] << b_.i16[i]) | (a_.u16[i] >> (-b_.i16[i] & 15)))); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_rot_epi16(a, b) simde_mm_rot_epi16((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rot_epi32 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_rot_epi32(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (b_.i32[i] < 0) ? + HEDLEY_STATIC_CAST(uint32_t, ((a_.u32[i] >> -b_.i32[i]) | (a_.u32[i] << ( b_.i32[i] & 31)))) : + HEDLEY_STATIC_CAST(uint32_t, ((a_.u32[i] << b_.i32[i]) | (a_.u32[i] >> (-b_.i32[i] & 31)))); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_rot_epi32(a, b) simde_mm_rot_epi32((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_rot_epi64 (simde__m128i a, simde__m128i b) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_rot_epi64(a, b); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (b_.i64[i] < 0) ? + HEDLEY_STATIC_CAST(uint64_t, ((a_.u64[i] >> -b_.i64[i]) | (a_.u64[i] << ( b_.i64[i] & 63)))) : + HEDLEY_STATIC_CAST(uint64_t, ((a_.u64[i] << b_.i64[i]) | (a_.u64[i] >> (-b_.i64[i] & 63)))); + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_rot_epi64(a, b) simde_mm_rot_epi64((a), (b)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_roti_epi8 (simde__m128i a, const int count) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u8) / sizeof(r_.u8[0])) ; i++) { + r_.u8[i] = (count < 0) ? + HEDLEY_STATIC_CAST(uint8_t, ((a_.u8[i] >> -count) | (a_.u8[i] << ( count & 7)))) : + HEDLEY_STATIC_CAST(uint8_t, ((a_.u8[i] << count) | (a_.u8[i] >> (-count & 7)))); + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_XOP_NATIVE) + #define simde_mm_roti_epi8(a, count) _mm_roti_epi8((a), (count)) +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_roti_epi8(a, b) simde_mm_roti_epi8((a), (count)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_roti_epi16 (simde__m128i a, const int count) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u16) / sizeof(r_.u16[0])) ; i++) { + r_.u16[i] = (count < 0) ? + HEDLEY_STATIC_CAST(uint16_t, ((a_.u16[i] >> -count) | (a_.u16[i] << ( count & 15)))) : + HEDLEY_STATIC_CAST(uint16_t, ((a_.u16[i] << count) | (a_.u16[i] >> (-count & 15)))); + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_XOP_NATIVE) + #define simde_mm_roti_epi16(a, count) _mm_roti_epi16((a), (count)) +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_roti_epi16(a, count) simde_mm_roti_epi16((a), (count)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_roti_epi32 (simde__m128i a, const int count) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u32) / sizeof(r_.u32[0])) ; i++) { + r_.u32[i] = (count < 0) ? + HEDLEY_STATIC_CAST(uint32_t, ((a_.u32[i] >> -count) | (a_.u32[i] << ( count & 31)))) : + HEDLEY_STATIC_CAST(uint32_t, ((a_.u32[i] << count) | (a_.u32[i] >> (-count & 31)))); + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_XOP_NATIVE) + #define simde_mm_roti_epi32(a, count) _mm_roti_epi32((a), (count)) +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_roti_epi32(a, count) simde_mm_roti_epi32((a), (count)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_roti_epi64 (simde__m128i a, const int count) { + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.u64) / sizeof(r_.u64[0])) ; i++) { + r_.u64[i] = (count < 0) ? + HEDLEY_STATIC_CAST(uint64_t, ((a_.u64[i] >> -count) | (a_.u64[i] << ( count & 63)))) : + HEDLEY_STATIC_CAST(uint64_t, ((a_.u64[i] << count) | (a_.u64[i] >> (-count & 63)))); + } + + return simde__m128i_from_private(r_); +} +#if defined(SIMDE_X86_XOP_NATIVE) + #define simde_mm_roti_epi64(a, count) _mm_roti_epi64((a), (count)) +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_roti_epi64(a, count) simde_mm_roti_epi64((a), (count)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128i +simde_mm_perm_epi8 (simde__m128i a, simde__m128i b, simde__m128i c) { + #if defined(SIMDE_X86_XOP_NATIVE) + return _mm_perm_epi8(a, b, c); + #else + simde__m128i_private + r_, + a_ = simde__m128i_to_private(a), + b_ = simde__m128i_to_private(b), + c_ = simde__m128i_to_private(c); + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.i8) / sizeof(r_.i8[0])) ; i++) { + int8_t src = (c_.u8[i] & 0x10) ? b_.i8[c_.u8[i] & 0xf] : a_.i8[c_.u8[i] & 0xf]; + + switch (c_.u8[i] & 0xc0) { + case 0x40: + #if HEDLEY_HAS_BUILTIN(__builtin_bitreverse8) && !defined(HEDLEY_IBM_VERSION) + src = HEDLEY_STATIC_CAST(int8_t, __builtin_bitreverse8(HEDLEY_STATIC_CAST(uint8_t, src))); + #else + src = HEDLEY_STATIC_CAST(int8_t, ((HEDLEY_STATIC_CAST(uint8_t, src) * UINT64_C(0x80200802)) & UINT64_C(0x0884422110)) * UINT64_C(0x0101010101) >> 32); + #endif + break; + case 0x80: + src = 0; + break; + case 0xc0: + src >>= 7; + break; + } + + r_.i8[i] = (c_.u8[i] & 0x20) ? ~src : src; + } + + return simde__m128i_from_private(r_); + #endif +} +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_perm_epi8(a, b, c) simde_mm_perm_epi8((a), (b), (c)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128 +simde_mm_permute2_ps (simde__m128 a, simde__m128 b, simde__m128i c, const int imm8) { + simde__m128_private + r_, + a_ = simde__m128_to_private(a), + b_ = simde__m128_to_private(b); + simde__m128i_private c_ = simde__m128i_to_private(c); + + const int m2z = imm8 & 0x03; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + const int sel = c_.i32[i] & 0x07; + const int m = c_.i32[i] & 0x08; + + switch (m | m2z) { + case 0xa: + case 0x3: + r_.i32[i] = 0; + break; + default: + r_.i32[i] = (sel > 3) ? b_.i32[sel - 4] : a_.i32[sel]; + break; + } + } + + return simde__m128_from_private(r_); +} +#if defined(SIMDE_X86_XOP_NATIVE) + #if defined(HEDLEY_MCST_LCC_VERSION) + #define simde_mm_permute2_ps(a, b, c, imm8) (__extension__ ({ \ + SIMDE_LCC_DISABLE_DEPRECATED_WARNINGS \ + _mm_permute2_ps((a), (b), (c), (imm8)); \ + SIMDE_LCC_REVERT_DEPRECATED_WARNINGS \ + })) + #else + #define simde_mm_permute2_ps(a, b, c, imm8) _mm_permute2_ps((a), (b), (c), (imm8)) + #endif +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_permute2_ps(a, b, c, imm8) simde_mm_permute2_ps((a), (b), (c), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m128d +simde_mm_permute2_pd (simde__m128d a, simde__m128d b, simde__m128i c, const int imm8) { + simde__m128d_private + r_, + a_ = simde__m128d_to_private(a), + b_ = simde__m128d_to_private(b); + simde__m128i_private c_ = simde__m128i_to_private(c); + + const int m2z = imm8 & 0x03; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + const int sel = (c_.i64[i] & 0x06) >> 1; + const int m = c_.i64[i] & 0x08; + + switch (m | m2z) { + case 0x0a: + case 0x03: + r_.i64[i] = 0; + break; + default: + r_.i64[i] = (sel > 1) ? b_.i64[sel - 2] : a_.i64[sel]; + break; + } + } + + return simde__m128d_from_private(r_); +} + +#if defined(SIMDE_X86_XOP_NATIVE) + #if defined(HEDLEY_MCST_LCC_VERSION) + #define simde_mm_permute2_pd(a, b, c, imm8) (__extension__ ({ \ + SIMDE_LCC_DISABLE_DEPRECATED_WARNINGS \ + _mm_permute2_pd((a), (b), (c), (imm8)); \ + SIMDE_LCC_REVERT_DEPRECATED_WARNINGS \ + })) + #else + #define simde_mm_permute2_pd(a, b, c, imm8) _mm_permute2_pd((a), (b), (c), (imm8)) + #endif +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm_permute2_pd(a, b, c, imm8) simde_mm_permute2_pd((a), (b), (c), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256 +simde_mm256_permute2_ps (simde__m256 a, simde__m256 b, simde__m256i c, const int imm8) { + simde__m256_private + r_, + a_ = simde__m256_to_private(a), + b_ = simde__m256_to_private(b); + simde__m256i_private c_ = simde__m256i_to_private(c); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128) / sizeof(r_.m128[0])) ; i++) { + r_.m128[i] = simde_mm_permute2_ps(a_.m128[i], b_.m128[i], c_.m128i[i], imm8); + } + #else + const int m2z = imm8 & 0x03; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f32) / sizeof(r_.f32[0])) ; i++) { + const int sel = c_.i32[i] & 0x07; + const int m = c_.i32[i] & 0x08; + + switch (m | m2z) { + case 0xa: + case 0x3: + r_.i32[i] = 0; + break; + default: + r_.i32[i] = (sel > 3) ? b_.i32[sel + (HEDLEY_STATIC_CAST(int, i) & 4) - 4] : a_.i32[sel + (HEDLEY_STATIC_CAST(int, i) & 4)]; + break; + } + } + #endif + + return simde__m256_from_private(r_); +} + +#if defined(SIMDE_X86_XOP_NATIVE) + #if defined(HEDLEY_MCST_LCC_VERSION) + #define simde_mm256_permute2_ps(a, b, c, imm8) (__extension__ ({ \ + SIMDE_LCC_DISABLE_DEPRECATED_WARNINGS \ + _mm256_permute2_ps((a), (b), (c), (imm8)); \ + SIMDE_LCC_REVERT_DEPRECATED_WARNINGS \ + })) + #else + #define simde_mm256_permute2_ps(a, b, c, imm8) _mm256_permute2_ps((a), (b), (c), (imm8)) + #endif +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm256_permute2_ps(a, b, c, imm8) simde_mm256_permute2_ps((a), (b), (c), (imm8)) +#endif + +SIMDE_FUNCTION_ATTRIBUTES +simde__m256d +simde_mm256_permute2_pd (simde__m256d a, simde__m256d b, simde__m256i c, const int imm8) { + simde__m256d_private + r_, + a_ = simde__m256d_to_private(a), + b_ = simde__m256d_to_private(b); + simde__m256i_private c_ = simde__m256i_to_private(c); + + #if SIMDE_NATURAL_VECTOR_SIZE_LE(128) + for (size_t i = 0 ; i < (sizeof(r_.m128d) / sizeof(r_.m128d[0])) ; i++) { + r_.m128d[i] = simde_mm_permute2_pd(a_.m128d[i], b_.m128d[i], c_.m128i[i], imm8); + } + #else + const int m2z = imm8 & 0x03; + + SIMDE_VECTORIZE + for (size_t i = 0 ; i < (sizeof(r_.f64) / sizeof(r_.f64[0])) ; i++) { + const int sel = (c_.i64[i] & 0x06) >> 1; + const int m = c_.i64[i] & 0x08; + + switch (m | m2z) { + case 0x0a: + case 0x03: + r_.i64[i] = 0; + break; + default: + r_.i64[i] = (sel > 1) ? b_.i64[sel + (HEDLEY_STATIC_CAST(int, i) & 2) - 2] : a_.i64[sel + (HEDLEY_STATIC_CAST(int, i) & 2)]; + break; + } + } + #endif + + return simde__m256d_from_private(r_); +} +#if defined(SIMDE_X86_XOP_NATIVE) + #if defined(HEDLEY_MCST_LCC_VERSION) + #define simde_mm256_permute2_pd(a, b, c, imm8) (__extension__ ({ \ + SIMDE_LCC_DISABLE_DEPRECATED_WARNINGS \ + _mm256_permute2_pd((a), (b), (c), (imm8)); \ + SIMDE_LCC_REVERT_DEPRECATED_WARNINGS \ + })) + #else + #define simde_mm256_permute2_pd(a, b, c, imm8) _mm256_permute2_pd((a), (b), (c), (imm8)) + #endif +#endif +#if defined(SIMDE_X86_XOP_ENABLE_NATIVE_ALIASES) + #define _mm256_permute2_pd(a, b, c, imm8) simde_mm256_permute2_pd((a), (b), (c), (imm8)) +#endif + +HEDLEY_DIAGNOSTIC_POP +SIMDE_END_DECLS_ + +#endif /* !defined(SIMDE_X86_XOP_H) */