#include "IGenARM64.h"

#include <tuple>

#include "common/arm64/encoding.h"
#include "common/util/Assert.h"

#include "goalc/emitter/IGen.h"
#include "goalc/emitter/Instruction.h"
#include "goalc/emitter/InstructionSet.h"
#include "goalc/emitter/Register.h"

#include "fmt/base.h"

// https://armconverter.com/?code=ret
// https://developer.arm.com/documentation/ddi0487/latest

namespace emitter {
namespace IGen {
namespace ARM64 {

const auto instr_set = emitter::InstructionSet::ARM64;
using namespace emitter::ARM64;

// Utility functions (not public facing instructions)
// used to encode instructions and match the same API

// Checks whether or not an immediate can be represented in 12 unsigned bits, either:
// - plain [0-4095] immediate
// - imm << 12 (some multiple of 4096)
std::tuple<bool, u16, bool> can_encode_single_imm12(u64 imm) {
  if (imm < 4096) {
    return {true, static_cast<u16>(imm), false};
  }
  if ((imm & 0xFFF) == 0) {  // divisible by 4096
    u64 upper = imm >> 12;
    if (upper < 4096) {
      return {true, static_cast<uint16_t>(upper), true};
    }
  }
  return {false, 0, false};
}

// TODO - imm12 decomposition produces way too many chunks for what will be common operations,
// update the instructions that we can to instead use the movz/movk pattern
//
// Decompose a 64-bit immediate into 16-bit chunks suitable for movz/movk.
// Returns {chunk, shift} pairs where shift is one of 0,16,32,48.
std::vector<std::tuple<u16, u8>> decompose_into_imm16_chunks(u64 imm) {
  std::vector<std::tuple<u16, u8>> result;
  if (imm == 0) {
    result.emplace_back(0, 0);
    return result;
  }
  for (u8 shift = 0; shift <= 48; shift += 16) {
    u16 chunk = static_cast<u16>((imm >> shift) & 0xFFFF);
    if (chunk != 0 || result.empty()) {
      result.emplace_back(chunk, shift / 16);
    }
  }
  return result;
}

// Given a larger than u12 immediate, decompose it into multiple (shifted or not)
// immediates that can be used to emit multiple instructions to produce the desired outcome
std::vector<std::tuple<u16, bool>> decompose_into_imm12_chunks(u64 imm) {
  std ::vector<std::tuple<u16, bool>> result;
  u64 upper = imm >> 12;
  while (upper > 0) {
    u16 chunk = (upper > 4095) ? 4095 : static_cast<u16>(upper);
    result.emplace_back(chunk, true);
    upper -= chunk;
  }

  u16 lower = imm & 0xFFF;
  if (lower > 0) {
    result.emplace_back(lower, false);
  }
  return result;
}

std::vector<InstructionARM64> construct_multiple_imm12_adds(int64_t imm, u32 register_id) {
  const auto chunks = decompose_into_imm12_chunks(imm);
  std::vector<InstructionARM64> instrs;
  for (const auto& [_imm12, _needs_shift] : chunks) {
    // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html
    // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
    instrs.emplace_back(InstructionARM64(Base(0b100100010, 9), Sh(_needs_shift ? 1 : 0),
                                         Imm12(_imm12), Rd(register_id), Rn(register_id)));
  }
  return instrs;
}

std::vector<InstructionARM64> construct_multiple_imm12_subs(int64_t imm, u32 register_id) {
  const auto chunks = decompose_into_imm12_chunks(imm);
  std::vector<InstructionARM64> instrs;
  for (const auto& [_imm12, _needs_shift] : chunks) {
    // https://www.scs.stanford.edu/~zyedidia/arm64/sub_addsub_imm.html
    // SUB <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
    instrs.emplace_back(InstructionARM64(Base(0b110100010, 9), Sh(_needs_shift ? 1 : 0),
                                         Imm12(_imm12), Rd(register_id), Rn(register_id)));
  }
  return instrs;
}

//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
//   MOVES
//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

InstructionARM64 mov_gpr64_gpr64(Register dst, Register src) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(src.is_gpr(instr_set));
  if (dst == SP || src == SP) {
    // use ADD with a zero immediate because ORR treats register 31 as XZR instead of SP
    // https://www.scs.stanford.edu/~zyedidia/arm64/mov_add_addsub_imm.html
    return InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rn(src.hw_id(instr_set)),
                            Rd(dst.hw_id(instr_set)));
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/mov_orr_log_shift.html
  // MOV <Xd>, <Xm>
  return InstructionARM64(Base(0b10101010000, 11), Rm(src.hw_id(instr_set)), Rn(0b11111),
                          Rd(dst.hw_id(instr_set)), Imm6(0));
}

std::vector<InstructionARM64> mov_gpr64_u64_instrs(Register dst, uint64_t val) {
  // Cannot be done in a single instruction, must combine multiple MOVZ/MOVKs
  std::vector<InstructionARM64> instrs;
  const auto reg = u32(dst.hw_id(instr_set));
  auto imm_chunks = decompose_into_imm16_chunks(val);
  for (const auto& [imm_chunk, shift] : imm_chunks) {
    if (shift == 0) {
      // https://www.scs.stanford.edu/~zyedidia/arm64/movz.html
      // MOVZ <Xd>, #<imm>{, LSL #<shift>/16}
      instrs.emplace_back(arm64::encode_movz_64(reg, imm_chunk, shift));
    } else {
      // https://www.scs.stanford.edu/~zyedidia/arm64/movk.html
      // MOVK <Xd>, #<imm>{, LSL #<shift>/16}
      instrs.emplace_back(arm64::encode_movk_64(reg, imm_chunk, shift));
    }
  }
  return instrs;
}

InstructionARM64 mov_gpr64_u64(Register dst, uint64_t val) {
  return InstructionARM64(mov_gpr64_u64_instrs(dst, val));
}

InstructionARM64 mov_gpr64_u32(Register dst, uint64_t val) {
  return mov_gpr64_u64(dst, val);
}

InstructionARM64 mov_gpr64_s32(Register dst, int64_t val) {
  // preserve sign -- but we are are simply moving the bits
  u64 raw_val = static_cast<u64>(val);  // via int64_t → sign already there
  return mov_gpr64_u64(dst, raw_val);
}

InstructionARM64 mov_gpr32_link_imm32(Register dst, u32 initial) {
  ASSERT(dst.is_gpr(instr_set));
  // always emit both words so the linker patch size stays fixed
  // https://www.scs.stanford.edu/~zyedidia/arm64/movz.html
  // MOVZ <Wd>, #<imm16>
  auto movz = InstructionARM64(Base(0b010100101, 9), Hw(0), Imm16(initial & 0xffff),
                               Rd(dst.hw_id(instr_set)));
  // https://www.scs.stanford.edu/~zyedidia/arm64/movk.html
  // MOVK <Wd>, #<imm16>, LSL #16
  auto movk =
      InstructionARM64(Base(0b011100101, 9), Hw(1), Imm16(initial >> 16), Rd(dst.hw_id(instr_set)));
  return InstructionARM64(movz, movk).with_reloc(emitter::ARM64::RelocKind::MOV32);
}

InstructionARM64 add_gpr64_gpr64_sxtw(Register dst, Register base, Register idx) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(base.is_gpr(instr_set));
  ASSERT(idx.is_gpr(instr_set));
  // SXTW is option 0b110 with no shift
  // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html
  // ADD <Xd|SP>, <Xn|SP>, <Wm>, SXTW
  return InstructionARM64(Base(0b10001011001000001100000000000000, 32), Rm(idx.hw_id(instr_set)),
                          Rn(base.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 movd_gpr32_f32(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fmov_float_gen.html
  // Single-precision to 32-bit (sf == 0 && ftype == 00 && rmode == 00 && opcode == 110)
  // FMOV <Wd>, <Sn>
  ASSERT(dst.is_gpr(instr_set));
  return InstructionARM64(Base(0b0001111000100110000000, 22), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)));
}

InstructionARM64 movd_f32_gpr32(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fmov_float_gen.html
  // 32-bit to single-precision (sf == 0 && ftype == 00 && rmode == 00 && opcode == 111)
  // FMOV <Sd>, <Wn>
  ASSERT(src.is_gpr(instr_set));
  return InstructionARM64(Base(0b0001111000100111000000, 22), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)));
}

InstructionARM64 movq_gpr64_f64(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fmov_float_gen.html
  // Double-precision to 64-bit (sf == 1 && ftype == 01 && rmode == 00 && opcode == 110)
  // FMOV <Xd>, <Dn>
  ASSERT(dst.is_gpr(instr_set));
  return InstructionARM64(Base(0b1001111001100110000000, 22), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)));
}

InstructionARM64 movq_f64_gpr64(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fmov_float_gen.html
  // 64-bit to double-precision (sf == 1 && ftype == 01 && rmode == 00 && opcode == 111)
  // FMOV <Xd>, <Dn>
  ASSERT(src.is_gpr(instr_set));
  return InstructionARM64(Base(0b1001111001100111000000, 22), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)));
}

InstructionARM64 mov_f32_f32(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fmov_float.html
  // Single-precision (ftype == 00)
  // FMOV <Sd>, <Sn>
  return InstructionARM64(Base(0b0001111000100000010000, 22), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)));
}

//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
//   GOAL Loads and Stores
//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

InstructionARM64 load8s_gpr64_gpr64_plus_gpr64(Register dst, Register addr1, Register addr2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldrsb_reg.html
  // 64-bit with extended register offset (opc == 10 && option != 011)
  // LDRSB <Xt>, [<Xn|SP>, (<Wm>|<Xm>), <extend> {<amount>}]
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  return InstructionARM64(Base(0b0011100010100000111010, 22), Rt(dst.hw_id(instr_set)),
                          Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set)));
}

InstructionARM64 store8_gpr64_gpr64_plus_gpr64(Register addr1, Register addr2, Register value) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/strb_reg.html
  // 64 bit - SXTX
  // strb Wt, [Xn, Xm]
  ASSERT(value.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  return InstructionARM64(Base(0b0011100000100000111010, 22), Rt(value.hw_id(instr_set)),
                          Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set)));
}

// x16 is reserved as an emitter scratch register.

InstructionARM64 load8s_gpr64_gpr64_plus_gpr64_plus_s8(Register dst,
                                                       Register addr1,
                                                       Register addr2,
                                                       s64 offset) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html
  // ADD <Xd|SP>, <Xn|SP>, <R><m>{, <extend> {#<amount>}}
  return InstructionARM64({InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16),
                                            Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))),
                           // https://www.scs.stanford.edu/~zyedidia/arm64/ldursb.html
                           // LDURSB <Xt>, [<Xn|SP>{, #<simm>}]
                           InstructionARM64(Base(0b0011100010000000000000, 22), Imm9s(offset),
                                            Rt(dst.hw_id(instr_set)), Rn(X16))});
}

InstructionARM64 store8_gpr64_gpr64_plus_gpr64_plus_s8(Register addr1,
                                                       Register addr2,
                                                       Register value,
                                                       s64 offset) {
  ASSERT(value.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  return InstructionARM64({// https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html
                           // ADD <Xd|SP>, <Xn|SP>, <R><m>{, <extend> {#<amount>}}
                           InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16),
                                            Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))),
                           // https://www.scs.stanford.edu/~zyedidia/arm64/sturb.html
                           // STURB <Wt>, [<Xn|SP>{, #<simm>}]
                           InstructionARM64(Base(0b0011100000000000000000, 22), Imm9s(offset),
                                            Rt(value.hw_id(instr_set)), Rn(X16))});
}

InstructionARM64 load8s_gpr64_gpr64_plus_gpr64_plus_s32(Register dst,
                                                        Register addr1,
                                                        Register addr2,
                                                        s64 offset) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // finally do the load
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldrsb_imm.html
  // LDRSB <Xt>, [<Xn|SP>], #<simm>
  instrs.emplace_back(
      InstructionARM64(Base(0b0011100110, 10), Imm12(0), Rt(dst.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 store8_gpr64_gpr64_plus_gpr64_plus_s32(Register addr1,
                                                        Register addr2,
                                                        Register value,
                                                        s64 offset) {
  ASSERT(value.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/strb_imm.html
  // unsigned offset
  // STRB <Wt>, [<Xn|SP>{, #<pimm>}]
  instrs.emplace_back(
      InstructionARM64(Base(0b0011100100, 10), Imm12(0), Rt(value.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load8u_gpr64_gpr64_plus_gpr64(Register dst, Register addr1, Register addr2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldrb_reg.html
  // SXTX extend option
  // LDRB <Wt>, [<Xn|SP>, <Xm>{, LSL <amount>}]
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  return InstructionARM64(Base(0b0011100001100000111010, 22), Rt(dst.hw_id(instr_set)),
                          Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set)));
}

InstructionARM64 load8u_gpr64_gpr64_plus_gpr64_plus_s8(Register dst,
                                                       Register addr1,
                                                       Register addr2,
                                                       s64 offset) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  std::vector<InstructionARM64> instrs;
  if (offset > 0) {
    instrs = {
        // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html
        // ADD <Xd|SP>, <Xn|SP>, <R><m>{, <extend> {#<amount>}}
        InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), Rn(addr1.hw_id(instr_set)),
                         Rm(addr2.hw_id(instr_set))),
        // https://www.scs.stanford.edu/~zyedidia/arm64/ldrb_imm.html
        // Unsigned offset mode
        // LDRB <Xt>, [<Xn|SP>], #<simm>
        InstructionARM64(Base(0b0011100101, 10), Imm12(offset), Rt(dst.hw_id(instr_set)), Rn(X16))};
  } else {
    // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html
    // ADD <Xd|SP>, <Xn|SP>, <R><m>{, <extend> {#<amount>}}
    instrs = {InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16),
                               Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))),
              // https://www.scs.stanford.edu/~zyedidia/arm64/ldurb.html
              // LDURB <Xt>, [<Xn|SP>{, #<simm>}]
              InstructionARM64(Base(0b0011100001000000000000, 22), Imm9s(offset),
                               Rt(dst.hw_id(instr_set)), Rn(X16))};
  }
  return InstructionARM64(instrs);
}

InstructionARM64 load8u_gpr64_gpr64_plus_gpr64_plus_s32(Register dst,
                                                        Register addr1,
                                                        Register addr2,
                                                        s64 offset) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  // TODO - movk instead eventually
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // finally do the load
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldrb_imm.html
  // LDRB <Xt>, [<Xn|SP>], #<simm>
  instrs.emplace_back(
      InstructionARM64(Base(0b0011100101, 10), Imm12(0), Rt(dst.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load16s_gpr64_gpr64_plus_gpr64(Register dst, Register addr1, Register addr2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldrsh_reg.html
  // LDRSH <Xt>, [<Xn|SP>, (<Wm>|<Xm>){, <extend> {<amount>}}]
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  return InstructionARM64(Base(0b0111100010100000111010, 22), Rt(dst.hw_id(instr_set)),
                          Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set)));
}

InstructionARM64 store16_gpr64_gpr64_plus_gpr64(Register addr1, Register addr2, Register value) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/strh_reg.html
  // STRH <Wt>, [<Xn|SP>, (<Wm>|<Xm>){, <extend> {<amount>}}]
  ASSERT(value.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  return InstructionARM64(Base(0b0111100000100000111010, 22), Rt(value.hw_id(instr_set)),
                          Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set)));
}

InstructionARM64 store16_gpr64_gpr64_plus_gpr64_plus_s8(Register addr1,
                                                        Register addr2,
                                                        Register value,
                                                        s64 offset) {
  ASSERT(value.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  return InstructionARM64({// https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html
                           // ADD <Xd|SP>, <Xn|SP>, <R><m>{, <extend> {#<amount>}}
                           InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16),
                                            Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))),
                           // https://www.scs.stanford.edu/~zyedidia/arm64/sturh.html
                           // STURH <Wt>, [<Xn|SP>{, #<simm>}]
                           InstructionARM64(Base(0b0111100000000000000000, 22), Imm9s(offset),
                                            Rt(value.hw_id(instr_set)), Rn(X16))});
}

InstructionARM64 store16_gpr64_gpr64_plus_gpr64_plus_s32(Register addr1,
                                                         Register addr2,
                                                         Register value,
                                                         s64 offset) {
  ASSERT(value.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // finally do the load
  // https://www.scs.stanford.edu/~zyedidia/arm64/strh_imm.html
  // STRH <Wt>, [<Xn|SP>{, #<pimm>}]
  instrs.emplace_back(
      InstructionARM64(Base(0b0111100100, 10), Imm12(0), Rt(value.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load16s_gpr64_gpr64_plus_gpr64_plus_s8(Register dst,
                                                        Register addr1,
                                                        Register addr2,
                                                        s64 offset) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html
  // ADD <Xd|SP>, <Xn|SP>, <R><m>{, <extend> {#<amount>}}
  return InstructionARM64({InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16),
                                            Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))),
                           // https://www.scs.stanford.edu/~zyedidia/arm64/ldursh.html
                           // LDURSH <Xt>, [<Xn|SP>{, #<simm>}]
                           InstructionARM64(Base(0b0111100010000000000000, 22), Imm9s(offset),
                                            Rt(dst.hw_id(instr_set)), Rn(X16))});
}

InstructionARM64 load16s_gpr64_gpr64_plus_gpr64_plus_s32(Register dst,
                                                         Register addr1,
                                                         Register addr2,
                                                         s64 offset) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // finally do the load
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldrsh_imm.html
  // LDRSH <Xt>, [<Xn|SP>], #<simm>
  instrs.emplace_back(
      InstructionARM64(Base(0b0111100110, 10), Imm12(0), Rt(dst.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load16u_gpr64_gpr64_plus_gpr64(Register dst, Register addr1, Register addr2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldrh_reg.html
  // SXTX extend option
  // LDRH <Wt>, [<Xn|SP>, <Xm>{, LSL <amount>}]
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  return InstructionARM64(Base(0b0111100001100000111010, 22), Rt(dst.hw_id(instr_set)),
                          Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set)));
}

InstructionARM64 load16u_gpr64_gpr64_plus_gpr64_plus_s8(Register dst,
                                                        Register addr1,
                                                        Register addr2,
                                                        s64 offset) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // LDURH covers the full signed byte offset range
  // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html
  // ADD <Xd|SP>, <Xn|SP>, <R><m>{, <extend> {#<amount>}}
  return InstructionARM64({InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16),
                                            Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))),
                           // https://www.scs.stanford.edu/~zyedidia/arm64/ldurh.html
                           // LDURH <Wt>, [<Xn|SP>{, #<simm>}]
                           InstructionARM64(Base(0b0111100001000000000000, 22), Imm9s(offset),
                                            Rt(dst.hw_id(instr_set)), Rn(X16))});
}

InstructionARM64 load16u_gpr64_gpr64_plus_gpr64_plus_s32(Register dst,
                                                         Register addr1,
                                                         Register addr2,
                                                         s64 offset) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // finally do the load
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldrh_imm.html
  // LDRH <Wt>, [<Xn|SP>{, #<pimm>}]
  instrs.emplace_back(
      InstructionARM64(Base(0b0111100101, 10), Imm12(0), Rt(dst.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load32s_gpr64_gpr64_plus_gpr64(Register dst, Register addr1, Register addr2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldrsw_reg.html
  // LDRSW <Xt>, [<Xn|SP>, (<Wm>|<Xm>){, <extend> {<amount>}}]
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  return InstructionARM64(Base(0b1011100010100000111010, 22), Rt(dst.hw_id(instr_set)),
                          Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set)));
}

InstructionARM64 store32_gpr64_gpr64_plus_gpr64(Register addr1, Register addr2, Register value) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_reg_gen.html
  // STR <Wt>, [<Xn|SP>, (<Wm>|<Xm>){, <extend> {<amount>}}]
  ASSERT(value.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  return InstructionARM64(Base(0b1011100000100000111010, 22), Rt(value.hw_id(instr_set)),
                          Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set)));
}

InstructionARM64 load32s_gpr64_gpr64_plus_gpr64_plus_s8(Register dst,
                                                        Register addr1,
                                                        Register addr2,
                                                        s64 offset) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html
  // ADD <Xd|SP>, <Xn|SP>, <R><m>{, <extend> {#<amount>}}
  return InstructionARM64({InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16),
                                            Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))),
                           // https://www.scs.stanford.edu/~zyedidia/arm64/ldursw.html
                           // LDURSW <Xt>, [<Xn|SP>{, #<simm>}]
                           InstructionARM64(Base(0b1011100010000000000000, 22), Imm9s(offset),
                                            Rt(dst.hw_id(instr_set)), Rn(X16))});
}

InstructionARM64 store32_gpr64_gpr64_plus_gpr64_plus_s8(Register addr1,
                                                        Register addr2,
                                                        Register value,
                                                        s64 offset) {
  ASSERT(value.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_gen.html
  // STR <Wt>, [<Xn|SP>{, #<pimm>}]
  instrs.emplace_back(
      InstructionARM64(Base(0b1011100100, 10), Imm12(0), Rt(value.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load32s_gpr64_gpr64_plus_gpr64_plus_s32(Register dst,
                                                         Register addr1,
                                                         Register addr2,
                                                         s64 offset) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // finally do the load
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldrsw_imm.html
  // LDRSW <Xt>, [<Xn|SP>], #<simm>
  instrs.emplace_back(
      InstructionARM64(Base(0b1011100110, 10), Imm12(0), Rt(dst.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 store32_gpr64_gpr64_plus_gpr64_plus_s32(Register addr1,
                                                         Register addr2,
                                                         Register value,
                                                         s64 offset) {
  ASSERT(value.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_gen.html
  // STR <Wt>, [<Xn|SP>{, #<pimm>}]
  instrs.emplace_back(
      InstructionARM64(Base(0b1011100100, 10), Imm12(0), Rt(value.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load32u_gpr64_gpr64_plus_gpr64(Register dst, Register addr1, Register addr2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_reg_gen.html
  // 32-bit variant, SXTX
  // LDR <Wt>, [<Xn|SP>, (<Wm>|<Xm>){, <extend> {<amount>}}]
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  return InstructionARM64(Base(0b1011100001100000111010, 22), Rt(dst.hw_id(instr_set)),
                          Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set)));
}

InstructionARM64 load32u_gpr64_gpr64_plus_gpr64_plus_s8(Register dst,
                                                        Register addr1,
                                                        Register addr2,
                                                        s64 offset) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html
  // ADD <Xd|SP>, <Xn|SP>, <R><m>{, <extend> {#<amount>}}
  return InstructionARM64({InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16),
                                            Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))),
                           // https://www.scs.stanford.edu/~zyedidia/arm64/ldur_gen.html
                           // 32 bit
                           // LDUR <Wt>, [<Xn|SP>{, #<simm>}]
                           InstructionARM64(Base(0b1011100001000000000000, 22), Imm9s(offset),
                                            Rt(dst.hw_id(instr_set)), Rn(X16))});
}

InstructionARM64 load32u_gpr64_gpr64_plus_gpr64_plus_s32(Register dst,
                                                         Register addr1,
                                                         Register addr2,
                                                         s64 offset) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // finally do the load
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldur_gen.html
  // 32-bit variant
  // LDUR <Wt>, [<Xn|SP>{, #<simm>}]
  instrs.emplace_back(InstructionARM64(Base(0b1011100001000000000000, 22), Imm9s(0),
                                       Rt(dst.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load64_gpr64_gpr64_plus_gpr64(Register dst, Register addr1, Register addr2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_reg_gen.html
  // 64 bit mode, SXTX
  // LDR <Xt>, [<Xn|SP>, (<Wm>|<Xm>){, <extend> {<amount>}}]
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  return InstructionARM64(Base(0b1111100001100000111010, 22), Rt(dst.hw_id(instr_set)),
                          Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set)));
}

InstructionARM64 store64_gpr64_gpr64_plus_gpr64(Register addr1, Register addr2, Register value) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_reg_gen.html
  // STR <Xt>, [<Xn|SP>, (<Wm>|<Xm>){, <extend> {<amount>}}]
  ASSERT(value.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  return InstructionARM64(Base(0b1111100000100000111010, 22), Rt(value.hw_id(instr_set)),
                          Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set)));
}

InstructionARM64 load64_gpr64_gpr64_plus_gpr64_plus_s8(Register dst,
                                                       Register addr1,
                                                       Register addr2,
                                                       s64 offset) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html
  // ADD <Xd|SP>, <Xn|SP>, <R><m>{, <extend> {#<amount>}}
  return InstructionARM64({InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16),
                                            Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))),
                           // https://www.scs.stanford.edu/~zyedidia/arm64/ldur_gen.html
                           // 64 bit
                           // LDUR <Xt>, [<Xn|SP>{, #<simm>}]
                           InstructionARM64(Base(0b1111100001000000000000, 22), Imm9s(offset),
                                            Rt(dst.hw_id(instr_set)), Rn(X16))});
}

InstructionARM64 store64_gpr64_gpr64_plus_gpr64_plus_s8(Register addr1,
                                                        Register addr2,
                                                        Register value,
                                                        s64 offset) {
  ASSERT(value.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_gen.html
  // STR <Xt>, [<Xn|SP>{, #<pimm>}]
  instrs.emplace_back(
      InstructionARM64(Base(0b1111100100, 10), Imm12(0), Rt(value.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load64_gpr64_gpr64_plus_gpr64_plus_s32(Register dst,
                                                        Register addr1,
                                                        Register addr2,
                                                        s64 offset) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // finally do the load
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldur_gen.html
  // 64-bit variant
  // LDUR <Xt>, [<Xn|SP>{, #<simm>}]
  instrs.emplace_back(InstructionARM64(Base(0b1111100001000000000000, 22), Imm9s(0),
                                       Rt(dst.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 store64_gpr64_gpr64_plus_gpr64_plus_s32(Register addr1,
                                                         Register addr2,
                                                         Register value,
                                                         s64 offset) {
  ASSERT(value.is_gpr(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_gen.html
  // STR <Xt>, [<Xn|SP>{, #<pimm>}]
  instrs.emplace_back(
      InstructionARM64(Base(0b1111100100, 10), Imm12(0), Rt(value.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load64_gpr64_plus_s32(Register dst_reg, int32_t offset, Register src_reg) {
  ASSERT(dst_reg.is_gpr(instr_set));
  ASSERT(src_reg.is_gpr(instr_set));
  // allow SP as the base for spill-slot addressing with immediate ADD
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html
      // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
      InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16),
                       Rn(src_reg.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // finally do the load
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldur_gen.html
  // 64-bit variant
  // LDUR <Xt>, [<Xn|SP>{, #<simm>}]
  instrs.emplace_back(InstructionARM64(Base(0b1111100001000000000000, 22), Imm9s(0),
                                       Rt(dst_reg.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 store64_gpr64_plus_s32(Register addr, int32_t offset, Register value) {
  ASSERT(value.is_gpr(instr_set));
  ASSERT(addr.is_gpr(instr_set));
  // allow SP as the base for immediate ADD
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html
      // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
      InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(addr.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_gen.html
  // STR <Xt>, [<Xn|SP>{, #<pimm>}]
  instrs.emplace_back(
      InstructionARM64(Base(0b1111100100, 10), Imm12(0), Rt(value.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 store_goal_vf(Register addr, Register value, Register off, s64 offset) {
  if (offset == 0) {
    return storevf_gpr64_plus_gpr64(value, addr, off);
  } else if (offset >= INT8_MIN && offset <= INT8_MAX) {
    return storevf_gpr64_plus_gpr64_plus_s8(value, addr, off, offset);
  } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
    return storevf_gpr64_plus_gpr64_plus_s32(value, addr, off, offset);
  }
  ASSERT(false);
  return {0};
}

InstructionARM64 store_goal_gpr(Register addr, Register value, Register off, int offset, int size) {
  switch (size) {
    case 1:
      if (offset == 0) {
        return store8_gpr64_gpr64_plus_gpr64(addr, off, value);
      } else if (offset >= INT8_MIN && offset <= INT8_MAX) {
        return store8_gpr64_gpr64_plus_gpr64_plus_s8(addr, off, value, offset);
      } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
        return store8_gpr64_gpr64_plus_gpr64_plus_s32(addr, off, value, offset);
      } else {
        ASSERT(false);
      }
    case 2:
      if (offset == 0) {
        return store16_gpr64_gpr64_plus_gpr64(addr, off, value);
      } else if (offset >= INT8_MIN && offset <= INT8_MAX) {
        return store16_gpr64_gpr64_plus_gpr64_plus_s8(addr, off, value, offset);
      } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
        return store16_gpr64_gpr64_plus_gpr64_plus_s32(addr, off, value, offset);
      } else {
        ASSERT(false);
      }
    case 4:
      if (offset == 0) {
        return store32_gpr64_gpr64_plus_gpr64(addr, off, value);
      } else if (offset >= INT8_MIN && offset <= INT8_MAX) {
        return store32_gpr64_gpr64_plus_gpr64_plus_s8(addr, off, value, offset);
      } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
        return store32_gpr64_gpr64_plus_gpr64_plus_s32(addr, off, value, offset);
      } else {
        ASSERT(false);
      }
    case 8:
      if (offset == 0) {
        return store64_gpr64_gpr64_plus_gpr64(addr, off, value);
      } else if (offset >= INT8_MIN && offset <= INT8_MAX) {
        return store64_gpr64_gpr64_plus_gpr64_plus_s8(addr, off, value, offset);
      } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
        return store64_gpr64_gpr64_plus_gpr64_plus_s32(addr, off, value, offset);
      } else {
        ASSERT(false);
      }
    default:
      ASSERT(false);
      return {0};
  }
}

InstructionARM64 load_goal_simd128(Register dst, Register addr, Register off, int offset) {
  if (offset == 0) {
    return loadvf_gpr64_plus_gpr64(dst, addr, off);
  } else if (offset >= INT8_MIN && offset <= INT8_MAX) {
    return loadvf_gpr64_plus_gpr64_plus_s8(dst, addr, off, offset);
  } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
    return loadvf_gpr64_plus_gpr64_plus_s32(dst, addr, off, offset);
  } else {
    ASSERT(false);
    return {0};
  }
}

InstructionARM64 load_goal_gpr(Register dst,
                               Register addr,
                               Register off,
                               int offset,
                               int size,
                               bool sign_extend) {
  switch (size) {
    case 1:
      if (offset == 0) {
        if (sign_extend) {
          return load8s_gpr64_gpr64_plus_gpr64(dst, addr, off);
        } else {
          return load8u_gpr64_gpr64_plus_gpr64(dst, addr, off);
        }
      } else if (offset >= INT8_MIN && offset <= INT8_MAX) {
        if (sign_extend) {
          return load8s_gpr64_gpr64_plus_gpr64_plus_s8(dst, addr, off, offset);
        } else {
          return load8u_gpr64_gpr64_plus_gpr64_plus_s8(dst, addr, off, offset);
        }
      } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
        if (sign_extend) {
          return load8s_gpr64_gpr64_plus_gpr64_plus_s32(dst, addr, off, offset);
        } else {
          return load8u_gpr64_gpr64_plus_gpr64_plus_s32(dst, addr, off, offset);
        }
      } else {
        ASSERT(false);
      }
    case 2:
      if (offset == 0) {
        if (sign_extend) {
          return load16s_gpr64_gpr64_plus_gpr64(dst, addr, off);
        } else {
          return load16u_gpr64_gpr64_plus_gpr64(dst, addr, off);
        }
      } else if (offset >= INT8_MIN && offset <= INT8_MAX) {
        if (sign_extend) {
          return load16s_gpr64_gpr64_plus_gpr64_plus_s8(dst, addr, off, offset);
        } else {
          return load16u_gpr64_gpr64_plus_gpr64_plus_s8(dst, addr, off, offset);
        }
      } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
        if (sign_extend) {
          return load16s_gpr64_gpr64_plus_gpr64_plus_s32(dst, addr, off, offset);
        } else {
          return load16u_gpr64_gpr64_plus_gpr64_plus_s32(dst, addr, off, offset);
        }
      } else {
        ASSERT(false);
      }
    case 4:
      if (offset == 0) {
        if (sign_extend) {
          return load32s_gpr64_gpr64_plus_gpr64(dst, addr, off);
        } else {
          return load32u_gpr64_gpr64_plus_gpr64(dst, addr, off);
        }
      } else if (offset >= INT8_MIN && offset <= INT8_MAX) {
        if (sign_extend) {
          return load32s_gpr64_gpr64_plus_gpr64_plus_s8(dst, addr, off, offset);
        } else {
          return load32u_gpr64_gpr64_plus_gpr64_plus_s8(dst, addr, off, offset);
        }
      } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
        if (sign_extend) {
          return load32s_gpr64_gpr64_plus_gpr64_plus_s32(dst, addr, off, offset);
        } else {
          return load32u_gpr64_gpr64_plus_gpr64_plus_s32(dst, addr, off, offset);
        }
      } else {
        ASSERT(false);
      }
    case 8:
      if (offset == 0) {
        return load64_gpr64_gpr64_plus_gpr64(dst, addr, off);

      } else if (offset >= INT8_MIN && offset <= INT8_MAX) {
        return load64_gpr64_gpr64_plus_gpr64_plus_s8(dst, addr, off, offset);

      } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
        return load64_gpr64_gpr64_plus_gpr64_plus_s32(dst, addr, off, offset);

      } else {
        ASSERT(false);
      }
    default:
      ASSERT(false);
      return {0};
  }
}

InstructionARM64 lea_reg_plus_off32(Register dest, Register base, s64 offset) {
  ASSERT(dest.is_gpr(instr_set));
  ASSERT(base.is_gpr(instr_set));
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base value in our destination register
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html
      // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
      InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(dest.hw_id(instr_set)),
                       Rn(base.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, dest.hw_id(instr_set));
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, dest.hw_id(instr_set));
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  return InstructionARM64(instrs);
}

InstructionARM64 lea_reg_plus_off8(Register dest, Register base, s64 offset) {
  ASSERT(dest.is_gpr(instr_set));
  ASSERT(base.is_gpr(instr_set));
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // first establish the base value in our destination register
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html
      // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
      InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(dest.hw_id(instr_set)),
                       Rn(base.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, dest.hw_id(instr_set));
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, dest.hw_id(instr_set));
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  return InstructionARM64(instrs);
}

InstructionARM64 lea_reg_plus_off(Register dest, Register base, s64 offset) {
  if (offset >= INT8_MIN && offset <= INT8_MAX) {
    return lea_reg_plus_off8(dest, base, offset);
  } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
    return lea_reg_plus_off32(dest, base, offset);
  } else {
    ASSERT(false);
    return {0};
  }
}

//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
//   LOADS n' STORES - SIMD32
//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

// TODO - rename these to f32

InstructionARM64 store32_simd32_gpr64_plus_gpr64(Register addr1,
                                                 Register addr2,
                                                 Register simd_value) {
  ASSERT(simd_value.is_128bit_simd(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_reg_fpsimd.html
  // 32-bit variant
  // STR <St>, [<Xn|SP>, (<Wm>|<Xm>){, <extend> {<amount>}}]
  return InstructionARM64(Base(0b1011110000100000111010, 22), Rt(simd_value.hw_id(instr_set)),
                          Rm(addr2.hw_id(instr_set)), Rn(addr1.hw_id(instr_set)));
}

InstructionARM64 load32_simd32_gpr64_plus_gpr64(Register simd_dest,
                                                Register addr1,
                                                Register addr2) {
  ASSERT(simd_dest.is_128bit_simd(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_reg_fpsimd.html
  // 32-bit variant
  // LDR <St>, [<Xn|SP>, (<Wm>|<Xm>){, <extend> {<amount>}}]
  return InstructionARM64(Base(0b1011110001100000111010, 22), Rt(simd_dest.hw_id(instr_set)),
                          Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set)));
}

InstructionARM64 store32_simd32_gpr64_plus_gpr64_plus_s8(Register addr1,
                                                         Register addr2,
                                                         Register simd_value,
                                                         s64 offset) {
  ASSERT(simd_value.is_128bit_simd(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  // TODO - optimization, if its less than imm12 we can just do an add/sub
  auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset));
  for (const auto& instr : mov_instrs) {
    instrs.push_back(InstructionARM64(instr));
  }
  if (offset < 0) {
    instrs.push_back(sub_gpr64_gpr64(X16, X17));
  } else {
    instrs.push_back(add_gpr64_gpr64(X16, X17));
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html
  // 32-bit variant
  // STR <St>, [<Xn|SP>], #<simm>
  instrs.emplace_back(InstructionARM64(Base(0b1011110100000000000000, 22), Imm12(0),
                                       Rt(simd_value.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load32_simd32_gpr64_plus_gpr64_plus_s8(Register simd_dest,
                                                        Register addr1,
                                                        Register addr2,
                                                        s64 offset) {
  ASSERT(simd_dest.is_128bit_simd(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  // TODO - optimization, if its less than imm12 we can just do an add/sub
  auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset));
  for (const auto& instr : mov_instrs) {
    instrs.push_back(InstructionARM64(instr));
  }
  if (offset < 0) {
    instrs.push_back(sub_gpr64_gpr64(X16, X17));
  } else {
    instrs.push_back(add_gpr64_gpr64(X16, X17));
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_fpsimd.html
  // 32-bit variant
  // LDR <St>, [<Xn|SP>{, #<pimm>}]
  instrs.emplace_back(
      InstructionARM64(Base(0b1011110101, 10), Imm12(0), Rt(simd_dest.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 store32_simd32_gpr64_plus_gpr64_plus_s32(Register addr1,
                                                          Register addr2,
                                                          Register simd_value,
                                                          s64 offset) {
  ASSERT(simd_value.is_128bit_simd(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  // TODO - optimization, if its less than imm12 we can just do an add/sub
  auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset));
  for (const auto& instr : mov_instrs) {
    instrs.push_back(InstructionARM64(instr));
  }
  if (offset < 0) {
    instrs.push_back(sub_gpr64_gpr64(X16, X17));
  } else {
    instrs.push_back(add_gpr64_gpr64(X16, X17));
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html
  // 32-bit variant
  // STR <St>, [<Xn|SP>], #<simm>
  instrs.emplace_back(InstructionARM64(Base(0b1011110100000000000000, 22), Imm12(0),
                                       Rt(simd_value.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 store32_simd32_gpr64_plus_s32(Register base, Register simd_value, s64 offset) {
  ASSERT(simd_value.is_128bit_simd(instr_set));
  ASSERT(base.is_gpr(instr_set));
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html
      // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
      InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(base.hw_id(instr_set))),
  };
  // TODO - optimization, if its less than imm12 we can just do an add/sub
  auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset));
  for (const auto& instr : mov_instrs) {
    instrs.push_back(InstructionARM64(instr));
  }
  if (offset < 0) {
    instrs.push_back(sub_gpr64_gpr64(X16, X17));
  } else {
    instrs.push_back(add_gpr64_gpr64(X16, X17));
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html
  // 32-bit variant
  // STR <St>, [<Xn|SP>], #<simm>
  instrs.emplace_back(InstructionARM64(Base(0b1011110100000000000000, 22), Imm12(0),
                                       Rt(simd_value.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 store32_simd32_gpr64_plus_s8(Register base, Register simd_value, s64 offset) {
  ASSERT(simd_value.is_128bit_simd(instr_set));
  ASSERT(base.is_gpr(instr_set));
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html
      // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
      InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(base.hw_id(instr_set))),
  };
  // TODO - optimization, if its less than imm12 we can just do an add/sub
  auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset));
  for (const auto& instr : mov_instrs) {
    instrs.push_back(InstructionARM64(instr));
  }
  if (offset < 0) {
    instrs.push_back(sub_gpr64_gpr64(X16, X17));
  } else {
    instrs.push_back(add_gpr64_gpr64(X16, X17));
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html
  // 32-bit variant, unsigned
  // STR <St>, [<Xn|SP>], #<simm>
  instrs.emplace_back(InstructionARM64(Base(0b1011110100000000000000, 22), Imm12(0),
                                       Rt(simd_value.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load32_simd32_gpr64_plus_gpr64_plus_s32(Register simd_dest,
                                                         Register addr1,
                                                         Register addr2,
                                                         s64 offset) {
  ASSERT(simd_dest.is_128bit_simd(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  // TODO - optimization, if its less than imm12 we can just do an add/sub
  auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset));
  for (const auto& instr : mov_instrs) {
    instrs.push_back(InstructionARM64(instr));
  }
  if (offset < 0) {
    instrs.push_back(sub_gpr64_gpr64(X16, X17));
  } else {
    instrs.push_back(add_gpr64_gpr64(X16, X17));
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_fpsimd.html
  // 32-bit variant
  // LDR <St>, [<Xn|SP>{, #<pimm>}]
  instrs.emplace_back(
      InstructionARM64(Base(0b1011110101, 10), Imm12(0), Rt(simd_dest.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load32_simd32_gpr64_plus_s32(Register simd_dest, Register base, s64 offset) {
  ASSERT(simd_dest.is_128bit_simd(instr_set));
  ASSERT(base.is_gpr(instr_set));
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html
      // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
      InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(base.hw_id(instr_set))),
  };
  // TODO - optimization, if its less than imm12 we can just do an add/sub
  auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset));
  for (const auto& instr : mov_instrs) {
    instrs.push_back(InstructionARM64(instr));
  }
  if (offset < 0) {
    instrs.push_back(sub_gpr64_gpr64(X16, X17));
  } else {
    instrs.push_back(add_gpr64_gpr64(X16, X17));
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_fpsimd.html
  // 32-bit variant
  // LDR <St>, [<Xn|SP>{, #<pimm>}]
  instrs.emplace_back(
      InstructionARM64(Base(0b1011110101, 10), Imm12(0), Rt(simd_dest.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load32_simd32_gpr64_plus_s8(Register simd_dest, Register base, s64 offset) {
  ASSERT(simd_dest.is_128bit_simd(instr_set));
  ASSERT(base.is_gpr(instr_set));
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html
      // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
      InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(base.hw_id(instr_set))),
  };
  // TODO - optimization, if its less than imm12 we can just do an add/sub
  auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset));
  for (const auto& instr : mov_instrs) {
    instrs.push_back(InstructionARM64(instr));
  }
  if (offset < 0) {
    instrs.push_back(sub_gpr64_gpr64(X16, X17));
  } else {
    instrs.push_back(add_gpr64_gpr64(X16, X17));
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_fpsimd.html
  // 32-bit variant
  // LDR <St>, [<Xn|SP>{, #<pimm>}]
  instrs.emplace_back(
      InstructionARM64(Base(0b1011110101, 10), Imm12(0), Rt(simd_dest.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load_goal_simd32(Register simd_dest, Register addr, Register off, s64 offset) {
  if (offset == 0) {
    return load32_simd32_gpr64_plus_gpr64(simd_dest, addr, off);
  } else if (offset >= INT8_MIN && offset <= INT8_MAX) {
    return load32_simd32_gpr64_plus_gpr64_plus_s8(simd_dest, addr, off, offset);
  } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
    return load32_simd32_gpr64_plus_gpr64_plus_s32(simd_dest, addr, off, offset);
  } else {
    ASSERT(false);
    return {0};
  }
}

InstructionARM64 store_goal_simd32(Register addr, Register simd_value, Register off, s64 offset) {
  if (offset == 0) {
    return store32_simd32_gpr64_plus_gpr64(addr, off, simd_value);
  } else if (offset >= INT8_MIN && offset <= INT8_MAX) {
    return store32_simd32_gpr64_plus_gpr64_plus_s8(addr, off, simd_value, offset);
  } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
    return store32_simd32_gpr64_plus_gpr64_plus_s32(addr, off, simd_value, offset);
  } else {
    ASSERT(false);
    return {0};
  }
}

InstructionARM64 store_reg_offset_simd32(Register base, Register simd_value, s64 offset) {
  if (offset >= INT8_MIN && offset <= INT8_MAX) {
    return store32_simd32_gpr64_plus_s8(base, simd_value, offset);
  } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
    return store32_simd32_gpr64_plus_s32(base, simd_value, offset);
  } else {
    ASSERT(false);
    return {0};
  }
}

InstructionARM64 load_reg_offset_simd32(Register simd_dest, Register base, s64 offset) {
  if (offset >= INT8_MIN && offset <= INT8_MAX) {
    return load32_simd32_gpr64_plus_s8(simd_dest, base, offset);
  } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
    return load32_simd32_gpr64_plus_s32(simd_dest, base, offset);
  } else {
    ASSERT(false);
    return {0};
  }
}

//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
//   LOADS n' STORES - SIMD (128-bit, QWORDS)
//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

InstructionARM64 store128_gpr64_simd128(Register gpr_addr, Register simd_reg) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html
  // - STR Qn, [Xn] (unsigned offset)
  ASSERT(gpr_addr.is_gpr(instr_set));
  ASSERT(simd_reg.is_128bit_simd(instr_set));
  return InstructionARM64(Base(0b0011110110, 10), Rn(gpr_addr.hw_id(instr_set)),
                          Rt(simd_reg.hw_id(instr_set)), Imm12(0));
}

InstructionARM64 store128_gpr64_simd128_s32(Register gpr_addr, Register simd_value, s64 offset) {
  ASSERT(gpr_addr.is_gpr(instr_set));
  ASSERT(simd_value.is_128bit_simd(instr_set));
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html
      // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
      InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16),
                       Rn(gpr_addr.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html
  // 128-bit variant, unsigned offset
  // STR <Qt>, [<Xn|SP>{, #<pimm>}]
  // width 10 keeps the literal in its opcode field
  instrs.emplace_back(
      InstructionARM64(Base(0b0011110110, 10), Imm12(0), Rt(simd_value.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 store128_gpr64_simd128_s8(Register gpr_addr, Register simd_value, s64 offset) {
  ASSERT(gpr_addr.is_gpr(instr_set));
  ASSERT(simd_value.is_128bit_simd(instr_set));
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html
      // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
      InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16),
                       Rn(gpr_addr.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html
  // 128-bit variant, unsigned offset
  // STR <Qt>, [<Xn|SP>{, #<pimm>}]
  // width 10 keeps the literal in its opcode field
  instrs.emplace_back(
      InstructionARM64(Base(0b0011110110, 10), Imm12(0), Rt(simd_value.hw_id(instr_set)), Rn(X16)));
  return InstructionARM64(instrs);
}

InstructionARM64 load128_simd128_gpr64(Register simd_dest, Register gpr_addr) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_fpsimd.html
  // - LDR <Qt>, [<Xn|SP>{, #<pimm>}]
  ASSERT(gpr_addr.is_gpr(instr_set));
  ASSERT(simd_dest.is_128bit_simd(instr_set));
  return InstructionARM64(Base(0b0011110111, 10), Rn(gpr_addr.hw_id(instr_set)),
                          Rt(simd_dest.hw_id(instr_set)), Imm12(0));
}

InstructionARM64 load128_simd128_gpr64_s32(Register simd_dest, Register gpr_addr, s64 offset) {
  ASSERT(gpr_addr.is_gpr(instr_set));
  ASSERT(simd_dest.is_128bit_simd(instr_set));
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html
      // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
      InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16),
                       Rn(gpr_addr.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_fpsimd.html
  // - LDR <Qt>, [<Xn|SP>{, #<pimm>}]
  instrs.emplace_back(
      InstructionARM64(Base(0b0011110111, 10), Rn(X16), Rt(simd_dest.hw_id(instr_set)), Imm12(0)));
  return InstructionARM64(instrs);
}

InstructionARM64 load128_simd128_gpr64_s8(Register simd_dest, Register gpr_addr, s64 offset) {
  ASSERT(gpr_addr.is_gpr(instr_set));
  ASSERT(simd_dest.is_128bit_simd(instr_set));
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html
      // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
      InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16),
                       Rn(gpr_addr.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_fpsimd.html
  // - LDR <Qt>, [<Xn|SP>{, #<pimm>}]
  instrs.emplace_back(
      InstructionARM64(Base(0b0011110111, 10), Rn(X16), Rt(simd_dest.hw_id(instr_set)), Imm12(0)));
  return InstructionARM64(instrs);
}

InstructionARM64 load128_simd128_reg_offset(Register simd_dest, Register base, s64 offset) {
  if (offset == 0) {
    return load128_simd128_gpr64(simd_dest, base);
  } else if (offset >= INT8_MIN && offset <= INT8_MAX) {
    return load128_simd128_gpr64_s8(simd_dest, base, offset);
  } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
    return load128_simd128_gpr64_s32(simd_dest, base, offset);
  } else {
    ASSERT(false);
    return {0};
  }
}

InstructionARM64 store128_simd128_reg_offset(Register base, Register simd_val, s64 offset) {
  if (offset == 0) {
    return store128_gpr64_simd128(base, simd_val);
  } else if (offset >= INT8_MIN && offset <= INT8_MAX) {
    return store128_gpr64_simd128_s8(base, simd_val, offset);
  } else if (offset >= INT32_MIN && offset <= INT32_MAX) {
    return store128_gpr64_simd128_s32(base, simd_val, offset);
  } else {
    ASSERT(false);
    return {0};
  }
}

//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
//   PC relative loads and stores
//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

// Implement with LDR but that has a 1MB range limit on ARM (not 2GB like on x86)
// Hopefully this is fine, however it could potentially not be if this is loading static data, which
// may not within 1MB of the current instruction -- that all depends on the linker layout.
//
// But keep it simple at first, add good assertions and we'll see what happens when we
// compile for real.

// TODO ARM64 - the offsets here are always 0 at the time the instruction is made,
// then they are patched later.  That patching also needs an assertion.

// const int ARM64_LDR_MIN = -(1 << 18) * 4;
// const int ARM64_LDR_MAX = ((1 << 18) - 1) * 4;

//! PC-relative static access does not work when EE mappings have different bases.
InstructionARM64 load64_pcRel_s32(Register dest, s64 offset) {
  ASSERT(dest.is_gpr(instr_set));
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  // ASSERT_MSG(offset >= ARM64_LDR_MIN && offset <= ARM64_LDR_MAX,
  //            "PC Relative offset is too large for ARM64, fix it.");
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_lit_gen.html
  // LDR <Xt>, <label>
  return InstructionARM64(Base(0b01011000, 8), Imm19(offset / 4), Rt(dest.hw_id(instr_set)));
}

InstructionARM64 load32s_pcRel_s32(Register dest, s64 offset) {
  ASSERT(dest.is_gpr(instr_set));
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  // ASSERT_MSG(offset >= ARM64_LDR_MIN && offset <= ARM64_LDR_MAX,
  //            "PC Relative offset is too large for ARM64, fix it.");
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldrsw_lit.html
  // LDRSW <Xt>, <label>
  return InstructionARM64(Base(0b10011000, 8), Imm19(offset / 4), Rt(dest.hw_id(instr_set)));
}

InstructionARM64 load32u_pcRel_s32(Register dest, s64 offset) {
  ASSERT(dest.is_gpr(instr_set));
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  // ASSERT_MSG(offset >= ARM64_LDR_MIN && offset <= ARM64_LDR_MAX,
  //            "PC Relative offset is too large for ARM64, fix it.");
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_lit_gen.html
  // LDR <Wt>, <label>
  return InstructionARM64(Base(0b00011000, 8), Imm19(offset / 4), Rt(dest.hw_id(instr_set)));
}

InstructionARM64 load16u_pcRel_s32(Register dest, s64 offset) {
  ASSERT(dest.is_gpr(instr_set));
  // NOTE - the offsets passed into these functions are always `0` and then later patched
  // so im not going to worry about properly encoding the offset
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  // ASSERT_MSG(offset >= ARM64_LDR_MIN && offset <= ARM64_LDR_MAX,
  //            "PC Relative offset is too large for ARM64, fix it.");
  return InstructionARM64(
      {// https://www.scs.stanford.edu/~zyedidia/arm64/adrp.html
       // ADRP <Xd>, <label>
       InstructionARM64(Base(0b100100000000000000000000000, 27), Rd(X16), Immhi(0), Immlo(0)),
       // https://www.scs.stanford.edu/~zyedidia/arm64/ldrh_imm.html
       // LDRH <Wt>, [<Xn|SP>{, #<pimm>}]
       InstructionARM64(Base(0b0111100101, 10), Imm12(offset), Rt(dest.hw_id(instr_set)),
                        Rn(X16))});
}

InstructionARM64 load16s_pcRel_s32(Register dest, s64 offset) {
  ASSERT(dest.is_gpr(instr_set));
  // NOTE - the offsets passed into these functions are always `0` and then later patched
  // so im not going to worry about properly encoding the offset
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  // ASSERT_MSG(offset >= ARM64_LDR_MIN && offset <= ARM64_LDR_MAX,
  //            "PC Relative offset is too large for ARM64, fix it.");
  return InstructionARM64(
      {// https://www.scs.stanford.edu/~zyedidia/arm64/adrp.html
       // ADRP <Xd>, <label>
       InstructionARM64(Base(0b100100000000000000000000000, 27), Rd(X16), Immhi(0), Immlo(0)),
       // https://www.scs.stanford.edu/~zyedidia/arm64/ldrsh_imm.html
       // LDRSH <Xt>, [<Xn|SP>{, #<pimm>}]
       InstructionARM64(Base(0b0111100110, 10), Imm12(offset), Rt(dest.hw_id(instr_set)),
                        Rn(X16))});
}

InstructionARM64 load8u_pcRel_s32(Register dest, s64 offset) {
  ASSERT(dest.is_gpr(instr_set));
  // NOTE - the offsets passed into these functions are always `0` and then later patched
  // so im not going to worry about properly encoding the offset
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  // ASSERT_MSG(offset >= ARM64_LDR_MIN && offset <= ARM64_LDR_MAX,
  //            "PC Relative offset is too large for ARM64, fix it.");
  return InstructionARM64(
      {// https://www.scs.stanford.edu/~zyedidia/arm64/adrp.html
       // ADRP <Xd>, <label>
       InstructionARM64(Base(0b100100000000000000000000000, 27), Rd(X16), Immhi(0), Immlo(0)),
       // https://www.scs.stanford.edu/~zyedidia/arm64/ldrb_imm.html
       // LDRB <Wt>, [<Xn|SP>{, #<pimm>}]
       InstructionARM64(Base(0b0011100101, 10), Imm12(offset), Rt(dest.hw_id(instr_set)),
                        Rn(X16))});
}

InstructionARM64 load8s_pcRel_s32(Register dest, s64 offset) {
  ASSERT(dest.is_gpr(instr_set));
  // NOTE - the offsets passed into these functions are always `0` and then later patched
  // so im not going to worry about properly encoding the offset
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  // ASSERT_MSG(offset >= ARM64_LDR_MIN && offset <= ARM64_LDR_MAX,
  //            "PC Relative offset is too large for ARM64, fix it.");
  return InstructionARM64(
      {// https://www.scs.stanford.edu/~zyedidia/arm64/adrp.html
       // ADRP <Xd>, <label>
       InstructionARM64(Base(0b100100000000000000000000000, 27), Rd(X16), Immhi(0), Immlo(0)),
       // https://www.scs.stanford.edu/~zyedidia/arm64/ldrsb_imm.html
       // LDRSB <Xt>, [<Xn|SP>{, #<pimm>}]
       InstructionARM64(Base(0b0011100110, 10), Imm12(offset), Rt(dest.hw_id(instr_set)),
                        Rn(X16))});
}

InstructionARM64 static_load(Register dest, s64 offset, int size, bool sign_extend) {
  switch (size) {
    case 1:
      if (sign_extend) {
        return load8s_pcRel_s32(dest, offset);
      } else {
        return load8u_pcRel_s32(dest, offset);
      }
      break;
    case 2:
      if (sign_extend) {
        return load16s_pcRel_s32(dest, offset);
      } else {
        return load16u_pcRel_s32(dest, offset);
      }
      break;
    case 4:
      if (sign_extend) {
        return load8s_pcRel_s32(dest, offset);
      } else {
        return load8u_pcRel_s32(dest, offset);
      }
      break;
    case 8:
      return load8s_pcRel_s32(dest, offset);
    default:
      ASSERT(false);
  }
}

// ARM64 PC relative stores materialize an address before storing.

InstructionARM64 store64_pcRel_s32(Register src, s64 offset) {
  ASSERT(src.is_gpr(instr_set));
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  return InstructionARM64(
      {// https://www.scs.stanford.edu/~zyedidia/arm64/adrp.html
       // ADRP <Xd>, <label>
       InstructionARM64(Base(0b100100000000000000000000000, 27), Rd(X16), Immhi(0), Immlo(0)),
       // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_gen.html
       // STR <Xt>, [<Xn|SP>{, #<pimm>}]
       InstructionARM64(Base(0b1111100100, 10), Imm12(offset), Rt(src.hw_id(instr_set)), Rn(X16))});
}

InstructionARM64 store32_pcRel_s32(Register src, s64 offset) {
  ASSERT(src.is_gpr(instr_set));
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  return InstructionARM64(
      {// https://www.scs.stanford.edu/~zyedidia/arm64/adrp.html
       // ADRP <Xd>, <label>
       InstructionARM64(Base(0b100100000000000000000000000, 27), Rd(X16), Immhi(0), Immlo(0)),
       // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_gen.html
       // STR <Wt>, [<Xn|SP>{, #<pimm>}]
       InstructionARM64(Base(0b1011100100, 10), Imm12(offset), Rt(src.hw_id(instr_set)), Rn(X16))});
}

InstructionARM64 store16_pcRel_s32(Register src, s64 offset) {
  ASSERT(src.is_gpr(instr_set));
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  return InstructionARM64(
      {// https://www.scs.stanford.edu/~zyedidia/arm64/adrp.html
       // ADRP <Xd>, <label>
       InstructionARM64(Base(0b100100000000000000000000000, 27), Rd(X16), Immhi(0), Immlo(0)),
       // https://www.scs.stanford.edu/~zyedidia/arm64/strh_imm.html
       // STRH <Wt>, [<Xn|SP>{, #<pimm>}]
       InstructionARM64(Base(0b0111100100, 10), Imm12(offset), Rt(src.hw_id(instr_set)), Rn(X16))});
}

InstructionARM64 store8_pcRel_s32(Register src, s64 offset) {
  ASSERT(src.is_gpr(instr_set));
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  return InstructionARM64(
      {// https://www.scs.stanford.edu/~zyedidia/arm64/adrp.html
       // ADRP <Xd>, <label>
       InstructionARM64(Base(0b100100000000000000000000000, 27), Rd(X16), Immhi(0), Immlo(0)),
       // https://www.scs.stanford.edu/~zyedidia/arm64/strb_imm.html
       // STRH <Wt>, [<Xn|SP>{, #<pimm>}]
       InstructionARM64(Base(0b0011100100, 10), Imm12(offset), Rt(src.hw_id(instr_set)), Rn(X16))});
}

InstructionARM64 static_store(Register value, s64 offset, int size) {
  switch (size) {
    case 1:
      return store8_pcRel_s32(value, offset);
    case 2:
      return store16_pcRel_s32(value, offset);
    case 4:
      return store32_pcRel_s32(value, offset);
    case 8:
      return store64_pcRel_s32(value, offset);
    default:
      ASSERT(false);
  }
}

InstructionARM64 static_addr(Register dest, s64 offset) {
  ASSERT(dest.is_gpr(instr_set));
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  ASSERT(offset >= -(1 << 20));
  ASSERT(offset < (1 << 20));
  u32 immlo = offset & 0x3;
  u32 immhi = (offset >> 2) & 0x7ffff;
  // https://www.scs.stanford.edu/~zyedidia/arm64/adr.html
  // ADR <Xd>, <label>
  return InstructionARM64(Base(0b00010000, 8), Rd(dest.hw_id(instr_set)), Immlo(immlo),
                          Immhi(immhi));
}

InstructionARM64 static_load_f32(Register simd_dest, s64 offset) {
  ASSERT(simd_dest.is_128bit_simd(instr_set));
  ASSERT_MSG(std::abs(offset) < 1000000, "LDR offset is greater than 1MB");
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_lit_fpsimd.html
  // LDR <St>, <label>
  return InstructionARM64(Base(0b00011100, 8), Imm19(offset / 4), Rt(simd_dest.hw_id(instr_set)));
}

InstructionARM64 static_store_f32(Register simd_value, s64 offset) {
  ASSERT(simd_value.is_128bit_simd(instr_set));
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  return InstructionARM64(
      {// https://www.scs.stanford.edu/~zyedidia/arm64/adrp.html
       // ADRP <Xd>, <label>
       InstructionARM64(Base(0b100100000000000000000000000, 27), Rd(X16), Immhi(0), Immlo(0)),
       // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html
       // STR <St>, [<Xn|SP>{, #<pimm>}]
       InstructionARM64(Base(0b1011110100, 10), Imm12(offset), Rt(simd_value.hw_id(instr_set)),
                        Rn(X16))});
}

//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
//   FUNCTION STUFF
//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

InstructionARM64 ret() {
  // https://www.scs.stanford.edu/~zyedidia/arm64/ret.html
  // - defaults to using X30 if Rn is absent
  return InstructionARM64(Base(0b1101011001011111000000, 22), Rn(30));
}

InstructionARM64 push_gpr64(Register reg) {
  // ARM64 stack grows down, so we subtract 16 from SP and store the register
  // Equivalent assembly: STR reg, [SP, #-16]!
  // - https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_gen.html
  // We use 16 because in ARM, the stack must be 16-byte aligned.
  // This does mean we are inefficiently using the stack, there are a few better options:
  // - Push in pairs, two registers at a time
  // - Preallocate stack-space
  // But we can't do either of these at this level, this is an optimization that has to come from
  // higher in the stack.  Here we are concerned with just satisfying the need to push a GPR
  ASSERT(reg.is_gpr(instr_set));
  return InstructionARM64(Base(0b1111100000000000000011, 22), Imm9s(-16), Rn(ARM64_REG::SP),
                          Rt(reg.hw_id(instr_set)));
}

//! Push two GPRs in one 16-byte stack slot with STP.
InstructionARM64 push_pair_gpr64(Register a, Register b) {
  ASSERT(a.is_gpr(instr_set));
  ASSERT(b.is_gpr(instr_set));
  return InstructionARM64(Base(0b10101001101111110000001111100000, 32), Rt(a.hw_id(instr_set)),
                          Rt2(b.hw_id(instr_set)));
}

//! Pop two GPRs from one 16-byte stack slot with LDP.
InstructionARM64 pop_pair_gpr64(Register a, Register b) {
  ASSERT(a.is_gpr(instr_set));
  ASSERT(b.is_gpr(instr_set));
  return InstructionARM64(Base(0b10101000110000010000001111100000, 32), Rt(a.hw_id(instr_set)),
                          Rt2(b.hw_id(instr_set)));
}

InstructionARM64 pop_gpr64(Register reg) {
  // ldr reg, [sp], #16
  // - https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_gen.html
  ASSERT(reg.is_gpr(instr_set));
  return InstructionARM64(Base(0b1111100001000000000001, 22), Imm9s(16), Rn(ARM64_REG::SP),
                          Rt(reg.hw_id(instr_set)));
}

InstructionARM64 call_r64(Register reg) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/blr.html
  // BLR <Xn>
  ASSERT(reg.is_gpr(instr_set));
  return InstructionARM64(Base(0b1101011000111111000000, 22), Rn(reg.hw_id(instr_set)));
}

InstructionARM64 jmp_r64(Register reg) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/br.html
  // BR <Xn>
  ASSERT(reg.is_gpr(instr_set));
  return InstructionARM64(Base(0b1101011000011111000000, 22), Rn(reg.hw_id(instr_set)));
}

//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
//   INTEGER MATH
//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

InstructionARM64 sub_gpr64_imm8s(Register reg, int64_t imm) {
  return sub_gpr64_imm(reg, imm);
}

InstructionARM64 add_gpr64_imm8s(Register reg, int64_t imm) {
  return add_gpr64_imm(reg, imm);
}

InstructionARM64 sub_gpr64_imm32s(Register reg, int64_t imm) {
  return sub_gpr64_imm(reg, imm);
}

InstructionARM64 add_gpr64_imm32s(Register reg, int64_t imm) {
  return add_gpr64_imm(reg, imm);
}

InstructionARM64 add_gpr64_imm(Register reg, int64_t imm) {
  ASSERT(reg.is_gpr(instr_set));
  if (imm < 0) {
    return sub_gpr64_imm(reg, std::abs(imm));
  }
  // Check to see if we can represent this addition in a single instruction
  // if not, then we need to emit multiple partial instructions
  const auto [is_single_instr, imm12, needs_shift] = can_encode_single_imm12(imm);
  if (is_single_instr) {
    // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html
    // ADD <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
    return InstructionARM64(Base(0b100100010, 9), Sh(needs_shift ? 1 : 0), Imm12(imm12),
                            Rd(reg.hw_id(instr_set)), Rn(reg.hw_id(instr_set)));
  } else {
    return InstructionARM64(mov_gpr64_u64(X16, imm), add_gpr64_gpr64(reg, X16));
  }
}

InstructionARM64 sub_gpr64_imm(Register reg, int64_t imm) {
  ASSERT(reg.is_gpr(instr_set));
  if (imm < 0) {
    return add_gpr64_imm(reg, std::abs(imm));
  }
  // Check to see if we can represent this subtraction in a single instruction
  // if not, then we need to emit multiple partial instructions
  const auto [is_single_instr, imm12, needs_shift] = can_encode_single_imm12(imm);
  if (is_single_instr) {
    // https://www.scs.stanford.edu/~zyedidia/arm64/sub_addsub_imm.html
    // SUB <Xd|SP>, <Xn|SP>, #<imm>{, <shift>}
    return InstructionARM64(Base(0b110100010, 9), Sh(needs_shift ? 1 : 0), Imm12(imm12),
                            Rd(reg.hw_id(instr_set)), Rn(reg.hw_id(instr_set)));
  } else {
    return InstructionARM64(mov_gpr64_u64(X16, imm), sub_gpr64_gpr64(reg, X16));
  }
}

InstructionARM64 add_gpr64_gpr64(Register dst, Register src) {
  if (dst == SP) {
    // shifted ADD treats register 31 as XZR, so use the extended form
    // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html
    ASSERT_MSG(src != SP, "ARM64 cannot use SP as the source of a register add");
    return InstructionARM64(Base(0b10001011001000000110000000000000, 32), Rm(src.hw_id(instr_set)),
                            Rn(dst.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
  // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
  return InstructionARM64(Base(0b10001011000, 11), Rd(dst.hw_id(instr_set)), Imm6(0),
                          Rn(dst.hw_id(instr_set)), Rm(src.hw_id(instr_set)));
}

InstructionARM64 sub_gpr64_gpr64(Register dst, Register src) {
  if (dst == SP) {
    // shifted SUB treats register 31 as XZR, so use the extended form
    // https://www.scs.stanford.edu/~zyedidia/arm64/sub_addsub_ext.html
    ASSERT_MSG(src != SP, "ARM64 cannot use SP as the source of a register subtract");
    return InstructionARM64(Base(0b11001011001000000110000000000000, 32), Rm(src.hw_id(instr_set)),
                            Rn(dst.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/sub_addsub_shift.html
  // SUB <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
  return InstructionARM64(Base(0b11001011000, 11), Rd(dst.hw_id(instr_set)), Imm6(0),
                          Rn(dst.hw_id(instr_set)), Rm(src.hw_id(instr_set)));
}

InstructionARM64 imul_gpr32_gpr32(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/mul_madd.html
  // MUL <Wd>, <Wn>, <Wm>
  return InstructionARM64(Base(0b0001101100000000011111, 22), Rd(dst.hw_id(instr_set)),
                          Rn(dst.hw_id(instr_set)), Rm(src.hw_id(instr_set)));
}

InstructionARM64 imul_gpr64_gpr64(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/mul_madd.html
  // MUL <Xd>, <Xn>, <Xm>
  return InstructionARM64(Base(0b1001101100000000011111, 22), Rd(dst.hw_id(instr_set)),
                          Rn(dst.hw_id(instr_set)), Rm(src.hw_id(instr_set)));
}

// ARM64 division uses SDIV, UDIV and MSUB.
InstructionARM64 idiv_gpr32(Register) {
  ASSERT_MSG(false, "ARM64 division uses sdiv_gpr32");
  return InstructionARM64(0b0);
}

InstructionARM64 unsigned_div_gpr32(Register) {
  ASSERT_MSG(false, "ARM64 division uses udiv_gpr32");
  return InstructionARM64(0b0);
}

InstructionARM64 cdq() {
  ASSERT_MSG(false, "ARM64 SDIV does not require CDQ sign extension");
  return InstructionARM64(0b0);
}

InstructionARM64 sdiv_gpr32(Register dst, Register n, Register m) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(n.is_gpr(instr_set));
  ASSERT(m.is_gpr(instr_set));
  // https://www.scs.stanford.edu/~zyedidia/arm64/sdiv.html
  // SDIV <Wd>, <Wn>, <Wm>
  return InstructionARM64(Base(0b00011010110000000000110000000000, 32), Rm(m.hw_id(instr_set)),
                          Rn(n.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 udiv_gpr32(Register dst, Register n, Register m) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(n.is_gpr(instr_set));
  ASSERT(m.is_gpr(instr_set));
  // https://www.scs.stanford.edu/~zyedidia/arm64/udiv.html
  // UDIV <Wd>, <Wn>, <Wm>
  return InstructionARM64(Base(0b00011010110000000000100000000000, 32), Rm(m.hw_id(instr_set)),
                          Rn(n.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 msub_gpr32(Register dst, Register n, Register m, Register a) {
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(n.is_gpr(instr_set));
  ASSERT(m.is_gpr(instr_set));
  ASSERT(a.is_gpr(instr_set));
  // https://www.scs.stanford.edu/~zyedidia/arm64/msub.html
  // MSUB <Wd>, <Wn>, <Wm>, <Wa>
  return InstructionARM64(Base(0b00011011000000001000000000000000, 32), Rm(m.hw_id(instr_set)),
                          Ra(a.hw_id(instr_set)), Rn(n.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 movsx_r64_r32(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/sxtw_sbfm.html
  // SXTW <Xd>, <Wn>
  return InstructionARM64(Base(0b1001001101000000011111, 22), Rd(dst.hw_id(instr_set)),
                          Rn(src.hw_id(instr_set)));
}

InstructionARM64 cmp_gpr64_gpr64(Register a, Register b) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/cmp_subs_addsub_shift.html
  // CMP <Xn>, <Xm>
  return InstructionARM64(Base(0b11101011000000000000000000011111, 32), Rn(a.hw_id(instr_set)),
                          Rm(b.hw_id(instr_set)));
}

//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
//   BIT STUFF
//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

InstructionARM64 or_gpr64_gpr64(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/orr_log_shift.html
  // ORR <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(src.is_gpr(instr_set));
  return InstructionARM64(Base(0b10101010000, 11), Rd(dst.hw_id(instr_set)),
                          Rn(dst.hw_id(instr_set)), Rm(src.hw_id(instr_set)));
}

InstructionARM64 and_gpr64_gpr64(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/and_log_shift.html
  // AND <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(src.is_gpr(instr_set));
  return InstructionARM64(Base(0b10001010000, 11), Rd(dst.hw_id(instr_set)),
                          Rn(dst.hw_id(instr_set)), Rm(src.hw_id(instr_set)));
}

InstructionARM64 xor_gpr64_gpr64(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/eor_log_shift.html
  // EOR <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
  ASSERT(dst.is_gpr(instr_set));
  ASSERT(src.is_gpr(instr_set));
  return InstructionARM64(Base(0b11001010000, 11), Rd(dst.hw_id(instr_set)),
                          Rn(dst.hw_id(instr_set)), Rm(src.hw_id(instr_set)));
}

InstructionARM64 not_gpr64(Register reg) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/mvn_orn_log_shift.html
  // MVN <Xd>, <Xm>{, <shift> #<amount>}
  // ==
  // ORN <Xd>, XZR, <Xm>{, <shift> #<amount>}
  ASSERT(reg.is_gpr(instr_set));
  return InstructionARM64(Base(0b101010100010000000000011111, 27), Rd(reg.hw_id(instr_set)),
                          Rm(reg.hw_id(instr_set)));
}

//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
//   SHIFTS
//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

InstructionARM64 shl_gpr64_reg(Register reg, Register shift_reg) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/lsl_lslv.html
  // LSL <Xd>, <Xn>, <Xm>
  ASSERT(reg.is_gpr(instr_set));
  ASSERT(shift_reg.is_gpr(instr_set));
  return InstructionARM64(Base(0b1001101011000000001000, 22), Rd(reg.hw_id(instr_set)),
                          Rn(reg.hw_id(instr_set)), Rm(shift_reg.hw_id(instr_set)));
}

InstructionARM64 shr_gpr64_reg(Register reg, Register shift_reg) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/lsr_lsrv.html
  // LSR <Xd>, <Xn>, <Xm>
  ASSERT(reg.is_gpr(instr_set));
  ASSERT(shift_reg.is_gpr(instr_set));
  return InstructionARM64(Base(0b1001101011000000001001, 22), Rd(reg.hw_id(instr_set)),
                          Rn(reg.hw_id(instr_set)), Rm(shift_reg.hw_id(instr_set)));
}

InstructionARM64 sar_gpr64_reg(Register reg, Register shift_reg) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/asr_asrv.html
  // ASR <Xd>, <Xn>, <Xm>
  ASSERT(reg.is_gpr(instr_set));
  ASSERT(shift_reg.is_gpr(instr_set));
  return InstructionARM64(Base(0b1001101011000000001010, 22), Rd(reg.hw_id(instr_set)),
                          Rn(reg.hw_id(instr_set)), Rm(shift_reg.hw_id(instr_set)));
}

InstructionARM64 shl_gpr64_u8(Register reg, uint8_t sa) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/lsl_ubfm.html
  // LSL <Xd>, <Xn>, #<shift>
  sa &= 63;
  ASSERT(reg.is_gpr(instr_set));
  return InstructionARM64(Base(0b1101001101, 10), Rd(reg.hw_id(instr_set)),
                          Rn(reg.hw_id(instr_set)), Immr((64 - sa) & 63), Imms(63 - sa));
}

InstructionARM64 shr_gpr64_u8(Register reg, uint8_t sa) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/lsr_ubfm.html
  // LSR <Xd>, <Xn>, #<shift>
  // sf	1	0	1	0	0	1	1	0	N
  sa &= 63;
  ASSERT(reg.is_gpr(instr_set));
  return InstructionARM64(Base(0b1101001101000000111111, 22), Rd(reg.hw_id(instr_set)),
                          Rn(reg.hw_id(instr_set)), Immr(sa));
}

InstructionARM64 sar_gpr64_u8(Register reg, uint8_t sa) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/asr_sbfm.html
  // ASR <Xd>, <Xn>, #<shift>
  sa &= 63;
  ASSERT(reg.is_gpr(instr_set));
  return InstructionARM64(Base(0b1001001101000000111111, 22), Rd(reg.hw_id(instr_set)),
                          Rn(reg.hw_id(instr_set)), Immr(sa));
}

//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
//   CONTROL FLOW
//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
//
// All of these instructions jump to a target that is zero
// and then its up to the IR to patch the actual target
//
// Critically, on arm these relative targets must be within
// 128MB, which is much less than x86 (~2GB)
//
// However, these functions are only really used for jumps within
// a given function...of which the largest we've seen isn't even in the MB
// of sizes
//
// So for now, keep it simple and don't implement something more
// complicated like veneers, this should be fine.

InstructionARM64 jmp_imm() {
  // https://www.scs.stanford.edu/~zyedidia/arm64/b_uncond.html
  // B <label>
  return InstructionARM64(Base(0b000101, 6), Imm26(0))
      .with_reloc(emitter::ARM64::RelocKind::BRANCH26);
}

// Now these instructions in ARM are even more limiting, conditional
// branches must be within 1MB relative to the instruction
//
// However once again, that is still WAY below our biggest functions of a few kb
//
// But still...be aware!  There should be some assertions in place in the patching so that
// said issues don't just fly under the radar
//
// Also, these instructions may have to be patched slightly differently since
// ARM uses a single branch instruction for all
//
// It's worth noting that these x86 instructions also have limitations, they cannot
// jump to far labels (labels in other code segments).  But that's more difficult to
// give a numeric value to like with ARM.

InstructionARM64 je_imm() {
  // https://www.scs.stanford.edu/~zyedidia/arm64/b_cond.html
  // B.<cond> <label>
  // 0000 	EQ
  return InstructionARM64(Base(0b01010100, 8), Imm19(0), Cond(0b0000))
      .with_reloc(emitter::ARM64::RelocKind::BRANCH19);
}

InstructionARM64 jne_imm() {
  // https://www.scs.stanford.edu/~zyedidia/arm64/b_cond.html
  // B.<cond> <label>
  // 0001 	NE
  return InstructionARM64(Base(0b01010100, 8), Imm19(0), Cond(0b0001))
      .with_reloc(emitter::ARM64::RelocKind::BRANCH19);
}

InstructionARM64 jle_imm() {
  // https://www.scs.stanford.edu/~zyedidia/arm64/b_cond.html
  // B.<cond> <label>
  // 1101 	LE
  return InstructionARM64(Base(0b01010100, 8), Imm19(0), Cond(0b1101))
      .with_reloc(emitter::ARM64::RelocKind::BRANCH19);
}

InstructionARM64 jge_imm() {
  // https://www.scs.stanford.edu/~zyedidia/arm64/b_cond.html
  // B.<cond> <label>
  // 1010 	GE
  return InstructionARM64(Base(0b01010100, 8), Imm19(0), Cond(0b1010))
      .with_reloc(emitter::ARM64::RelocKind::BRANCH19);
}

InstructionARM64 jl_imm() {
  // https://www.scs.stanford.edu/~zyedidia/arm64/b_cond.html
  // B.<cond> <label>
  // 1011 	LT
  return InstructionARM64(Base(0b01010100, 8), Imm19(0), Cond(0b1011))
      .with_reloc(emitter::ARM64::RelocKind::BRANCH19);
}

InstructionARM64 jg_imm() {
  // https://www.scs.stanford.edu/~zyedidia/arm64/b_cond.html
  // B.<cond> <label>
  // 1100 	GT
  return InstructionARM64(Base(0b01010100, 8), Imm19(0), Cond(0b1100))
      .with_reloc(emitter::ARM64::RelocKind::BRANCH19);
}

InstructionARM64 jbe_imm() {
  // https://www.scs.stanford.edu/~zyedidia/arm64/b_cond.html
  // B.<cond> <label>
  // 1001 	LS
  return InstructionARM64(Base(0b01010100, 8), Imm19(0), Cond(0b1001))
      .with_reloc(emitter::ARM64::RelocKind::BRANCH19);
}

InstructionARM64 jae_imm() {
  // https://www.scs.stanford.edu/~zyedidia/arm64/b_cond.html
  // B.<cond> <label>
  // 0010 	CS
  return InstructionARM64(Base(0b01010100, 8), Imm19(0), Cond(0b0010))
      .with_reloc(emitter::ARM64::RelocKind::BRANCH19);
}

InstructionARM64 jb_imm() {
  // https://www.scs.stanford.edu/~zyedidia/arm64/b_cond.html
  // B.<cond> <label>
  // 0011 	CC
  return InstructionARM64(Base(0b01010100, 8), Imm19(0), Cond(0b0011))
      .with_reloc(emitter::ARM64::RelocKind::BRANCH19);
}

InstructionARM64 ja_imm() {
  // https://www.scs.stanford.edu/~zyedidia/arm64/b_cond.html
  // B.<cond> <label>
  // 1000 	HI
  return InstructionARM64(Base(0b01010100, 8), Imm19(0), Cond(0b1000))
      .with_reloc(emitter::ARM64::RelocKind::BRANCH19);
}

//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
//   FLOAT MATH
//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

InstructionARM64 cmp_f32_f32(Register a, Register b) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fcmp_float.html
  // Single-precision (ftype == 00 && opc == 00)
  // FCMP <Sn>, <Sm>
  return InstructionARM64(Base(0b00011110001000000010000000000000, 32), Rn(a.hw_id(instr_set)),
                          Rm(b.hw_id(instr_set)));
}

InstructionARM64 sqrt_f32(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fsqrt_float.html
  // Single-precision (ftype == 00)
  // FSQRT <Sd>, <Sn>
  // Rd holds the square root destination
  return InstructionARM64(Base(0b0001111000100001110000, 22), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)));
}

InstructionARM64 mul_f32_f32(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fmul_float.html
  // Single-precision (ftype == 00)
  // FMUL <Sd>, <Sn>, <Sm>
  return InstructionARM64(Base(0b0001111000100000000010, 22), Rd(dst.hw_id(instr_set)),
                          Rn(dst.hw_id(instr_set)), Rm(src.hw_id(instr_set)));
}

InstructionARM64 div_f32_f32(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fdiv_float.html
  // Single-precision (ftype == 00)
  // FDIV <Sd>, <Sn>, <Sm>
  return InstructionARM64(Base(0b0001111000100000000110, 22), Rd(dst.hw_id(instr_set)),
                          Rn(dst.hw_id(instr_set)), Rm(src.hw_id(instr_set)));
}

InstructionARM64 sub_f32_f32(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fsub_float.html
  // Single-precision (ftype == 00)
  // FSUB <Sd>, <Sn>, <Sm>
  return InstructionARM64(Base(0b0001111000100000001110, 22), Rd(dst.hw_id(instr_set)),
                          Rn(dst.hw_id(instr_set)), Rm(src.hw_id(instr_set)));
}

InstructionARM64 add_f32_f32(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fadd_float.html
  // Single-precision (ftype == 00)
  // FADD <Sd>, <Sn>, <Sm>
  return InstructionARM64(Base(0b0001111000100000001010, 22), Rd(dst.hw_id(instr_set)),
                          Rn(dst.hw_id(instr_set)), Rm(src.hw_id(instr_set)));
}

InstructionARM64 min_f32_f32(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fmin_float.html
  // Single-precision (ftype == 00)
  // FMIN <Sd>, <Sn>, <Sm>
  return InstructionARM64(Base(0b0001111000100000010110, 22), Rd(dst.hw_id(instr_set)),
                          Rn(dst.hw_id(instr_set)), Rm(src.hw_id(instr_set)));
}

InstructionARM64 max_f32_f32(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fmax_float.html
  // Single-precision (ftype == 00)
  // FMAX <Sd>, <Sn>, <Sm>
  return InstructionARM64(Base(0b0001111000100000010010, 22), Rd(dst.hw_id(instr_set)),
                          Rn(dst.hw_id(instr_set)), Rm(src.hw_id(instr_set)));
}

InstructionARM64 int32_to_f32(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/scvtf_float_int.html
  // 32-bit to single-precision (sf == 0 && ftype == 00)
  // SCVTF <Sd>, <Wn>
  return InstructionARM64(Base(0b0001111000100010000000, 22), Rd(dst.hw_id(instr_set)),
                          Rn(src.hw_id(instr_set)));
}

//! Convert a scalar float to int. FCVTZS matches the PS2 finite overflow clamps.
InstructionARM64 f32_to_int32(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fcvtzs_float_int.html
  // 32-bit to single-precision (sf == 0 && ftype == 00)
  // FCVTZS <Wd>, <Sn>
  return InstructionARM64(Base(0b0001111000111000000000, 22), Rd(dst.hw_id(instr_set)),
                          Rn(src.hw_id(instr_set)));
}

InstructionARM64 nop() {
  // https://www.scs.stanford.edu/~zyedidia/arm64/nop.html
  return InstructionARM64(Base(0b11010101000000110010000000011111, 32));
}

//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
//   UTILITIES
//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

//! Emit BRK #1 for GOAL (break) because ARM64 division by zero does not trap.
InstructionARM64 trap() {
  // https://www.scs.stanford.edu/~zyedidia/arm64/brk.html
  // BRK #1
  return InstructionARM64(Base(0b11010100001, 11), Imm16(1));
}

InstructionARM64 null() {
  // dummy empty byte
  return InstructionARM64(0b0);
}

/////////////////////////////
// AVX (VF - Vector Float) //
/////////////////////////////

InstructionARM64 nop_vf() {
  // Not sure if this one was even needed for x86, but it does not really exist on ARM64
  // just use a normal nop
  return nop();
}

InstructionARM64 wait_vf() {
  // Another instruction that doesnt really map to arm64 because there is no annoying
  // x87 FPU behaviour
  return nop();
}

InstructionARM64 mov_vf_vf(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/mov_orr_advsimd_reg.html
  // MOV <Vd>.<T>, <Vn>.<T>
  // Q 	<T>
  // 0 	8B
  // 1 	16B
  ASSERT(dst.is_128bit_simd(instr_set));
  ASSERT(src.is_128bit_simd(instr_set));
  // orr's Rm has to be the source too, or this comes out as `orr vD.16b, vN.16b, v0.16b`,
  // which only moves when v0 is zero. checked against clang: mov.16b v5, v3 is 4ea31c65.
  return InstructionARM64(Base(0b0100111010100000000111, 22), Rm(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)), Rn(src.hw_id(instr_set)));
}

InstructionARM64 loadvf_gpr64_plus_gpr64(Register dst, Register addr1, Register addr2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_reg_fpsimd.html
  // 128-bit variant
  // LDR <Qt>, [<Xn|SP>, (<Wm>|<Xm>){, <extend> {<amount>}}]
  ASSERT(dst.is_128bit_simd(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  return InstructionARM64(Base(0b0011110011100000111010, 22), Rt(dst.hw_id(instr_set)),
                          Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set)));
}

InstructionARM64 loadvf_gpr64_plus_gpr64_plus_s8(Register dst,
                                                 Register addr1,
                                                 Register addr2,
                                                 s64 offset) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_fpsimd.html
  // 128-bit variant
  // LDR <Qt>, [<Xn|SP>], #<simm>
  ASSERT(dst.is_128bit_simd(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // base + index in x16, then ldur. imm9 covers the whole s8 range, so the offset folds in
  // and this needs no second add.
  // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
  // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
  return InstructionARM64({InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0),
                                            Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))),
                           // https://www.scs.stanford.edu/~zyedidia/arm64/ldur_fpsimd.html
                           // LDUR <Qt>, [<Xn|SP>{, #<simm>}]
                           // checked against clang, ldur q7, [x19, #124] is 3cc7c267
                           InstructionARM64(Base(0b0011110011000000000000, 22),
                                            Rt(dst.hw_id(instr_set)), Rn(X16), Imm9s(offset))});
}

InstructionARM64 loadvf_gpr64_plus_gpr64_plus_s32(Register dst,
                                                  Register addr1,
                                                  Register addr2,
                                                  s64 offset) {
  ASSERT(dst.is_128bit_simd(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/ldur_fpsimd.html
  // 128-bit variant
  // LDUR <Qt>, [<Xn|SP>{, #<simm>}]
  // the whole address is in x16 already, so the offset is 0. the post indexed LDR is a
  // different instruction that also writes x16 back.
  instrs.emplace_back(InstructionARM64(Base(0b0011110011000000000000, 22), Rt(dst.hw_id(instr_set)),
                                       Rn(X16), Imm9s(0)));
  return InstructionARM64(instrs);
}

InstructionARM64 storevf_gpr64_plus_gpr64(Register value, Register addr1, Register addr2) {
  ASSERT(value.is_128bit_simd(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_reg_fpsimd.html
  // STR <Qt>, [<Xn|SP>, (<Wm>|<Xm>){, <extend> {<amount>}}]
  return InstructionARM64(Base(0b0011110010100000111010, 22), Rt(value.hw_id(instr_set)),
                          Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set)));
}

InstructionARM64 storevf_gpr64_plus_gpr64_plus_s8(Register value,
                                                  Register addr1,
                                                  Register addr2,
                                                  s64 offset) {
  ASSERT(value.is_128bit_simd(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT8_MIN && offset <= INT8_MAX);
  // base + index in x16, then stur. imm9 covers the whole s8 range, so the offset folds in.
  // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
  // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
  return InstructionARM64({InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0),
                                            Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))),
                           // https://www.scs.stanford.edu/~zyedidia/arm64/stur_fpsimd.html
                           // STUR <Qt>, [<Xn|SP>{, #<simm>}]
                           // checked against clang, stur q7, [x16, #124] is 3c87c207
                           InstructionARM64(Base(0b0011110010000000000000, 22),
                                            Rt(value.hw_id(instr_set)), Rn(X16), Imm9s(offset))});
}

InstructionARM64 storevf_gpr64_plus_gpr64_plus_s32(Register value,
                                                   Register addr1,
                                                   Register addr2,
                                                   s64 offset) {
  ASSERT(value.is_128bit_simd(instr_set));
  ASSERT(addr1.is_gpr(instr_set));
  ASSERT(addr2.is_gpr(instr_set));
  ASSERT(addr1 != addr2);
  ASSERT(addr1 != SP);
  ASSERT(addr2 != SP);
  ASSERT(offset >= INT32_MIN && offset <= INT32_MAX);
  // first establish the base+index+offset value in x16
  std::vector<InstructionARM64> instrs = {
      // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html
      // ADD <Xd>, <Xn>, <Xm>{, <shift> #<amount>}
      InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)),
                       Rm(addr2.hw_id(instr_set))),
  };
  if (offset < 0) {
    // we'll subtract instead
    offset = std::abs(offset);
    const auto sub_instrs = construct_multiple_imm12_subs(offset, X16);
    instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end());
  } else {
    const auto add_instrs = construct_multiple_imm12_adds(offset, X16);
    instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end());
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html
  // STR <Qt>, [<Xn|SP>], #<simm>
  instrs.emplace_back(InstructionARM64(Base(0b0011110010000000000000, 22),
                                       Rt(value.hw_id(instr_set)), Rn(X16), Imm9s(0)));
  return InstructionARM64(instrs);
}

InstructionARM64 loadvf_rip_plus_s32(Register dest, s64 offset) {
  ASSERT(dest.is_128bit_simd(instr_set));
  ASSERT_MSG(offset != 0,
             "PC Relative offset isn't 0 at encoding time, actually encode it properly!");
  return InstructionARM64(
      {// https://www.scs.stanford.edu/~zyedidia/arm64/adrp.html
       // ADRP <Xd>, <label>
       InstructionARM64(Base(0b100100000000000000000000000, 27), Rd(X16), Immhi(0), Immlo(0)),
       // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_fpsimd.html
       // LDR <Qt>, [<Xn|SP>{, #<pimm>}]
       InstructionARM64(Base(0b0011110111, 10), Imm12(offset), Rt(dest.hw_id(instr_set)),
                        Rn(X16))});
}

/*!
 * MOV <Vd>.S[dst_lane], <Vn>.S[src_lane]. copies one 32 bit lane and leaves the rest of Vd alone.
 */
InstructionARM64 ins_vf_lane(Register dst, u8 dst_lane, Register src, u8 src_lane) {
  ASSERT(dst.is_128bit_simd(instr_set));
  ASSERT(src.is_128bit_simd(instr_set));
  ASSERT(dst_lane < 4 && src_lane < 4);
  // https://www.scs.stanford.edu/~zyedidia/arm64/mov_ins_advsimd_elt.html
  // for 32 bit lanes imm5 is (index << 3) | 0b100 and imm4 is index << 2
  return InstructionARM64(Base(0b01101110000000000000010000000000, 32),
                          Imm5(u32(dst_lane << 3) | 0b100), Imm4(u32(src_lane << 2)),
                          Rn(src.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 dup_vf_lane(Register dst, Register src, u8 lane) {
  ASSERT(dst.is_128bit_simd(instr_set));
  ASSERT(src.is_128bit_simd(instr_set));
  ASSERT(lane < 4);
  // https://www.scs.stanford.edu/~zyedidia/arm64/dup_advsimd_elt.html
  // DUP <Vd>.4S, <Vn>.S[lane]
  // imm5 is the same shape as ins_vf_lane's
  return InstructionARM64(Base(0b01001110000000000000010000000000, 32),
                          Imm5(u32(lane << 3) | 0b100), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)));
}

InstructionARM64 blend_vf(Register dst, Register src1, Register src2, u8 mask) {
  ASSERT(!(mask & 0b11110000));
  ASSERT(dst.is_128bit_simd(instr_set));
  ASSERT(src1.is_128bit_simd(instr_set));
  ASSERT(src2.is_128bit_simd(instr_set));
  // x86 does this in one blendps. arm64 has no blend with an immediate lane mask, so take
  // src1 and overwrite the lanes the mask selects. built in v16 first, so dst is allowed to
  // be either source.
  std::vector<InstructionARM64> instrs = {mov_vf_vf(V16, src1)};
  for (u8 lane = 0; lane < 4; lane++) {
    if (mask & (1 << lane)) {
      instrs.push_back(ins_vf_lane(V16, lane, src2, lane));
    }
  }
  instrs.push_back(mov_vf_vf(dst, V16));
  return InstructionARM64(instrs);
}

InstructionARM64 swizzle_vf(Register dst, Register src, u8 controlBytes) {
  ASSERT(dst.is_128bit_simd(instr_set));
  ASSERT(src.is_128bit_simd(instr_set));
  // x86 uses shufps with both sources the same, so lane i of the result is lane
  // (controlBytes >> 2i) & 3 of the source. same lane at a time approach as blend_vf, via v16
  // so dst may alias src. all four lanes get written, so v16's previous contents don't matter.
  std::vector<InstructionARM64> instrs;
  for (u8 lane = 0; lane < 4; lane++) {
    instrs.push_back(ins_vf_lane(V16, lane, src, (controlBytes >> (lane * 2)) & 0b11));
  }
  instrs.push_back(mov_vf_vf(dst, V16));
  return InstructionARM64(instrs);
}

InstructionARM64 shuffle_vf(Register dst, Register src, u8 dx, u8 dy, u8 dz, u8 dw) {
  ASSERT(dst.is_128bit_simd(instr_set));
  ASSERT(src.is_128bit_simd(instr_set));
  ASSERT(dx < 4);
  ASSERT(dy < 4);
  ASSERT(dz < 4);
  ASSERT(dw < 4);
  // pack the four lane picks into one control byte and hand it to swizzle_vf. nothing in the
  // compiler calls this on either backend, only the emitter tests.
  u8 imm = dx + (dy << 2) + (dz << 4) + (dw << 6);
  return swizzle_vf(dst, src, imm);
}

InstructionARM64 splat_vf(Register dst, Register src, Register::VF_ELEMENT element) {
  // broadcasting one lane to all four is what DUP does, in one instruction. going through
  // swizzle_vf would take five, an INS per lane and a move.
  switch (element) {
    case Register::VF_ELEMENT::X:
      return dup_vf_lane(dst, src, 0);
    case Register::VF_ELEMENT::Y:
      return dup_vf_lane(dst, src, 1);
    case Register::VF_ELEMENT::Z:
      return dup_vf_lane(dst, src, 2);
    case Register::VF_ELEMENT::W:
      return dup_vf_lane(dst, src, 3);
    default:
      ASSERT(false);
      return {0};
  }
}

InstructionARM64 xor_vf(Register dst, Register src1, Register src2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/eor_advsimd.html
  // EOR <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0110111000100000000111, 22), Rn(src1.hw_id(instr_set)),
                          Rm(src2.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 sub_vf(Register dst, Register src1, Register src2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fsub_advsimd.html
  // FSUB <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  // 4 single precision floats
  return InstructionARM64(Base(0b0100111010100000110101, 22), Rn(src1.hw_id(instr_set)),
                          Rm(src2.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 add_vf(Register dst, Register src1, Register src2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fadd_advsimd.html
  // FADD <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  // 4 single precision floats
  return InstructionARM64(Base(0b0100111000100000110101, 22), Rn(src1.hw_id(instr_set)),
                          Rm(src2.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 mul_vf(Register dst, Register src1, Register src2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fmul_advsimd_vec.html
  // FMUL <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  // 4 single precision floats
  return InstructionARM64(Base(0b0110111000100000110111, 22), Rn(src1.hw_id(instr_set)),
                          Rm(src2.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 max_vf(Register dst, Register src1, Register src2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fmax_advsimd.html
  // FMAX <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  // 4 single precision floats
  return InstructionARM64(Base(0b0100111000100000111101, 22), Rn(src1.hw_id(instr_set)),
                          Rm(src2.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 min_vf(Register dst, Register src1, Register src2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fmin_advsimd.html
  // FMIN <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  // 4 single precision floats
  return InstructionARM64(Base(0b0100111010100000111101, 22), Rn(src1.hw_id(instr_set)),
                          Rm(src2.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 div_vf(Register dst, Register src1, Register src2) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fdiv_advsimd.html
  // FDIV <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  // 4 single precision floats
  return InstructionARM64(Base(0b0110111000100000111111, 22), Rn(src1.hw_id(instr_set)),
                          Rm(src2.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 sqrt_vf(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fsqrt_advsimd.html
  // FSQRT <Vd>.<T>, <Vn>.<T>
  // 4 single precision floats
  return InstructionARM64(Base(0b0110111010100001111110, 22), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)));
}

InstructionARM64 itof_vf(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/scvtf_advsimd_int.html
  // SCVTF <Vd>.<T>, <Vn>.<T>
  // s32 int -> 4 single precision floats
  return InstructionARM64(Base(0b0100111000100001110110, 22), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)));
}

InstructionARM64 ftoi_vf(Register dst, Register src) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/fcvtzs_advsimd_int.html
  // FCVTZS <Vd>.<T>, <Vn>.<T>
  // 4 single precision floats -> s32 ints
  // FCVTZS rounds toward zero.
  return InstructionARM64(Base(0b0100111010100001101110, 22), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)));
}

// TODO - rename these instructions

namespace {
u32 vec_shl_immhb(u32 elem_bits, u8 shift) {
  ASSERT_MSG(shift < elem_bits, "arm64 vector shift left amount is out of range");
  return elem_bits + shift;
}

u32 vec_shr_immhb(u32 elem_bits, u8 shift) {
  // immh:immb holds 2 * elem_bits minus the shift, so the smallest shift it can encode is 1.
  // x86 allows a shift of 0, which would need a move here instead of a shift.
  ASSERT_MSG(shift >= 1 && shift <= elem_bits, "arm64 vector shift right amount is out of range");
  return 2 * elem_bits - shift;
}
}  // namespace

// - arithmetic_shift_right_32bit_vf
InstructionARM64 pw_sra(Register dst, Register src, u8 imm) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/sshr_advsimd.html
  // - vector, 4S
  // SSHR <Vd>.<T>, <Vn>.<T>, #<shift>
  const u32 v = vec_shr_immhb(32, imm);
  return InstructionARM64(Base(0b0100111100000000000001, 22), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)), Immh(v >> 3), Immb(v & 0b111));
}

// - logical_shift_right_32bit_vf
InstructionARM64 pw_srl(Register dst, Register src, u8 imm) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/ushr_advsimd.html
  // - vector, 4S
  // USHR <Vd>.<T>, <Vn>.<T>, #<shift>
  const u32 v = vec_shr_immhb(32, imm);
  return InstructionARM64(Base(0b0110111100000000000001, 22), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)), Immh(v >> 3), Immb(v & 0b111));
}

// - logical_shift_left_32bit_vf
InstructionARM64 pw_sll(Register dst, Register src, u8 imm) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/shl_advsimd.html
  // - vector, 4S
  // SHL <Vd>.<T>, <Vn>.<T>, #<shift>
  const u32 v = vec_shl_immhb(32, imm);
  return InstructionARM64(Base(0b0100111100000000010101, 22), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)), Immh(v >> 3), Immb(v & 0b111));
}

// - logical_shift_right_16bit_vf
InstructionARM64 ph_srl(Register dst, Register src, u8 imm) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/ushr_advsimd.html
  // - vector, 8H
  // USHR <Vd>.<T>, <Vn>.<T>, #<shift>
  const u32 v = vec_shr_immhb(16, imm);
  return InstructionARM64(Base(0b0110111100000000000001, 22), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)), Immh(v >> 3), Immb(v & 0b111));
}

// - logical_shift_left_16bit_vf
InstructionARM64 ph_sll(Register dst, Register src, u8 imm) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/shl_advsimd.html
  // - vector, 8H
  // SHL <Vd>.<T>, <Vn>.<T>, #<shift>
  const u32 v = vec_shl_immhb(16, imm);
  return InstructionARM64(Base(0b0100111100000000010101, 22), Rn(src.hw_id(instr_set)),
                          Rd(dst.hw_id(instr_set)), Immh(v >> 3), Immb(v & 0b111));
}

InstructionARM64 parallel_add_byte(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/add_advsimd.html
  // - vector, 16B
  // ADD <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0100111000100000100001, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 parallel_bitwise_or(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/orr_advsimd_reg.html
  // - vector, 16B
  // ORR <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0100111010100000000111, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 parallel_bitwise_xor(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/eor_advsimd.html
  // - vector, 16B
  // EOR <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0110111000100000000111, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 parallel_bitwise_and(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/and_advsimd.html
  // - vector, 16B
  // AND <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0100111000100000000111, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

// The pext family is x86's punpck. PUNPCKL/H interleave two vectors, and arm64's interleave
// is ZIP1/ZIP2. UZP1/UZP2 look like the match but deinterleave, which is the opposite.
InstructionARM64 pextub_swapped(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/zip2_advsimd.html
  // - 16B
  // ZIP2 <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0100111000000000011110, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 pextuh_swapped(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/zip2_advsimd.html
  // - 8H
  // ZIP2 <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0100111001000000011110, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 pextuw_swapped(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/zip2_advsimd.html
  // - 4S
  // ZIP2 <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0100111010000000011110, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 pextlb_swapped(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/zip1_advsimd.html
  // - 16B
  // ZIP1 <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0100111000000000001110, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 pextlh_swapped(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/zip1_advsimd.html
  // - 8H
  // ZIP1 <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0100111001000000001110, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 pextlw_swapped(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/zip1_advsimd.html
  // - 4S
  // ZIP1 <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0100111010000000001110, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 parallel_compare_e_b(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/cmeq_advsimd_reg.html
  // - vector, 16B
  // CMEQ <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0110111000100000100011, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 parallel_compare_e_h(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/cmeq_advsimd_reg.html
  // - vector, 8H
  // CMEQ <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0110111001100000100011, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 parallel_compare_e_w(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/cmeq_advsimd_reg.html
  // - vector, 4S
  // CMEQ <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0110111010100000100011, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 parallel_compare_gt_b(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/cmgt_advsimd_reg.html
  // - vector, 16B
  // CMGT <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0100111000100000001101, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 parallel_compare_gt_h(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/cmgt_advsimd_reg.html
  // - vector, 8H
  // CMGT <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0100111001100000001101, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 parallel_compare_gt_w(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/cmgt_advsimd_reg.html
  // - vector, 4S
  // CMGT <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0100111010100000001101, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

// TODO - rename this monstrosity from x86

InstructionARM64 vpunpcklqdq(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/zip1_advsimd.html
  // - vector, 2D
  // ZIP1 <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0100111011000000001110, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 pcpyld_swapped(Register dst, Register src0, Register src1) {
  return vpunpcklqdq(dst, src0, src1);
}

InstructionARM64 pcpyud(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/zip2_advsimd.html
  // - vector, 2D
  // ZIP2 <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0100111011000000011110, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

// TODO - more x86 rename candidates

// lane-wise-32bit-substraction
InstructionARM64 vpsubd(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/sub_advsimd.html
  // - vector, 4S
  // SUB <Vd>.<T>, <Vn>.<T>, <Vm>.<T>
  return InstructionARM64(Base(0b0110111010100000100001, 22), Rn(src0.hw_id(instr_set)),
                          Rm(src1.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

// shift-right-logical-entire-simd-reg
InstructionARM64 vpsrldq(Register dst, Register src, u8 imm) {
  ASSERT(dst.is_128bit_simd(instr_set));
  ASSERT(src.is_128bit_simd(instr_set));
  // zero the high half so EXT does not rotate discarded bytes
  if (imm == 0) {
    // index 0 returns Vn unchanged
    return InstructionARM64(Base(0b0110111000000000000000, 22), Rn(src.hw_id(instr_set)),
                            Rm(src.hw_id(instr_set)), Rd(dst.hw_id(instr_set)), Imm4(0));
  }
  if (imm >= 16) {
    return xor_vf(dst, dst, dst);
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/ext_advsimd.html
  // - 16B
  // EXT <Vd>.<T>, <Vn>.<T>, <Vm>.<T>, #<index>
  // EXT <Vd>.16B, <Vn>.16B, <Vm>.16B, #<index>
  return InstructionARM64({
      xor_vf(V16, V16, V16),
      InstructionARM64(Base(0b0110111000000000000000, 22), Rn(src.hw_id(instr_set)),
                       Rm(Register(V16).hw_id(instr_set)), Rd(dst.hw_id(instr_set)), Imm4(imm)),
  });
}

// shift-left-logical-entire-simd-reg
InstructionARM64 vpslldq(Register dst, Register src, u8 imm) {
  ASSERT(dst.is_128bit_simd(instr_set));
  ASSERT(src.is_128bit_simd(instr_set));
  // zero the low half so EXT does not rotate discarded bytes
  if (imm == 0) {
    return InstructionARM64(Base(0b0110111000000000000000, 22), Rn(src.hw_id(instr_set)),
                            Rm(src.hw_id(instr_set)), Rd(dst.hw_id(instr_set)), Imm4(0));
  }
  if (imm >= 16) {
    return xor_vf(dst, dst, dst);
  }
  // https://www.scs.stanford.edu/~zyedidia/arm64/ext_advsimd.html
  // - 16B
  // EXT <Vd>.<T>, <Vn>.<T>, <Vm>.<T>, #<index>
  // EXT <Vd>.16B, <Vn>.16B, <Vm>.16B, #<index>
  return InstructionARM64({
      xor_vf(V16, V16, V16),
      InstructionARM64(Base(0b0110111000000000000000, 22), Rn(Register(V16).hw_id(instr_set)),
                       Rm(src.hw_id(instr_set)), Rd(dst.hw_id(instr_set)), Imm4(16 - imm)),
  });
}

/*!
 * MOV <Vd>.H[dst_lane], <Vn>.H[src_lane]. the 16 bit version of ins_vf_lane.
 */
InstructionARM64 ins_vf_lane_h(Register dst, u8 dst_lane, Register src, u8 src_lane) {
  ASSERT(dst.is_128bit_simd(instr_set));
  ASSERT(src.is_128bit_simd(instr_set));
  ASSERT(dst_lane < 8 && src_lane < 8);
  // for 16 bit lanes imm5 is (index << 2) | 0b10 and imm4 is index << 1
  return InstructionARM64(Base(0b01101110000000000000010000000000, 32),
                          Imm5(u32(dst_lane << 2) | 0b10), Imm4(u32(src_lane << 1)),
                          Rn(src.hw_id(instr_set)), Rd(dst.hw_id(instr_set)));
}

InstructionARM64 vpshuflw(Register dst, Register src, u8 imm) {
  ASSERT(dst.is_128bit_simd(instr_set));
  ASSERT(src.is_128bit_simd(instr_set));
  // pshuflw shuffles the low four 16 bit lanes by two bits each and copies the high four
  // through. start from a whole copy, then rewrite the low four. v16 so dst may alias src.
  std::vector<InstructionARM64> instrs = {mov_vf_vf(V16, src)};
  for (u8 lane = 0; lane < 4; lane++) {
    instrs.push_back(ins_vf_lane_h(V16, lane, src, (imm >> (lane * 2)) & 0b11));
  }
  instrs.push_back(mov_vf_vf(dst, V16));
  return InstructionARM64(instrs);
}

InstructionARM64 vpshufhw(Register dst, Register src, u8 imm) {
  ASSERT(dst.is_128bit_simd(instr_set));
  ASSERT(src.is_128bit_simd(instr_set));
  // same as vpshuflw, but the high four lanes, selecting from the high four
  std::vector<InstructionARM64> instrs = {mov_vf_vf(V16, src)};
  for (u8 lane = 0; lane < 4; lane++) {
    instrs.push_back(ins_vf_lane_h(V16, u8(4 + lane), src, u8(4 + ((imm >> (lane * 2)) & 0b11))));
  }
  instrs.push_back(mov_vf_vf(dst, V16));
  return InstructionARM64(instrs);
}

InstructionARM64 vpackuswb(Register dst, Register src0, Register src1) {
  // https://www.scs.stanford.edu/~zyedidia/arm64/sqxtun_advsimd.html
  // SQXTUN{2} <Vd>.<Tb>, <Vn>.<Ta>
  return InstructionARM64({
      // sqxtun  vDst.8b,  vSrc0.8h
      InstructionARM64(Base(0b0010111000100001001010, 22), Rn(src0.hw_id(instr_set)),
                       Rd(dst.hw_id(instr_set))),
      // sqxtun2 vDst.16b, vSrc1.8h
      InstructionARM64(Base(0b0110111000100001001010, 22), Rn(src1.hw_id(instr_set)),
                       Rd(dst.hw_id(instr_set))),
  });
}
}  // namespace ARM64
}  // namespace IGen
}  // namespace emitter