From de25f394394187fe800d55505a426c18139f119e Mon Sep 17 00:00:00 2001 From: Parker <317473362+Parker-Aphelion@users.noreply.github.com> Date: Mon, 24 Aug 2026 20:22:47 -0700 Subject: [PATCH] arm64: Apple Silicon support (#4390) Got OpenGOAL building and running natively on Apple Silicon. This is the rest of the port after the smaller arm64 emitter PRs. I was told pushing one big PR was okay. This covers goalc, the runtime, linker, kernel and the GOAL asm. The new paths have native tests. I also found four more emitter bugs while running it. They're in scalar sqrt, 128-bit stores, scalar max and indexed stores above 4 GB. Spent a while cleaning it up so it's easier to read. 4am gotta sleep lol Closes #3841 --------- Co-authored-by: Tyler Wilding --- .github/workflows/compiler-output-check.yaml | 7 +- .github/workflows/macos-build-arm.yaml | 10 +- .github/workflows/release-pipeline.yaml | 8 +- .gitignore | 1 + common/arm64/encoding.h | 20 + common/cross_os_debug/xdbg.cpp | 1 + common/link_types.h | 16 +- game/CMakeLists.txt | 15 +- game/graphics/opengl_renderer/SkyBlendCPU.cpp | 107 +- game/graphics/opengl_renderer/SkyBlendCPU.h | 6 +- game/kernel/asm_funcs_arm64.s | 203 ++- game/kernel/common/codegen.h | 112 ++ game/kernel/common/klink.cpp | 31 + game/kernel/common/klink.h | 16 + game/kernel/common/kscheme.cpp | 56 +- game/kernel/common/kscheme.h | 10 +- game/kernel/jak1/kboot.cpp | 2 +- game/kernel/jak1/klink.cpp | 35 +- game/kernel/jak1/kscheme.cpp | 36 +- game/kernel/jak2/kboot.cpp | 2 +- game/kernel/jak2/klink.cpp | 39 +- game/kernel/jak2/kscheme.cpp | 36 +- game/kernel/jak3/kboot.cpp | 2 +- game/kernel/jak3/klink.cpp | 45 +- game/kernel/jak3/kscheme.cpp | 36 +- game/kernel/jakx/kboot.cpp | 2 +- game/kernel/jakx/klink.cpp | 45 +- game/kernel/jakx/kscheme.cpp | 36 +- game/mips2c/mips2c_private.h | 30 +- game/mips2c/mips2c_table.cpp | 34 +- game/runtime.cpp | 48 +- game/runtime.h | 2 + game/system/IOP_Kernel.cpp | 2 +- game/system/IOP_Kernel.h | 2 +- goal_src/goal-lib.gc | 2 + goal_src/jak1/kernel/gkernel-h.gc | 3 +- goal_src/jak1/kernel/gkernel.gc | 1305 ++++++++++----- goal_src/jak1/kernel/gstate.gc | 123 +- goal_src/jak2/kernel/gkernel-h.gc | 6 +- goal_src/jak2/kernel/gkernel.gc | 1409 ++++++++++------ goal_src/jak2/kernel/gstate.gc | 98 +- goal_src/jak3/kernel/gkernel-h.gc | 6 +- goal_src/jak3/kernel/gkernel.gc | 1403 ++++++++++------ goal_src/jak3/kernel/gstate.gc | 98 +- goal_src/jakx/kernel/gkernel-h.gc | 6 +- goal_src/jakx/kernel/gkernel.gc | 1403 ++++++++++------ goalc/compiler/CodeGenerator.cpp | 215 ++- goalc/compiler/CodeGenerator.h | 3 +- goalc/compiler/Compiler.cpp | 8 +- goalc/compiler/Compiler.h | 2 + goalc/compiler/IR.cpp | 488 ++++-- goalc/compiler/IR.h | 20 + goalc/compiler/StaticObject.cpp | 4 +- goalc/compiler/StaticObject.h | 2 +- goalc/compiler/Val.cpp | 28 +- goalc/compiler/Val.h | 10 +- goalc/compiler/compilation/Asm.cpp | 137 +- goalc/compiler/compilation/Atoms.cpp | 2 + goalc/compiler/compilation/Block.cpp | 10 +- goalc/compiler/compilation/Define.cpp | 28 +- goalc/compiler/compilation/Function.cpp | 12 +- goalc/compiler/compilation/Math.cpp | 41 +- goalc/compiler/compilation/Static.cpp | 7 +- goalc/compiler/compilation/Type.cpp | 25 +- goalc/debugger/Debugger.cpp | 1 + goalc/emitter/CallingConvention.cpp | 24 +- goalc/emitter/CallingConvention.h | 7 +- goalc/emitter/CodeTester.cpp | 49 +- goalc/emitter/CodeTester.h | 15 +- goalc/emitter/IGen.cpp | 200 +-- goalc/emitter/IGen.h | 177 +- goalc/emitter/IGenARM64.cpp | 1199 ++++++++------ goalc/emitter/IGenARM64.h | 136 +- goalc/emitter/IGenX86.cpp | 89 +- goalc/emitter/IGenX86.h | 59 +- goalc/emitter/Instruction.h | 46 +- goalc/emitter/InstructionSet.h | 11 +- goalc/emitter/ObjectGenerator.cpp | 113 +- goalc/emitter/ObjectGenerator.h | 2 + goalc/emitter/Register.cpp | 209 ++- goalc/emitter/Register.h | 143 +- goalc/main.cpp | 19 +- goalc/regalloc/Allocator.cpp | 106 +- goalc/regalloc/Allocator.h | 3 +- goalc/regalloc/Allocator_v2.cpp | 203 ++- goalc/regalloc/IRegister.cpp | 7 +- goalc/regalloc/IRegister.h | 2 +- goalc/regalloc/allocator_interface.cpp | 40 +- goalc/regalloc/allocator_interface.h | 35 +- test/CMakeLists.txt | 4 + test/goalc/CMakeLists.txt | 54 +- .../source_templates/jak2/kernel-test.gc | 144 +- .../source_templates/jak3/kernel-test.gc | 144 +- .../source_templates/kernel/kernel-test.gc | 114 +- .../variables/inline-asm.static.gc | 69 +- .../with_game/string-to-float.static.gc | 7 + .../with_game/test-vf-load-and-store.gc | 16 +- test/goalc/test_CodeTester.cpp | 75 +- test/goalc/test_arithmetic.cpp | 2 +- test/goalc/test_collections.cpp | 2 +- test/goalc/test_compiler.cpp | 4 +- test/goalc/test_control_statements.cpp | 2 +- test/goalc/test_game_no_debug.cpp | 2 +- test/goalc/test_goal_kernel.cpp | 30 +- test/goalc/test_goal_kernel2.cpp | 30 +- test/goalc/test_goal_kernel3.cpp | 30 +- test/goalc/test_jak2_compiler.cpp | 2 +- test/goalc/test_type_consistency.cpp | 16 +- test/goalc/test_variables.cpp | 2 +- test/goalc/test_vector_float.cpp | 2 +- test/goalc/test_with_game.cpp | 32 +- test/test_emitter_arm64.cpp | 1426 ++++++++++++++--- test/test_emitter_arm64_alu.cpp | 1036 ++++++++++++ test/test_emitter_arm64_memory.cpp | 231 +++ test/test_emitter_x86.cpp | 69 +- test/test_goal_trampolines_arm64.cpp | 360 +++++ test/test_sky_blend.cpp | 98 ++ 117 files changed, 10816 insertions(+), 4039 deletions(-) create mode 100644 common/arm64/encoding.h create mode 100644 game/kernel/common/codegen.h create mode 100644 test/goalc/source_templates/with_game/string-to-float.static.gc create mode 100644 test/test_emitter_arm64_alu.cpp create mode 100644 test/test_emitter_arm64_memory.cpp create mode 100644 test/test_goal_trampolines_arm64.cpp create mode 100644 test/test_sky_blend.cpp diff --git a/.github/workflows/compiler-output-check.yaml b/.github/workflows/compiler-output-check.yaml index db08001a95..3a7aec60a1 100644 --- a/.github/workflows/compiler-output-check.yaml +++ b/.github/workflows/compiler-output-check.yaml @@ -66,9 +66,10 @@ jobs: - name: Compile and preserve (master) run: | - ./build.master/goalc/goalc --game jak1 --cmd "(make-group \"all-code\")" - ./build.master/goalc/goalc --game jak2 --cmd "(make-group \"all-code\")" - ./build.master/goalc/goalc --game jak3 --cmd "(make-group \"all-code\")" + # master predates the .break compiler form, so use its old expansion for the comparison + ./build.master/goalc/goalc --game jak1 --cmd '(begin (defmacro .break () (quote (/ 0 0))) (make-group "all-code"))' + ./build.master/goalc/goalc --game jak2 --cmd '(begin (defmacro .break () (quote (/ 0 0))) (make-group "all-code"))' + ./build.master/goalc/goalc --game jak3 --cmd '(begin (defmacro .break () (quote (/ 0 0))) (make-group "all-code"))' mv ./out/jak1/obj ./out/jak1/obj.master mv ./out/jak2/obj ./out/jak2/obj.master mv ./out/jak3/obj ./out/jak3/obj.master diff --git a/.github/workflows/macos-build-arm.yaml b/.github/workflows/macos-build-arm.yaml index 6798d4c80e..91ae7c82a9 100644 --- a/.github/workflows/macos-build-arm.yaml +++ b/.github/workflows/macos-build-arm.yaml @@ -22,8 +22,16 @@ jobs: steps: - name: Checkout Repository + if: ${{ ! inputs.uploadArtifacts }} uses: actions/checkout@v7 + - name: Checkout Repository with Tags + if: ${{ inputs.uploadArtifacts }} + uses: actions/checkout@v7 + with: + fetch-depth: 0 + fetch-tags: true + - name: Install Package Dependencies env: HOMEBREW_NO_INSTALL_CLEANUP: 1 @@ -74,7 +82,7 @@ jobs: uses: actions/upload-artifact@v7 if: ${{ inputs.uploadArtifacts }} with: - name: opengoal-macos-arm-${{ inputs.cachePrefix }} + name: opengoal-macos-${{ inputs.cachePrefix }} if-no-files-found: error path: | ./build/goalc/goalc diff --git a/.github/workflows/release-pipeline.yaml b/.github/workflows/release-pipeline.yaml index f7697db675..e51df4c38f 100644 --- a/.github/workflows/release-pipeline.yaml +++ b/.github/workflows/release-pipeline.yaml @@ -79,13 +79,13 @@ jobs: uploadArtifacts: true secrets: inherit - build_macos_arm_rosetta: + build_macos_arm: needs: - cut_release name: "🍎 MacOS" - uses: ./.github/workflows/macos-build.yaml + uses: ./.github/workflows/macos-build-arm.yaml with: - cmakePreset: "Release-macos-x86_64-clang-static" + cmakePreset: "Release-macos-arm64-clang-static" cachePrefix: "static-arm" uploadArtifacts: true secrets: inherit @@ -98,7 +98,7 @@ jobs: - build_windows_clang - build_linux_clang - build_macos_intel - - build_macos_arm_rosetta + - build_macos_arm name: "Upload Artifacts" runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 95afe75b78..999cf40e70 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ cmake-build-debug/* cmake-build-debug--o0/* .idea/* build/ +build-debug/ build.master/ /decompiler_out* logs/* diff --git a/common/arm64/encoding.h b/common/arm64/encoding.h new file mode 100644 index 0000000000..58dd89b6fb --- /dev/null +++ b/common/arm64/encoding.h @@ -0,0 +1,20 @@ +#pragma once + +#include "common/common_types.h" +#include "common/util/Assert.h" + +namespace arm64 { + +constexpr u32 encode_movz_64(u32 reg, u16 value, u32 halfword) { + ASSERT(reg < 32); + ASSERT(halfword < 4); + return 0xd2800000 | (halfword << 21) | (u32(value) << 5) | reg; +} + +constexpr u32 encode_movk_64(u32 reg, u16 value, u32 halfword) { + ASSERT(reg < 32); + ASSERT(halfword < 4); + return 0xf2800000 | (halfword << 21) | (u32(value) << 5) | reg; +} + +} // namespace arm64 diff --git a/common/cross_os_debug/xdbg.cpp b/common/cross_os_debug/xdbg.cpp index 69758605c8..e1f7cf3b9b 100644 --- a/common/cross_os_debug/xdbg.cpp +++ b/common/cross_os_debug/xdbg.cpp @@ -726,6 +726,7 @@ bool set_regs_now(const ThreadID& tid, const Regs& out) { return true; } #elif __APPLE__ +// macOS debugger backend is not implemented ThreadID::ThreadID(const std::string& str) {} std::string ThreadID::to_string() const { diff --git a/common/link_types.h b/common/link_types.h index 4feae73d8e..032a8de975 100644 --- a/common/link_types.h +++ b/common/link_types.h @@ -14,8 +14,22 @@ enum LinkKind { LINK_DISTANCE_TO_OTHER_SEG_64 = 3, //! link to another segment LINK_DISTANCE_TO_OTHER_SEG_32 = 4, //! link to another segment LINK_PTR = 5, //! link a pointer within this segment + LINK_ARM64_SYMBOL_MOV32 = 6, //! symbol offset encoded in a movz/movk pair + LINK_ARM64_OTHER_SEG_MOV32 = 7, //! cross-segment address encoded in a movz/movk pair }; +/*! + * Read a 32-bit value from an ARM64 movz/movk pair. + */ +inline u32 arm64_read_mov32(const u32* words) { + return ((words[0] >> 5) & 0xffff) | (((words[1] >> 5) & 0xffff) << 16); +} + +inline void arm64_write_mov32(u32* words, u32 value) { + words[0] = (words[0] & ~(0xffffu << 5)) | ((value & 0xffff) << 5); + words[1] = (words[1] & ~(0xffffu << 5)) | (((value >> 16) & 0xffff) << 5); +} + enum SegmentTypes { MAIN_SEGMENT = 0, DEBUG_SEGMENT = 1, TOP_LEVEL_SEGMENT = 2 }; constexpr const char* SEGMENT_NAMES[3] = {"main", "debug", "top-level"}; @@ -71,4 +85,4 @@ struct LinkHeaderV5 { // when a u32/s32 symbol link contains this value, (s7 + ) should be a 4-byte aligned address, // not including the 1 byte symbol offset. (no effect in jak 1). -constexpr u32 LINK_SYM_NO_OFFSET_FLAG = 0xbadbeef; \ No newline at end of file +constexpr u32 LINK_SYM_NO_OFFSET_FLAG = 0xbadbeef; diff --git a/game/CMakeLists.txt b/game/CMakeLists.txt index 2a7b5c4002..efdf194ee6 100644 --- a/game/CMakeLists.txt +++ b/game/CMakeLists.txt @@ -1,10 +1,12 @@ # Set a more convenient ARM flag -if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") +# match the __aarch64__ source guards +string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" OG_PROCESSOR_LOWER) +if(OG_PROCESSOR_LOWER MATCHES "^(arm64|aarch64)$") set(ARM64_ARCH TRUE) - message(STATUS "ARM64 architecture detected") + message(STATUS "ARM64 architecture detected (${CMAKE_SYSTEM_PROCESSOR})") else() set(ARM64_ARCH FALSE) - message(STATUS "Non-ARM64 architecture detected") + message(STATUS "Non-ARM64 architecture detected (${CMAKE_SYSTEM_PROCESSOR})") endif() if(ARM64_ARCH) @@ -12,7 +14,12 @@ if(ARM64_ARCH) set(OG_ASM_FUNCS_FILE kernel/asm_funcs_arm64.s) enable_language(ASM) set(CMAKE_ASM_SOURCE_FILE_EXTENSIONS ${CMAKE_ASM_SOURCE_FILE_EXTENSIONS} s) - set_source_files_properties(${OG_ASM_FUNCS_FILE} PROPERTIES COMPILE_FLAGS "-arch arm64 -g") + # pass -arch only to Apple Clang + if(APPLE) + set_source_files_properties(${OG_ASM_FUNCS_FILE} PROPERTIES COMPILE_FLAGS "-arch arm64 -g") + else() + set_source_files_properties(${OG_ASM_FUNCS_FILE} PROPERTIES COMPILE_FLAGS "-g") + endif() else() set(OG_ASM_FUNCS_FILE kernel/asm_funcs_x86_64.asm) enable_language(ASM_NASM) diff --git a/game/graphics/opengl_renderer/SkyBlendCPU.cpp b/game/graphics/opengl_renderer/SkyBlendCPU.cpp index c70d47d32a..b73f29eb3f 100644 --- a/game/graphics/opengl_renderer/SkyBlendCPU.cpp +++ b/game/graphics/opengl_renderer/SkyBlendCPU.cpp @@ -5,6 +5,10 @@ #include "game/graphics/opengl_renderer/AdgifHandler.h" +#ifdef __aarch64__ +#include +#endif + SkyBlendCPU::SkyBlendCPU() { for (int i = 0; i < 2; i++) { glGenTextures(1, &m_textures[i].gl); @@ -21,10 +25,28 @@ SkyBlendCPU::~SkyBlendCPU() { } } +/*! + * out[i] = saturate_u8((in[i] * intensity) >> 7) + */ void blend_sky_initial_fast(u8 intensity, u8* out, const u8* in, u32 size) { -#ifndef __arm64__ - if (get_cpu_info().has_avx2) { +#ifdef __aarch64__ + // widen u8 to u16, multiply, shift, then narrow with saturation. 255*255 fits in a u16 so + // the multiply can't overflow, and vqmovn_u16 clamps at 255 like packus does. + const uint16x8_t intensity_vec = vdupq_n_u16(intensity); + u32 i = 0; + for (; i + 16 <= size; i += 16) { + const uint8x16_t tex = vld1q_u8(in + i); + const uint16x8_t lo = vshrq_n_u16(vmulq_u16(vmovl_u8(vget_low_u8(tex)), intensity_vec), 7); + const uint16x8_t hi = vshrq_n_u16(vmulq_u16(vmovl_u8(vget_high_u8(tex)), intensity_vec), 7); + vst1q_u8(out + i, vcombine_u8(vqmovn_u16(lo), vqmovn_u16(hi))); + } + for (; i < size; i++) { + const u32 v = ((u32)in[i] * intensity) >> 7; + out[i] = v > 255 ? 255 : (u8)v; + } +#else #ifdef __AVX2__ + if (get_cpu_info().has_avx2) { __m256i intensity_vec = _mm256_set1_epi16(intensity); for (u32 i = 0; i < size / 16; i++) { __m128i tex_data8 = _mm_loadu_si128((const __m128i*)(in + (i * 16))); @@ -35,27 +57,47 @@ void blend_sky_initial_fast(u8 intensity, u8* out, const u8* in, u32 size) { auto result = _mm_packus_epi16(_mm256_castsi256_si128(tex_data16), hi); _mm_storeu_si128((__m128i*)(out + (i * 16)), result); } -#else - ASSERT(false); + return; + } #endif - } else { - __m128i intensity_vec = _mm_set1_epi16(intensity); - for (u32 i = 0; i < size / 8; i++) { - __m128i tex_data8 = _mm_loadu_si64((const __m128i*)(in + (i * 8))); - __m128i tex_data16 = _mm_cvtepu8_epi16(tex_data8); - tex_data16 = _mm_mullo_epi16(tex_data16, intensity_vec); - tex_data16 = _mm_srli_epi16(tex_data16, 7); - auto result = _mm_packus_epi16(tex_data16, tex_data16); - _mm_storel_epi64((__m128i*)(out + (i * 8)), result); - } + + __m128i intensity_vec = _mm_set1_epi16(intensity); + for (u32 i = 0; i < size / 8; i++) { + __m128i tex_data8 = _mm_loadu_si64((const __m128i*)(in + (i * 8))); + __m128i tex_data16 = _mm_cvtepu8_epi16(tex_data8); + tex_data16 = _mm_mullo_epi16(tex_data16, intensity_vec); + tex_data16 = _mm_srli_epi16(tex_data16, 7); + auto result = _mm_packus_epi16(tex_data16, tex_data16); + _mm_storel_epi64((__m128i*)(out + (i * 8)), result); } #endif } +/*! + * out[i] = saturating_add_u8(out[i], saturate_u8((in[i] * intensity) >> 7)) + */ void blend_sky_fast(u8 intensity, u8* out, const u8* in, u32 size) { -#ifndef __arm64__ - if (get_cpu_info().has_avx2) { +#ifdef __aarch64__ + const uint16x8_t intensity_vec = vdupq_n_u16(intensity); + u32 i = 0; + for (; i + 16 <= size; i += 16) { + const uint8x16_t tex = vld1q_u8(in + i); + const uint8x16_t cur = vld1q_u8(out + i); + const uint16x8_t lo = vshrq_n_u16(vmulq_u16(vmovl_u8(vget_low_u8(tex)), intensity_vec), 7); + const uint16x8_t hi = vshrq_n_u16(vmulq_u16(vmovl_u8(vget_high_u8(tex)), intensity_vec), 7); + // vqmovn_u16 clamps at 255, so it covers both the min and the packus the x86 path does + vst1q_u8(out + i, vqaddq_u8(cur, vcombine_u8(vqmovn_u16(lo), vqmovn_u16(hi)))); + } + for (; i < size; i++) { + u32 v = ((u32)in[i] * intensity) >> 7; + if (v > 255) + v = 255; + const u32 sum = out[i] + v; + out[i] = sum > 255 ? 255 : (u8)sum; + } +#else #ifdef __AVX2__ + if (get_cpu_info().has_avx2) { __m256i intensity_vec = _mm256_set1_epi16(intensity); __m256i max_intensity = _mm256_set1_epi16(255); for (u32 i = 0; i < size / 16; i++) { @@ -70,23 +112,22 @@ void blend_sky_fast(u8 intensity, u8* out, const u8* in, u32 size) { out_val = _mm_adds_epu8(out_val, result); _mm_storeu_si128((__m128i*)(out + (i * 16)), out_val); } -#else - ASSERT(false); + return; + } #endif - } else { - __m128i intensity_vec = _mm_set1_epi16(intensity); - __m128i max_intensity = _mm_set1_epi16(255); - for (u32 i = 0; i < size / 8; i++) { - __m128i tex_data8 = _mm_loadu_si64((const __m128i*)(in + (i * 8))); - __m128i out_val = _mm_loadu_si64((const __m128i*)(out + (i * 8))); - __m128i tex_data16 = _mm_cvtepu8_epi16(tex_data8); - tex_data16 = _mm_mullo_epi16(tex_data16, intensity_vec); - tex_data16 = _mm_srli_epi16(tex_data16, 7); - tex_data16 = _mm_min_epi16(max_intensity, tex_data16); - auto result = _mm_packus_epi16(tex_data16, tex_data16); - out_val = _mm_adds_epu8(out_val, result); - _mm_storel_epi64((__m128i*)(out + (i * 8)), out_val); - } + + __m128i intensity_vec = _mm_set1_epi16(intensity); + __m128i max_intensity = _mm_set1_epi16(255); + for (u32 i = 0; i < size / 8; i++) { + __m128i tex_data8 = _mm_loadu_si64((const __m128i*)(in + (i * 8))); + __m128i out_val = _mm_loadu_si64((const __m128i*)(out + (i * 8))); + __m128i tex_data16 = _mm_cvtepu8_epi16(tex_data8); + tex_data16 = _mm_mullo_epi16(tex_data16, intensity_vec); + tex_data16 = _mm_srli_epi16(tex_data16, 7); + tex_data16 = _mm_min_epi16(max_intensity, tex_data16); + auto result = _mm_packus_epi16(tex_data16, tex_data16); + out_val = _mm_adds_epu8(out_val, result); + _mm_storel_epi64((__m128i*)(out + (i * 8)), out_val); } #endif } @@ -205,4 +246,4 @@ void SkyBlendCPU::init_textures(TexturePool& tex_pool, GameVersion version) { m_textures[i].tex = tex_pool.give_texture_and_load_to_vram(in, tbp); m_textures[i].tbp = tbp; } -} \ No newline at end of file +} diff --git a/game/graphics/opengl_renderer/SkyBlendCPU.h b/game/graphics/opengl_renderer/SkyBlendCPU.h index e4e0d8bef4..a6dc3913a3 100644 --- a/game/graphics/opengl_renderer/SkyBlendCPU.h +++ b/game/graphics/opengl_renderer/SkyBlendCPU.h @@ -6,6 +6,10 @@ #include "game/graphics/opengl_renderer/SkyBlendCommon.h" #include "game/graphics/pipelines/opengl.h" +// declared here so the tests can call them +void blend_sky_initial_fast(u8 intensity, u8* out, const u8* in, u32 size); +void blend_sky_fast(u8 intensity, u8* out, const u8* in, u32 size); + class SkyBlendCPU { public: SkyBlendCPU(); @@ -25,4 +29,4 @@ class SkyBlendCPU { u32 tbp; GpuTexture* tex; } m_textures[2]; -}; \ No newline at end of file +}; diff --git a/game/kernel/asm_funcs_arm64.s b/game/kernel/asm_funcs_arm64.s index 9244ee84ae..2b1a5feb51 100644 --- a/game/kernel/asm_funcs_arm64.s +++ b/game/kernel/asm_funcs_arm64.s @@ -16,9 +16,8 @@ _arg_call_arm64: stp x29, x30, [sp, #-16]! mov x29, sp - ldr x8, [sp], #16 - ; Putting an exclamation point after the close-bracket + ; Putting an exclamation point after the close-bracket ; means that the calculated effective address is written back to the base register. (pre-indexing) stp q15, q14, [sp, #-32]! stp q13, q12, [sp, #-32]! @@ -27,10 +26,11 @@ _arg_call_arm64: blr x8 + ;; restore the vector pairs in the same order they were saved ldp q9, q8, [sp], #32 - ldp q10, q11, [sp], #32 - ldp q12, q13, [sp], #32 - ldp q14, q15, [sp], #32 + ldp q11, q10, [sp], #32 + ldp q13, q12, [sp], #32 + ldp q15, q14, [sp], #32 ldp x29, x30, [sp], #16 ret @@ -40,14 +40,13 @@ _arg_call_arm64: ;; ;; Put arguments on the stack and put a pointer to this array in the first arg. ;; this function pushes all 8 OpenGOAL registers into a stack array. -;; then it calls the function pointed to by x0 (RAX in x86) with a pointer to this array. +;; it calls the function in x8 with a pointer to this array. ;; it returns the return value of the called function. .global _stack_call_arm64 .align 4 _stack_call_arm64: stp x29, x30, [sp, #-16]! mov x29, sp - ldr x8, [sp], #16 stp q15, q14, [sp, #-32]! stp q13, q12, [sp, #-32]! @@ -63,25 +62,29 @@ _stack_call_arm64: ; arg 2 (RDX in x86) ; arg 1 (RSI in x86) ; arg 0 (RDI in x86) - stp x7, x6, [sp, #-16]! - stp x5, x4, [sp, #-16]! - stp x3, x2, [sp, #-16]! - stp x1, x0, [sp, #-16]! + ; put x0 at the lowest address and x7 at the highest + stp x6, x7, [sp, #-16]! + stp x4, x5, [sp, #-16]! + stp x2, x3, [sp, #-16]! + stp x0, x1, [sp, #-16]! ; set first argument - mov x19, sp + ; pass the array address as the first argument + mov x0, sp ; call function blr x8 - ; restore arguments - ldp x1, x0, [sp], #16 - ldp x3, x2, [sp], #16 - ldp x5, x4, [sp], #16 - ldp x7, x6, [sp], #16 + ; keep x0 because it holds the return value + ldr x1, [sp, #8] + ldp x2, x3, [sp, #16] + ldp x4, x5, [sp, #32] + ldp x6, x7, [sp, #48] + add sp, sp, #64 + ;; restore the vector pairs in the same order they were saved ldp q9, q8, [sp], #32 - ldp q10, q11, [sp], #32 - ldp q12, q13, [sp], #32 - ldp q14, q15, [sp], #32 + ldp q11, q10, [sp], #32 + ldp q13, q12, [sp], #32 + ldp q15, q14, [sp], #32 ldp x29, x30, [sp], #16 ; return! @@ -89,71 +92,78 @@ _stack_call_arm64: ;; Call c++ code through mips2c. ;; GOAL will call a dynamically generated trampoline. -;; The trampoline will have pushed the exec function and stack offset onto the stack +;; x9 holds the C function and x10 holds the fake GOAL stack size. .global _mips2c_call_arm64 .align 4 _mips2c_call_arm64: stp x29, x30, [sp, #-16]! mov x29, sp - ;; TODO - this is really weird using half an XMM, this makes the arm assembly - ;; more difficult - this probably isn't required for arm? - ;; grab the address to call and put it in xmm0 - ;; TODO - this stack pointer manipulation might be a problem for ARM64 which requires 16byte alignment - ;; sub sp, 8 - ldr q0, [sp, #+16] - ;; grab the stack offset - ldr x0, [sp, #+8] ;; first, save quadword registers - stp q15, q14, [sp, #-32]! - stp q13, q12, [sp, #-32]! - stp q11, q10, [sp, #-32]! - stp q9, q8, [sp, #-32]! - - ; NOTE - in x86 the 2 special registers are saved (R10 and R11) - ; we don't need to do that in ARM64, there are plenty of registers to work with + ;; save all 128 bits because AAPCS64 only preserves the low half + sub sp, sp, #128 + stp q8, q9, [sp] + stp q10, q11, [sp, #32] + stp q12, q13, [sp, #64] + stp q14, q15, [sp, #96] ;; oof - sub sp, sp, 1280 - str x0, [sp, #+64] ; arg 0 (RDI in x86) and - str x1, [sp, #+80] ; arg 1 (RSI in x86) - str x2, [sp, #+96] ; arg 2 (RDX in x86) and arg 3 (RCX in x86) - str x3, [sp, #+112] ; arg 2 (RDX in x86) and arg 3 (RCX in x86) - str x4, [sp, #+128] ; arg 4 (R8 in x86) and arg 5 (R8 in x86) - str x5, [sp, #+144] ; arg 4 (R8 in x86) and arg 5 (R8 in x86) - str x6, [sp, #+160] ; arg 6 (R10 in x86) and arg 7 (R11 in x86) - str x7, [sp, #+176] ; arg 6 (R10 in x86) and arg 7 (R11 in x86) - str x20, [sp, #+352] ;; s6 (pp) (R13 in x86) and s7 (st) (R14 in x86) - str x21, [sp, #+368] ;; s6 (pp) (R13 in x86) and s7 (st) (R14 in x86) + ;; 1280-byte MIPS register context + sub sp, sp, #1280 + ; arg 0 (RDI in x86) and + str x0, [sp, #64] ;; a0 + ; arg 1 (RSI in x86) + str x1, [sp, #80] ;; a1 + ; arg 2 (RDX in x86) and arg 3 (RCX in x86) + str x2, [sp, #96] ;; a2 + ; arg 2 (RDX in x86) and arg 3 (RCX in x86) + str x3, [sp, #112] ;; a3 + ; arg 4 (R8 in x86) and arg 5 (R8 in x86) + str x4, [sp, #128] ;; t0 + ; arg 4 (R8 in x86) and arg 5 (R8 in x86) + str x5, [sp, #144] ;; t1 + ; arg 6 (R10 in x86) and arg 7 (R11 in x86) + str x6, [sp, #160] ;; t2 + ; arg 6 (R10 in x86) and arg 7 (R11 in x86) + str x7, [sp, #176] ;; t3 + ;; s6 (pp) (R13 in x86) and s7 (st) (R14 in x86) + str x20, [sp, #352] ;; s6 (pp) + ;; s6 (pp) (R13 in x86) and s7 (st) (R14 in x86) + str x21, [sp, #368] ;; s7 (st) - mov x0, sp ; move the stack pointer to arg 0 - sub x0, x0, x22 ; R15 is a "special" offset TODO - whats special about it? - str x0, [sp, #+464] ;; mip2c code's MIPS stack + ;; store the context as a GOAL pointer in the MIPS sp slot + mov x11, sp + sub x11, x11, x22 + ;; mip2c code's MIPS stack + str x11, [sp, #464] ;; r29 (sp) - mov x0, sp ;; move the stack pointer to the new position + ; move the stack pointer to arg 0 + mov x0, sp ;; pass the context as the sole argument - sub sp, sp, x8 ;; allocate space on the stack for GOAL fake stack - stp x8, x8, [sp, #-16]! ;; and remember this so we can find our way back + ;; allocate space on the stack for GOAL fake stack + ;; round the fake GOAL stack to 16 bytes + add x11, x10, #15 + and x11, x11, #-16 + sub sp, sp, x11 + ;; and remember this so we can find our way back + str x11, [sp, #-16]! ;; save the stack size across the C call - ;; TODO - this used to be a movq rax, xmm0 - ;; TODO - not sure why an `xmm` was used because that movq only uses the lower 64bits anyway - mov x0, v0.d[0] ; represents the lower 64 bits of q0 - blr x8 ;; call! + ;; call! + blr x9 ;; unallocate - ldp x8, x8, [sp], #16 - add sp, sp, x8 + ldr x11, [sp], #16 + add sp, sp, x11 ;; restore sp to the context base - ldr x8, [sp, #+32] + ldr x0, [sp, #32] ;; return the v0 slot written by mips2c - add sp, sp, 1280 ; reset the stackpointer back - - ldp q9, q8, [sp], #32 - ldp q10, q11, [sp], #32 - ldp q12, q13, [sp], #32 - ldp q14, q15, [sp], #32 - - add sp, sp, 24 ;; 16 for the stuff pushed by trampoline + ; reset the stackpointer back + add sp, sp, #1280 + ldp q8, q9, [sp] + ldp q10, q11, [sp, #32] + ldp q12, q13, [sp, #64] + ldp q14, q15, [sp, #96] + add sp, sp, #128 ldp x29, x30, [sp], #16 ret @@ -175,7 +185,12 @@ _call_goal_asm_arm64: ;; saved registers we need to modify for GOAL should be preserved ; ARM64 requires 16-byte stack pointer alignment stp x20, x21, [sp, #-16]! - str x22, [sp, #-16]! + stp x22, x27, [sp, #-16]! + sub sp, sp, #128 + stp q8, q9, [sp] + stp q10, q11, [sp, #32] + stp q12, q13, [sp, #64] + stp q14, q15, [sp, #96] ;; x0 - first arg ;; x1 - second arg @@ -183,6 +198,7 @@ _call_goal_asm_arm64: ;; x3 - function pointer ;; x4 - st (goes in x20 and x21) ;; x5 - off (goes in x22) + ;; x6 holds the executable mapping base for x27 ;; set GOAL process mov x20, x4 @@ -190,11 +206,17 @@ _call_goal_asm_arm64: mov x21, x4 ;; offset mov x22, x5 + mov x27, x6 ;; call GOAL by function pointer blr x3 ;; restore saved registers. - ldr x22, [sp], #16 + ldp q8, q9, [sp] + ldp q10, q11, [sp, #32] + ldp q12, q13, [sp, #64] + ldp q14, q15, [sp, #96] + add sp, sp, #128 + ldp x22, x27, [sp], #16 ldp x20, x21, [sp], #16 ldp x29, x30, [sp], #16 ret @@ -207,7 +229,12 @@ _call_goal8_asm_arm64: ;; saved registers we need to modify for GOAL should be preserved ; ARM64 requires 16-byte stack pointer alignment stp x20, x21, [sp, #-16]! - str x22, [sp, #-16]! + stp x22, x27, [sp, #-16]! + sub sp, sp, #128 + stp q8, q9, [sp] + stp q10, q11, [sp, #32] + stp q12, q13, [sp, #64] + stp q14, q15, [sp, #96] ;; x0 - first arg (func) ;; x1 - second arg (arg array) @@ -215,6 +242,7 @@ _call_goal8_asm_arm64: ;; x3 - pp (goes in r13) ;; x4 - st (goes in r14) ;; x5 - off (goes in r15) + ;; x6 holds the executable mapping base for x27 ;; set GOAL function pointer mov x20, x3 @@ -222,6 +250,8 @@ _call_goal8_asm_arm64: mov x21, x4 ;; offset mov x22, x5 + ;; save the executable mapping before loading x6 from the argument array + mov x27, x6 ;; move function to temp mov x8, x0 ;; extract arguments @@ -237,7 +267,12 @@ _call_goal8_asm_arm64: blr x8 ;; retore registers. - ldr x22, [sp], #16 + ldp q8, q9, [sp] + ldp q10, q11, [sp, #32] + ldp q12, q13, [sp, #64] + ldp q14, q15, [sp, #96] + add sp, sp, #128 + ldp x22, x27, [sp], #16 ldp x20, x21, [sp], #16 ldp x29, x30, [sp], #16 ret @@ -254,27 +289,43 @@ _call_goal_on_stack_asm_arm64: ;; x3 - function pointer ;; x4 - st (goes in x21 and x20) ;; x5 - offset (goes in x22) + ;; x6 holds the executable mapping base for x27 ;; saved registers we need to modify for GOAL should be preserved ; ARM64 requires 16-byte stack pointer alignment stp x20, x21, [sp, #-16]! + stp x22, x27, [sp, #-16]! + sub sp, sp, #128 + stp q8, q9, [sp] + stp q10, q11, [sp, #32] + stp q12, q13, [sp, #64] + stp q14, q15, [sp, #96] + ;; also stash the current stack pointer on the stack ;; NOTE - you cannot directly store or load the `sp` register in arm64 + ;; save the native sp through x9 because str cannot use sp as data mov x9, sp - stp x22, x9, [sp, #-16]! - ;; switch to new stack mov sp, x0 + str x9, [sp, #-16]! - mov x20, x4 ;; set GOAL function pointer + ;; set GOAL function pointer + mov x20, x4 ;; set GOAL process mov x21, x4 ;; symbol table mov x22, x5 ;; offset + mov x27, x6 ;; call GOAL by function pointer blr x3 ;; restore registers - ldp x22, x9, [sp], #16 + ldr x9, [sp], #16 mov sp, x9 + ldp q8, q9, [sp] + ldp q10, q11, [sp, #32] + ldp q12, q13, [sp, #64] + ldp q14, q15, [sp, #96] + add sp, sp, #128 + ldp x22, x27, [sp], #16 ldp x20, x21, [sp], #16 ldp x29, x30, [sp], #16 ret diff --git a/game/kernel/common/codegen.h b/game/kernel/common/codegen.h new file mode 100644 index 0000000000..dad04bde7c --- /dev/null +++ b/game/kernel/common/codegen.h @@ -0,0 +1,112 @@ +#pragma once + +/*! + * @file codegen.h + * Helpers for the small machine code stubs the kernel writes at runtime. + */ + +#include + +#include "common/arm64/encoding.h" +#include "common/common_types.h" + +#include "game/runtime.h" + +#ifdef __APPLE__ +#include +#endif + +/*! + * Makes freshly written instructions visible to the CPU. + */ +inline void flush_icache(void* addr, int size) { +#ifdef __aarch64__ +#ifdef __APPLE__ + sys_icache_invalidate(addr, size); +#else + __builtin___clear_cache((char*)addr, (char*)addr + size); +#endif +#else + (void)addr; + (void)size; +#endif +} + +/*! + * Makes EE instructions visible through their executable mapping. + */ +inline void flush_icache_goal(u32 goal_addr, u32 size) { + flush_icache(g_ee_main_mem_exec + goal_addr, (int)size); +} + +#ifdef __aarch64__ + +/*! + * Emits a fixed movz/movk sequence for a 64-bit value. + */ +inline int emit_arm64_mov64(u8* dst, u32 reg, u64 val) { + u32 instr = arm64::encode_movz_64(reg, u16(val & 0xffff), 0); + memcpy(dst, &instr, 4); + int offset = 4; + for (u32 halfword = 1; halfword < 4; halfword++) { + instr = arm64::encode_movk_64(reg, u16((val >> (halfword * 16)) & 0xffff), halfword); + memcpy(dst + offset, &instr, 4); + offset += 4; + } + return offset; +} + +/*! + * Emits a GOAL-to-C stub and returns its size in bytes. + */ +inline int emit_arm64_c_stub(u8* dst, u64 target, u64 trampoline, bool arg3_is_pp) { + int offset = emit_arm64_mov64(dst, 8, target); + + if (arg3_is_pp) { + // pp goes from x20 to C argument x3 + u32 instr = 0xaa0003e0 | (20 << 16) | 3; + memcpy(dst + offset, &instr, 4); + offset += 4; + } + + offset += emit_arm64_mov64(dst + offset, 9, trampoline); + + u32 br = 0xd61f0000 | (9 << 5); // br x9 + memcpy(dst + offset, &br, 4); + offset += 4; + + return offset; +} + +#endif + +/*! + * Emits the host return stub used by GOAL `nothing`. + */ +inline int emit_return_stub(u8* dst) { +#ifdef __aarch64__ + const u32 instr = 0xd65f03c0; // ret + memcpy(dst, &instr, 4); + return 4; +#else + dst[0] = 0xc3; // ret + return 1; +#endif +} + +/*! + * Emits a GOAL function that returns zero. + */ +inline int emit_zero_stub(u8* dst) { +#ifdef __aarch64__ + // movz x0, #0 followed by ret + const u32 instrs[2] = {arm64::encode_movz_64(0, 0, 0), 0xd65f03c0}; + memcpy(dst, instrs, 8); + return 8; +#else + dst[0] = 0x31; // xor eax, eax + dst[1] = 0xc0; + dst[2] = 0xc3; // ret + return 3; +#endif +} diff --git a/game/kernel/common/klink.cpp b/game/kernel/common/klink.cpp index b8a96e57b5..70047c80a2 100644 --- a/game/kernel/common/klink.cpp +++ b/game/kernel/common/klink.cpp @@ -3,11 +3,42 @@ #include "common/goal_constants.h" #include "common/symbols.h" +#include "game/kernel/common/codegen.h" #include "game/kernel/common/fileio.h" #include "game/kernel/common/kscheme.h" #include "fmt/format.h" +void flush_icache_for_linked_object(const ObjectFileHeader* ofh) { + // link state 0 turns code_infos file offsets into heap addresses + for (int seg = 0; seg < N_SEG; seg++) { + if (ofh->code_infos[seg].offset && ofh->code_infos[seg].size) { + flush_icache_goal(ofh->code_infos[seg].offset, ofh->code_infos[seg].size); + } + } +} + +void flush_icache_for_linked_object_v2(Ptr code_start, uint32_t code_size) { + if (code_start.offset && code_size) { + flush_icache_goal(code_start.offset, code_size); + } +} + +uint32_t arm64_other_seg_mov32_link_v3(Ptr link, ObjectFileHeader* ofh, int current_seg) { + uint8_t target_seg = *link; + ASSERT(target_seg < ofh->segment_count); + + uint32_t* link_data = (link + 1).cast().c(); + uint32_t patch_loc = link_data[1] + ofh->code_infos[current_seg].offset; + + // discarded debug segments use a null GOAL pointer + uint32_t value = + ofh->code_infos[target_seg].offset ? (link_data[0] + ofh->code_infos[target_seg].offset) : 0; + + arm64_write_mov32(Ptr(patch_loc).c(), value); + return 1 + 2 * 4; +} + namespace { // turn on printf's for debugging linking issues. constexpr bool link_debug_printfs = false; diff --git a/game/kernel/common/klink.h b/game/kernel/common/klink.h index 60fb374e07..1ba77c42f3 100644 --- a/game/kernel/common/klink.h +++ b/game/kernel/common/klink.h @@ -40,6 +40,22 @@ struct SegmentInfoV5 { }; void klink_init_globals(); + +/*! + * Makes each linked code segment visible after relocation. + */ +void flush_icache_for_linked_object(const ObjectFileHeader* ofh); + +/*! + * Makes the contiguous code block from a v2 object visible to instruction fetch. + */ +void flush_icache_for_linked_object_v2(Ptr code_start, uint32_t code_size); + +/*! + * Patches a 32-bit cross-segment GOAL address into a movz/movk pair. + */ +uint32_t arm64_other_seg_mov32_link_v3(Ptr link, ObjectFileHeader* ofh, int current_seg); + /*! * Stores the state of the linker. Used for multi-threaded linking, so it can be suspended. */ diff --git a/game/kernel/common/kscheme.cpp b/game/kernel/common/kscheme.cpp index ae43de721d..8c18600092 100644 --- a/game/kernel/common/kscheme.cpp +++ b/game/kernel/common/kscheme.cpp @@ -84,7 +84,39 @@ u64 goal_malloc(u32 heap, u32 size, u32 flags, u32 name) { extern "C" { // defined in asm_funcs.asm -#ifdef __linux__ +#ifdef __aarch64__ +#ifdef __APPLE__ +uint64_t _call_goal_asm_arm64(u64 a0, + u64 a1, + u64 a2, + void* fptr, + void* st_ptr, + void* offset, + void* exec_offset) asm("_call_goal_asm_arm64"); +uint64_t _call_goal_on_stack_asm_arm64(u64 rsp, + u64 u0, + u64 u1, + void* fptr, + void* st_ptr, + void* offset, + void* exec_offset) asm("_call_goal_on_stack_asm_arm64"); +#else +uint64_t _call_goal_asm_arm64(u64 a0, + u64 a1, + u64 a2, + void* fptr, + void* st_ptr, + void* offset, + void* exec_offset); +uint64_t _call_goal_on_stack_asm_arm64(u64 rsp, + u64 u0, + u64 u1, + void* fptr, + void* st_ptr, + void* offset, + void* exec_offset); +#endif +#elif defined __linux__ uint64_t _call_goal_asm_systemv(u64 a0, u64 a1, u64 a2, void* fptr, void* st_ptr, void* offset); uint64_t _call_goal_on_stack_asm_systemv(u64 rsp, u64 u0, @@ -115,13 +147,20 @@ u64 call_goal(Ptr f, u64 a, u64 b, u64 c, u64 st, void* offset) { // auto st_ptr = (void*)((uint8_t*)(offset) + st); updated for the new compiler! void* st_ptr = (void*)st; - void* fptr = f.c(); -#ifdef __linux__ + [[maybe_unused]] void* fptr = f.c(); +#ifdef __aarch64__ + // use the executable mapping for the function and GOAL base + return _call_goal_asm_arm64(a, b, c, (void*)(g_ee_main_mem_exec + f.offset), st_ptr, offset, + g_ee_main_mem_exec); +#elif defined __linux__ return _call_goal_asm_systemv(a, b, c, fptr, st_ptr, offset); #elif defined __APPLE__ && defined __x86_64__ return _call_goal_asm_systemv(a, b, c, fptr, st_ptr, offset); #elif _WIN32 return _call_goal_asm_win32(a, b, c, fptr, st_ptr, offset); +#else + ASSERT_MSG(false, "call_goal: no GOAL call trampoline for this architecture or platform"); + return 0; #endif } @@ -131,13 +170,20 @@ u64 call_goal(Ptr f, u64 a, u64 b, u64 c, u64 st, void* offset) { u64 call_goal_on_stack(Ptr f, u64 rsp, u64 st, void* offset) { void* st_ptr = (void*)st; - void* fptr = f.c(); -#ifdef __linux__ + [[maybe_unused]] void* fptr = f.c(); +#ifdef __aarch64__ + return _call_goal_on_stack_asm_arm64(rsp, 0, 0, (void*)(g_ee_main_mem_exec + f.offset), st_ptr, + offset, g_ee_main_mem_exec); +#elif defined __linux__ return _call_goal_on_stack_asm_systemv(rsp, 0, 0, fptr, st_ptr, offset); #elif defined __APPLE__ && defined __x86_64__ return _call_goal_on_stack_asm_systemv(rsp, 0, 0, fptr, st_ptr, offset); #elif _WIN32 return _call_goal_on_stack_asm_win32(rsp, fptr, st_ptr, offset); +#else + ASSERT_MSG(false, + "call_goal_on_stack: no GOAL call trampoline for this architecture or platform"); + return 0; #endif } diff --git a/game/kernel/common/kscheme.h b/game/kernel/common/kscheme.h index b78d1e759a..f8864cdd27 100644 --- a/game/kernel/common/kscheme.h +++ b/game/kernel/common/kscheme.h @@ -33,6 +33,14 @@ u64 goal_malloc(u32 heap, u32 size, u32 flags, u32 name); u64 call_goal(Ptr f, u64 a, u64 b, u64 c, u64 st, void* offset); u64 call_goal_on_stack(Ptr f, u64 rsp, u64 st, void* offset); + +// align the host stack for each ABI +#ifdef __aarch64__ +constexpr u64 GOAL_STACK_TOP_OFFSET = 16; +#else +constexpr u64 GOAL_STACK_TOP_OFFSET = 8; +#endif + u64 call_goal_function(Ptr func); u64 print_structure(u32 s); u64 print_integer(u64 obj); @@ -46,4 +54,4 @@ u64 inspect_binteger(u64 obj); u64 inspect_float(u32 f); u64 inspect_structure(u32 obj); u64 inspect_vu_function(u32 obj); -u64 inspect_kheap(u32 obj); \ No newline at end of file +u64 inspect_kheap(u32 obj); diff --git a/game/kernel/jak1/kboot.cpp b/game/kernel/jak1/kboot.cpp index 859b7e13a6..5c5d328125 100644 --- a/game/kernel/jak1/kboot.cpp +++ b/game/kernel/jak1/kboot.cpp @@ -106,7 +106,7 @@ s32 goal_main(int argc, const char* const* argv) { * Main loop to dispatch the GOAL kernel. */ void KernelCheckAndDispatch() { - u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - 8; + u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - GOAL_STACK_TOP_OFFSET; while (MasterExit == RuntimeExitStatus::RUNNING) { // try to get a message from the listener, and process it if needed diff --git a/game/kernel/jak1/klink.cpp b/game/kernel/jak1/klink.cpp index fd71999592..2363d7bced 100644 --- a/game/kernel/jak1/klink.cpp +++ b/game/kernel/jak1/klink.cpp @@ -131,9 +131,10 @@ uint32_t typelink_v3(Ptr link, Ptr data) { /*! * Link symbols (both offsets and pointers) in "v3 equivalent" link data. + * If patch_mov32 is true, locations point to ARM64 movz/movk pairs instead of data words. * Returns a pointer to the link table data after the linking data for this symbol. */ -uint32_t symlink_v3(Ptr link, Ptr data) { +uint32_t symlink_v3(Ptr link, Ptr data, bool patch_mov32) { // get the symbol name uint32_t seek = 0; char sym_name[256]; @@ -161,12 +162,21 @@ uint32_t symlink_v3(Ptr link, Ptr data) { seek += 4; auto data_ptr = (data + offset).cast(); - if (*data_ptr == -1) { + // ARM64 movz/movk links use the same s32 sentinel values as direct data links. + s32 current = patch_mov32 ? s32(arm64_read_mov32((data + offset).cast().c())) : *data_ptr; + s32 value; + if (current == -1) { // a "-1" indicates that we should store the address. - *(data + offset).cast() = sym_addr; + value = sym_addr; } else { // otherwise store the offset to st. Eventually this should become an s16 instead. - *(data + offset).cast() = sym_offset; + value = sym_offset; + } + + if (patch_mov32) { + arm64_write_mov32((data + offset).cast().c(), u32(value)); + } else { + *data_ptr = value; } } @@ -265,7 +275,11 @@ uint32_t link_control::jak1_work_v3() { break; case LINK_SYMBOL_OFFSET: lp = lp + 1; - lp = lp + symlink_v3(lp, Ptr(ofh->code_infos[m_segment_process].offset)); + lp = lp + symlink_v3(lp, Ptr(ofh->code_infos[m_segment_process].offset), false); + break; + case LINK_ARM64_SYMBOL_MOV32: + lp = lp + 1; + lp = lp + symlink_v3(lp, Ptr(ofh->code_infos[m_segment_process].offset), true); break; case LINK_TYPE_PTR: lp = lp + 1; // seek past id @@ -279,6 +293,10 @@ uint32_t link_control::jak1_work_v3() { lp = lp + 1; lp = lp + cross_seg_dist_link_v3(lp, ofh, m_segment_process, 4); break; + case LINK_ARM64_OTHER_SEG_MOV32: + lp = lp + 1; + lp = lp + arm64_other_seg_mov32_link_v3(lp, ofh, m_segment_process); + break; case LINK_PTR: lp = lp + 1; lp = lp + ptr_link_v3(lp, ofh, m_segment_process); @@ -530,6 +548,9 @@ void link_control::jak1_finish(bool jump_from_c_to_goal) { if (ofh->object_file_version == 3) { // todo check function type of entry + // flush the executable view after relocation + flush_icache_for_linked_object(ofh); + // setup mips2c functions const auto& it = Mips2C::gMips2CLinkCallbacks[GameVersion::Jak1].find(m_object_name); if (it != Mips2C::gMips2CLinkCallbacks[GameVersion::Jak1].end()) { @@ -541,7 +562,7 @@ void link_control::jak1_finish(bool jump_from_c_to_goal) { // execute top level! if (m_entry.offset && (m_flags & LINK_FLAG_EXECUTE)) { if (jump_from_c_to_goal) { - u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - 8; + u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - GOAL_STACK_TOP_OFFSET; call_goal_on_stack(m_entry.cast(), goal_stack, s7.offset, g_ee_main_mem); } else { call_goal(m_entry.cast(), 0, 0, 0, s7.offset, g_ee_main_mem); @@ -554,6 +575,8 @@ void link_control::jak1_finish(bool jump_from_c_to_goal) { } } else { if (m_flags & LINK_FLAG_EXECUTE) { + // work_v2 can move the code block, so flush its final range + flush_icache_for_linked_object_v2(m_object_data, m_code_size); auto entry = m_entry; auto name = basename_goal(m_object_name); strcpy(Ptr(LINK_CONTROL_NAME_ADDR).c(), name); diff --git a/game/kernel/jak1/kscheme.cpp b/game/kernel/jak1/kscheme.cpp index c8287de5a5..1ec481cc8e 100644 --- a/game/kernel/jak1/kscheme.cpp +++ b/game/kernel/jak1/kscheme.cpp @@ -7,6 +7,7 @@ #include "common/symbols.h" #include "common/util/Timer.h" +#include "game/kernel/common/codegen.h" #include "game/kernel/common/fileio.h" #include "game/kernel/common/kdgo.h" #include "game/kernel/common/kdsnetm.h" @@ -294,10 +295,14 @@ Ptr make_function_from_c_systemv(void* func, bool arg3_is_pp) { auto trampoline_function_addr = _arg_call_arm64; #endif auto trampoline = (u8*)&trampoline_function_addr; - // TODO - x86 code still being emitted below - // movabs rax, target_function int offset = 0; +#ifdef __aarch64__ + u64 tramp; + memcpy(&tramp, trampoline, 8); + offset = emit_arm64_c_stub(mem.c(), (u64)func, tramp, arg3_is_pp); +#else + // movabs rax, target_function mem.c()[offset++] = 0x48; mem.c()[offset++] = 0xb8; for (int i = 0; i < 8; i++) { @@ -325,8 +330,9 @@ Ptr make_function_from_c_systemv(void* func, bool arg3_is_pp) { mem.c()[offset++] = 0xff; mem.c()[offset++] = 0xe0; // the asm function's ret will return to the caller of this (GOAL code) directlyz. +#endif - // CacheFlush(mem, 0x34); + flush_icache_goal(mem.offset, offset); return mem.cast(); } @@ -429,8 +435,13 @@ Ptr make_stack_arg_function_from_c_systemv(void* func) { #endif auto trampoline = (u8*)&trampoline_function_addr; - // movabs rax, target_function int offset = 0; +#ifdef __aarch64__ + u64 tramp; + memcpy(&tramp, trampoline, 8); + offset = emit_arm64_c_stub(mem.c(), (u64)func, tramp, false); +#else + // movabs rax, target_function mem.c()[offset++] = 0x48; mem.c()[offset++] = 0xb8; for (int i = 0; i < 8; i++) { @@ -450,8 +461,9 @@ Ptr make_stack_arg_function_from_c_systemv(void* func) { // jmp rax mem.c()[offset++] = 0xff; mem.c()[offset++] = 0xe0; +#endif - // CacheFlush(mem, 0x34); + flush_icache_goal(mem.offset, offset); return mem.cast(); } @@ -532,9 +544,9 @@ Ptr make_nothing_func() { auto mem = Ptr(alloc_heap_object(s7.offset + FIX_SYM_GLOBAL_HEAP, *(s7 + FIX_SYM_FUNCTION_TYPE), 0x14, UNKNOWN_PP)); - // a single x86-64 ret. - mem.c()[0] = 0xc3; - // CacheFlush(mem, 8); + const int written = emit_return_stub(mem.c()); + // flush the executable view + flush_icache_goal(mem.offset, written); return mem.cast(); } @@ -544,12 +556,8 @@ Ptr make_nothing_func() { Ptr make_zero_func() { auto mem = Ptr(alloc_heap_object(s7.offset + FIX_SYM_GLOBAL_HEAP, *(s7 + FIX_SYM_FUNCTION_TYPE), 0x14, UNKNOWN_PP)); - // xor eax, eax - mem.c()[0] = 0x31; - mem.c()[1] = 0xc0; - // ret - mem.c()[2] = 0xc3; - // CacheFlush(mem, 8); + const int written = emit_zero_stub(mem.c()); + flush_icache_goal(mem.offset, written); return mem.cast(); } diff --git a/game/kernel/jak2/kboot.cpp b/game/kernel/jak2/kboot.cpp index ce50468e94..ecab015f61 100644 --- a/game/kernel/jak2/kboot.cpp +++ b/game/kernel/jak2/kboot.cpp @@ -90,7 +90,7 @@ s32 goal_main(int argc, const char* const* argv) { void KernelDispatch(u32 dispatcher_func) { // place our stack at the end of EE memory - u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - 8; + u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - GOAL_STACK_TOP_OFFSET; // try to get a message from the listener, and process it if needed Ptr new_message = WaitForMessageAndAck(); diff --git a/game/kernel/jak2/klink.cpp b/game/kernel/jak2/klink.cpp index 5f00d652d4..edab0764dd 100644 --- a/game/kernel/jak2/klink.cpp +++ b/game/kernel/jak2/klink.cpp @@ -140,9 +140,10 @@ uint32_t typelink_v3(Ptr link, Ptr data) { } /*! * Link symbols (both offsets and pointers) in "v3 equivalent" link data. + * If patch_mov32 is true, locations point to ARM64 movz/movk pairs instead of data words. * Returns a pointer to the link table data after the linking data for this symbol. */ -uint32_t symlink_v3(Ptr link, Ptr data) { +uint32_t symlink_v3(Ptr link, Ptr data, bool patch_mov32) { // get the symbol name uint32_t seek = 0; char sym_name[256]; @@ -170,14 +171,23 @@ uint32_t symlink_v3(Ptr link, Ptr data) { seek += 4; auto data_ptr = (data + offset).cast(); - if (*data_ptr == -1) { + // ARM64 movz/movk links use the same s32 sentinel values as direct data links. + s32 current = patch_mov32 ? s32(arm64_read_mov32((data + offset).cast().c())) : *data_ptr; + s32 value; + if (current == -1) { // a "-1" indicates that we should store the address. - *(data + offset).cast() = sym_addr; - } else if (*(data_ptr.cast()) == LINK_SYM_NO_OFFSET_FLAG) { - *(data + offset).cast() = sym_offset - 1; + value = sym_addr; + } else if (u32(current) == LINK_SYM_NO_OFFSET_FLAG) { + value = sym_offset - 1; } else { // otherwise store the offset to st. - *(data + offset).cast() = sym_offset; + value = sym_offset; + } + + if (patch_mov32) { + arm64_write_mov32((data + offset).cast().c(), u32(value)); + } else { + *data_ptr = value; } } @@ -276,7 +286,11 @@ uint32_t link_control::jak2_work_v3() { break; case LINK_SYMBOL_OFFSET: lp = lp + 1; - lp = lp + symlink_v3(lp, Ptr(ofh->code_infos[m_segment_process].offset)); + lp = lp + symlink_v3(lp, Ptr(ofh->code_infos[m_segment_process].offset), false); + break; + case LINK_ARM64_SYMBOL_MOV32: + lp = lp + 1; + lp = lp + symlink_v3(lp, Ptr(ofh->code_infos[m_segment_process].offset), true); break; case LINK_TYPE_PTR: lp = lp + 1; // seek past id @@ -290,6 +304,10 @@ uint32_t link_control::jak2_work_v3() { lp = lp + 1; lp = lp + cross_seg_dist_link_v3(lp, ofh, m_segment_process, 4); break; + case LINK_ARM64_OTHER_SEG_MOV32: + lp = lp + 1; + lp = lp + arm64_other_seg_mov32_link_v3(lp, ofh, m_segment_process); + break; case LINK_PTR: lp = lp + 1; lp = lp + ptr_link_v3(lp, ofh, m_segment_process); @@ -548,6 +566,9 @@ void link_control::jak2_finish(bool jump_from_c_to_goal) { if (ofh->object_file_version == 3) { // todo check function type of entry + // flush the executable view after relocation + flush_icache_for_linked_object(ofh); + // setup mips2c functions const auto& it = Mips2C::gMips2CLinkCallbacks[GameVersion::Jak2].find(m_object_name); if (it != Mips2C::gMips2CLinkCallbacks[GameVersion::Jak2].end()) { @@ -559,7 +580,7 @@ void link_control::jak2_finish(bool jump_from_c_to_goal) { // execute top level! if (m_entry.offset && (m_flags & LINK_FLAG_EXECUTE)) { if (jump_from_c_to_goal) { - u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - 8; + u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - GOAL_STACK_TOP_OFFSET; call_goal_on_stack(m_entry.cast(), goal_stack, s7.offset, g_ee_main_mem); } else { call_goal(m_entry.cast(), 0, 0, 0, s7.offset, g_ee_main_mem); @@ -572,6 +593,8 @@ void link_control::jak2_finish(bool jump_from_c_to_goal) { } } else { if (m_flags & LINK_FLAG_EXECUTE) { + // work_v2 can move the code block, so flush its final range + flush_icache_for_linked_object_v2(m_object_data, m_code_size); auto entry = m_entry; auto name = basename_goal(m_object_name); strcpy(Ptr(LINK_CONTROL_NAME_ADDR).c(), name); diff --git a/game/kernel/jak2/kscheme.cpp b/game/kernel/jak2/kscheme.cpp index 37946a4120..1eecaa4302 100644 --- a/game/kernel/jak2/kscheme.cpp +++ b/game/kernel/jak2/kscheme.cpp @@ -12,6 +12,7 @@ #include "common/log/log.h" #include "common/symbols.h" +#include "game/kernel/common/codegen.h" #include "game/kernel/common/fileio.h" #include "game/kernel/common/kdsnetm.h" #include "game/kernel/common/klink.h" @@ -314,10 +315,14 @@ Ptr make_function_from_c_systemv(void* func, bool arg3_is_pp) { auto trampoline_function_addr = _arg_call_arm64; #endif auto trampoline = (u8*)&trampoline_function_addr; - // TODO - x86 code still being emitted below - // movabs rax, target_function int offset = 0; +#ifdef __aarch64__ + u64 tramp; + memcpy(&tramp, trampoline, 8); + offset = emit_arm64_c_stub(mem.c(), (u64)func, tramp, arg3_is_pp); +#else + // movabs rax, target_function mem.c()[offset++] = 0x48; mem.c()[offset++] = 0xb8; for (int i = 0; i < 8; i++) { @@ -345,8 +350,9 @@ Ptr make_function_from_c_systemv(void* func, bool arg3_is_pp) { mem.c()[offset++] = 0xff; mem.c()[offset++] = 0xe0; // the asm function's ret will return to the caller of this (GOAL code) directlyz. +#endif - // CacheFlush(mem, 0x34); + flush_icache_goal(mem.offset, offset); return mem.cast(); } @@ -427,8 +433,13 @@ Ptr make_stack_arg_function_from_c_systemv(void* func) { #endif auto trampoline = (u8*)&trampoline_function_addr; - // movabs rax, target_function int offset = 0; +#ifdef __aarch64__ + u64 tramp; + memcpy(&tramp, trampoline, 8); + offset = emit_arm64_c_stub(mem.c(), (u64)func, tramp, false); +#else + // movabs rax, target_function mem.c()[offset++] = 0x48; mem.c()[offset++] = 0xb8; for (int i = 0; i < 8; i++) { @@ -448,8 +459,9 @@ Ptr make_stack_arg_function_from_c_systemv(void* func) { // jmp rax mem.c()[offset++] = 0xff; mem.c()[offset++] = 0xe0; +#endif - // CacheFlush(mem, 0x34); + flush_icache_goal(mem.offset, offset); return mem.cast(); } @@ -530,9 +542,9 @@ Ptr make_nothing_func() { auto mem = Ptr(alloc_heap_object(s7.offset + FIX_SYM_GLOBAL_HEAP, u32_in_fixed_sym(FIX_SYM_FUNCTION_TYPE), 0x14, UNKNOWN_PP)); - // a single x86-64 ret. - mem.c()[0] = 0xc3; - // CacheFlush(mem, 8); + const int written = emit_return_stub(mem.c()); + // flush the executable view + flush_icache_goal(mem.offset, written); return mem.cast(); } @@ -542,12 +554,8 @@ Ptr make_nothing_func() { Ptr make_zero_func() { auto mem = Ptr(alloc_heap_object(s7.offset + FIX_SYM_GLOBAL_HEAP, u32_in_fixed_sym(FIX_SYM_FUNCTION_TYPE), 0x14, UNKNOWN_PP)); - // xor eax, eax - mem.c()[0] = 0x31; - mem.c()[1] = 0xc0; - // ret - mem.c()[2] = 0xc3; - // CacheFlush(mem, 8); + const int written = emit_zero_stub(mem.c()); + flush_icache_goal(mem.offset, written); return mem.cast(); } diff --git a/game/kernel/jak3/kboot.cpp b/game/kernel/jak3/kboot.cpp index bcc8acfcaa..dbc08958f0 100644 --- a/game/kernel/jak3/kboot.cpp +++ b/game/kernel/jak3/kboot.cpp @@ -94,7 +94,7 @@ s32 goal_main(int argc, const char* const* argv) { void KernelDispatch(u32 dispatcher_func) { // place our stack at the end of EE memory - u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - 8; + u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - GOAL_STACK_TOP_OFFSET; // try to get a message from the listener, and process it if needed Ptr new_message = WaitForMessageAndAck(); diff --git a/game/kernel/jak3/klink.cpp b/game/kernel/jak3/klink.cpp index 4a703abbfa..5dfcea9fe6 100644 --- a/game/kernel/jak3/klink.cpp +++ b/game/kernel/jak3/klink.cpp @@ -7,6 +7,7 @@ #include "game/kernel/common/fileio.h" #include "game/kernel/common/klink.h" #include "game/kernel/common/kprint.h" +#include "game/kernel/common/kscheme.h" #include "game/kernel/common/memory_layout.h" #include "game/kernel/jak3/kmalloc.h" #include "game/kernel/jak3/kscheme.h" @@ -553,9 +554,10 @@ uint32_t typelink_v3(Ptr link, Ptr data) { } /*! * Link symbols (both offsets and pointers) in "v3 equivalent" link data. + * If patch_mov32 is true, locations point to ARM64 movz/movk pairs instead of data words. * Returns a pointer to the link table data after the linking data for this symbol. */ -uint32_t symlink_v3(Ptr link, Ptr data) { +uint32_t symlink_v3(Ptr link, Ptr data, bool patch_mov32) { // get the symbol name uint32_t seek = 0; char sym_name[256]; @@ -583,14 +585,23 @@ uint32_t symlink_v3(Ptr link, Ptr data) { seek += 4; auto data_ptr = (data + offset).cast(); - if (*data_ptr == -1) { + // ARM64 movz/movk links use the same s32 sentinel values as direct data links. + s32 current = patch_mov32 ? s32(arm64_read_mov32((data + offset).cast().c())) : *data_ptr; + s32 value; + if (current == -1) { // a "-1" indicates that we should store the address. - *(data + offset).cast() = sym_addr; - } else if (*(data_ptr.cast()) == LINK_SYM_NO_OFFSET_FLAG) { - *(data + offset).cast() = sym_offset - 1; + value = sym_addr; + } else if (u32(current) == LINK_SYM_NO_OFFSET_FLAG) { + value = sym_offset - 1; } else { // otherwise store the offset to st. - *(data + offset).cast() = sym_offset; + value = sym_offset; + } + + if (patch_mov32) { + arm64_write_mov32((data + offset).cast().c(), u32(value)); + } else { + *data_ptr = value; } } @@ -687,7 +698,11 @@ uint32_t link_control::jak3_work_opengoal() { break; case LINK_SYMBOL_OFFSET: lp = lp + 1; - lp = lp + symlink_v3(lp, Ptr(ofh->code_infos[m_segment_process].offset)); + lp = lp + symlink_v3(lp, Ptr(ofh->code_infos[m_segment_process].offset), false); + break; + case LINK_ARM64_SYMBOL_MOV32: + lp = lp + 1; + lp = lp + symlink_v3(lp, Ptr(ofh->code_infos[m_segment_process].offset), true); break; case LINK_TYPE_PTR: lp = lp + 1; // seek past id @@ -701,6 +716,10 @@ uint32_t link_control::jak3_work_opengoal() { lp = lp + 1; lp = lp + cross_seg_dist_link_v3(lp, ofh, m_segment_process, 4); break; + case LINK_ARM64_OTHER_SEG_MOV32: + lp = lp + 1; + lp = lp + arm64_other_seg_mov32_link_v3(lp, ofh, m_segment_process); + break; case LINK_PTR: lp = lp + 1; lp = lp + ptr_link_v3(lp, ofh, m_segment_process); @@ -743,6 +762,9 @@ void link_control::jak3_finish(bool jump_from_c_to_goal) { // printf("finish %s\n", m_object_name); if (m_opengoal) { + // flush the executable view after relocation + flush_icache_for_linked_object(m_link_block_ptr.cast().c()); + // setup mips2c functions const auto& it = Mips2C::gMips2CLinkCallbacks[GameVersion::Jak3].find(m_object_name); if (it != Mips2C::gMips2CLinkCallbacks[GameVersion::Jak3].end()) { @@ -754,7 +776,7 @@ void link_control::jak3_finish(bool jump_from_c_to_goal) { // execute top level! if (m_entry.offset && (m_flags & LINK_FLAG_EXECUTE)) { if (jump_from_c_to_goal) { - u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - 8; + u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - GOAL_STACK_TOP_OFFSET; call_goal_on_stack(m_entry.cast(), goal_stack, s7.offset, g_ee_main_mem); } else { call_goal(m_entry.cast(), 0, 0, 0, s7.offset, g_ee_main_mem); @@ -767,6 +789,13 @@ void link_control::jak3_finish(bool jump_from_c_to_goal) { } } else { if (m_flags & LINK_FLAG_EXECUTE) { + // version 5 keeps the final code range in m_link_segments_table + if (m_version == 5 && m_link_segments_table) { + flush_icache_for_linked_object_v2(Ptr(m_link_segments_table[0].data), + m_link_segments_table[0].size); + } else { + flush_icache_for_linked_object_v2(m_object_data, m_code_size); + } auto entry = m_entry; auto name = basename_goal(m_object_name); strcpy(Ptr(LINK_CONTROL_NAME_ADDR).c(), name); diff --git a/game/kernel/jak3/kscheme.cpp b/game/kernel/jak3/kscheme.cpp index 7a9d7c1efa..0c9e9e7efd 100644 --- a/game/kernel/jak3/kscheme.cpp +++ b/game/kernel/jak3/kscheme.cpp @@ -8,6 +8,7 @@ #include "common/symbols.h" #include "game/kernel/common/Symbol4.h" +#include "game/kernel/common/codegen.h" #include "game/kernel/common/fileio.h" #include "game/kernel/common/kdsnetm.h" #include "game/kernel/common/klink.h" @@ -301,10 +302,14 @@ Ptr make_function_from_c_systemv(void* func, bool arg3_is_pp) { auto trampoline_function_addr = _arg_call_arm64; #endif auto trampoline = (u8*)&trampoline_function_addr; - // TODO - x86 code still being emitted below - // movabs rax, target_function int offset = 0; +#ifdef __aarch64__ + u64 tramp; + memcpy(&tramp, trampoline, 8); + offset = emit_arm64_c_stub(mem.c(), (u64)func, tramp, arg3_is_pp); +#else + // movabs rax, target_function mem.c()[offset++] = 0x48; mem.c()[offset++] = 0xb8; for (int i = 0; i < 8; i++) { @@ -332,8 +337,9 @@ Ptr make_function_from_c_systemv(void* func, bool arg3_is_pp) { mem.c()[offset++] = 0xff; mem.c()[offset++] = 0xe0; // the asm function's ret will return to the caller of this (GOAL code) directlyz. +#endif - // CacheFlush(mem, 0x34); + flush_icache_goal(mem.offset, offset); return mem.cast(); } @@ -414,8 +420,13 @@ Ptr make_stack_arg_function_from_c_systemv(void* func) { #endif auto trampoline = (u8*)&trampoline_function_addr; - // movabs rax, target_function int offset = 0; +#ifdef __aarch64__ + u64 tramp; + memcpy(&tramp, trampoline, 8); + offset = emit_arm64_c_stub(mem.c(), (u64)func, tramp, false); +#else + // movabs rax, target_function mem.c()[offset++] = 0x48; mem.c()[offset++] = 0xb8; for (int i = 0; i < 8; i++) { @@ -435,8 +446,9 @@ Ptr make_stack_arg_function_from_c_systemv(void* func) { // jmp rax mem.c()[offset++] = 0xff; mem.c()[offset++] = 0xe0; +#endif - // CacheFlush(mem, 0x34); + flush_icache_goal(mem.offset, offset); return mem.cast(); } @@ -517,9 +529,9 @@ Ptr make_nothing_func() { auto mem = Ptr(alloc_heap_object(s7.offset + FIX_SYM_GLOBAL_HEAP, u32_in_fixed_sym(FIX_SYM_FUNCTION_TYPE), 0x14, UNKNOWN_PP)); - // a single x86-64 ret. - mem.c()[0] = 0xc3; - // CacheFlush(mem, 8); + const int written = emit_return_stub(mem.c()); + // flush the executable view + flush_icache_goal(mem.offset, written); return mem.cast(); } @@ -529,12 +541,8 @@ Ptr make_nothing_func() { Ptr make_zero_func() { auto mem = Ptr(alloc_heap_object(s7.offset + FIX_SYM_GLOBAL_HEAP, u32_in_fixed_sym(FIX_SYM_FUNCTION_TYPE), 0x14, UNKNOWN_PP)); - // xor eax, eax - mem.c()[0] = 0x31; - mem.c()[1] = 0xc0; - // ret - mem.c()[2] = 0xc3; - // CacheFlush(mem, 8); + const int written = emit_zero_stub(mem.c()); + flush_icache_goal(mem.offset, written); return mem.cast(); } diff --git a/game/kernel/jakx/kboot.cpp b/game/kernel/jakx/kboot.cpp index bf4f74f1d9..3e4c501901 100644 --- a/game/kernel/jakx/kboot.cpp +++ b/game/kernel/jakx/kboot.cpp @@ -110,7 +110,7 @@ s32 goal_main(int argc, const char* const* argv) { void KernelDispatch(u32 dispatcher_func) { // place our stack at the end of EE memory - u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - 8; + u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - GOAL_STACK_TOP_OFFSET; // try to get a message from the listener, and process it if needed Ptr new_message = WaitForMessageAndAck(); diff --git a/game/kernel/jakx/klink.cpp b/game/kernel/jakx/klink.cpp index 2490a0bb5a..91cea7798e 100644 --- a/game/kernel/jakx/klink.cpp +++ b/game/kernel/jakx/klink.cpp @@ -7,6 +7,7 @@ #include "game/kernel/common/fileio.h" #include "game/kernel/common/klink.h" #include "game/kernel/common/kprint.h" +#include "game/kernel/common/kscheme.h" #include "game/kernel/common/memory_layout.h" #include "game/kernel/jakx/kmalloc.h" #include "game/kernel/jakx/kscheme.h" @@ -553,9 +554,10 @@ uint32_t typelink_v3(Ptr link, Ptr data) { } /*! * Link symbols (both offsets and pointers) in "v3 equivalent" link data. + * If patch_mov32 is true, locations point to ARM64 movz/movk pairs instead of data words. * Returns a pointer to the link table data after the linking data for this symbol. */ -uint32_t symlink_v3(Ptr link, Ptr data) { +uint32_t symlink_v3(Ptr link, Ptr data, bool patch_mov32) { // get the symbol name uint32_t seek = 0; char sym_name[256]; @@ -583,14 +585,23 @@ uint32_t symlink_v3(Ptr link, Ptr data) { seek += 4; auto data_ptr = (data + offset).cast(); - if (*data_ptr == -1) { + // ARM64 movz/movk links use the same s32 sentinel values as direct data links. + s32 current = patch_mov32 ? s32(arm64_read_mov32((data + offset).cast().c())) : *data_ptr; + s32 value; + if (current == -1) { // a "-1" indicates that we should store the address. - *(data + offset).cast() = sym_addr; - } else if (*(data_ptr.cast()) == LINK_SYM_NO_OFFSET_FLAG) { - *(data + offset).cast() = sym_offset - 1; + value = sym_addr; + } else if (u32(current) == LINK_SYM_NO_OFFSET_FLAG) { + value = sym_offset - 1; } else { // otherwise store the offset to st. - *(data + offset).cast() = sym_offset; + value = sym_offset; + } + + if (patch_mov32) { + arm64_write_mov32((data + offset).cast().c(), u32(value)); + } else { + *data_ptr = value; } } @@ -687,7 +698,11 @@ uint32_t link_control::jakx_work_opengoal() { break; case LINK_SYMBOL_OFFSET: lp = lp + 1; - lp = lp + symlink_v3(lp, Ptr(ofh->code_infos[m_segment_process].offset)); + lp = lp + symlink_v3(lp, Ptr(ofh->code_infos[m_segment_process].offset), false); + break; + case LINK_ARM64_SYMBOL_MOV32: + lp = lp + 1; + lp = lp + symlink_v3(lp, Ptr(ofh->code_infos[m_segment_process].offset), true); break; case LINK_TYPE_PTR: lp = lp + 1; // seek past id @@ -701,6 +716,10 @@ uint32_t link_control::jakx_work_opengoal() { lp = lp + 1; lp = lp + cross_seg_dist_link_v3(lp, ofh, m_segment_process, 4); break; + case LINK_ARM64_OTHER_SEG_MOV32: + lp = lp + 1; + lp = lp + arm64_other_seg_mov32_link_v3(lp, ofh, m_segment_process); + break; case LINK_PTR: lp = lp + 1; lp = lp + ptr_link_v3(lp, ofh, m_segment_process); @@ -743,6 +762,9 @@ void link_control::jakx_finish(bool jump_from_c_to_goal) { // printf("finish %s\n", m_object_name); if (m_opengoal) { + // flush the executable view after relocation + flush_icache_for_linked_object(m_link_block_ptr.cast().c()); + // setup mips2c functions const auto& it = Mips2C::gMips2CLinkCallbacks[GameVersion::JakX].find(m_object_name); if (it != Mips2C::gMips2CLinkCallbacks[GameVersion::JakX].end()) { @@ -754,7 +776,7 @@ void link_control::jakx_finish(bool jump_from_c_to_goal) { // execute top level! if (m_entry.offset && (m_flags & LINK_FLAG_EXECUTE)) { if (jump_from_c_to_goal) { - u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - 8; + u64 goal_stack = u64(g_ee_main_mem) + EE_MAIN_MEM_SIZE - GOAL_STACK_TOP_OFFSET; call_goal_on_stack(m_entry.cast(), goal_stack, s7.offset, g_ee_main_mem); } else { call_goal(m_entry.cast(), 0, 0, 0, s7.offset, g_ee_main_mem); @@ -767,6 +789,13 @@ void link_control::jakx_finish(bool jump_from_c_to_goal) { } } else { if (m_flags & LINK_FLAG_EXECUTE) { + // version 5 keeps the final code range in m_link_segments_table + if (m_version == 5 && m_link_segments_table) { + flush_icache_for_linked_object_v2(Ptr(m_link_segments_table[0].data), + m_link_segments_table[0].size); + } else { + flush_icache_for_linked_object_v2(m_object_data, m_code_size); + } auto entry = m_entry; auto name = basename_goal(m_object_name); strcpy(Ptr(LINK_CONTROL_NAME_ADDR).c(), name); diff --git a/game/kernel/jakx/kscheme.cpp b/game/kernel/jakx/kscheme.cpp index b05c237097..0db55b458c 100644 --- a/game/kernel/jakx/kscheme.cpp +++ b/game/kernel/jakx/kscheme.cpp @@ -8,6 +8,7 @@ #include "common/symbols.h" #include "game/kernel/common/Symbol4.h" +#include "game/kernel/common/codegen.h" #include "game/kernel/common/fileio.h" #include "game/kernel/common/kdsnetm.h" #include "game/kernel/common/klink.h" @@ -301,10 +302,14 @@ Ptr make_function_from_c_systemv(void* func, bool arg3_is_pp) { auto trampoline_function_addr = _arg_call_arm64; #endif auto trampoline = (u8*)&trampoline_function_addr; - // TODO - x86 code still being emitted below - // movabs rax, target_function int offset = 0; +#ifdef __aarch64__ + u64 tramp; + memcpy(&tramp, trampoline, 8); + offset = emit_arm64_c_stub(mem.c(), (u64)func, tramp, arg3_is_pp); +#else + // movabs rax, target_function mem.c()[offset++] = 0x48; mem.c()[offset++] = 0xb8; for (int i = 0; i < 8; i++) { @@ -332,8 +337,9 @@ Ptr make_function_from_c_systemv(void* func, bool arg3_is_pp) { mem.c()[offset++] = 0xff; mem.c()[offset++] = 0xe0; // the asm function's ret will return to the caller of this (GOAL code) directlyz. +#endif - // CacheFlush(mem, 0x34); + flush_icache_goal(mem.offset, offset); return mem.cast(); } @@ -414,8 +420,13 @@ Ptr make_stack_arg_function_from_c_systemv(void* func) { #endif auto trampoline = (u8*)&trampoline_function_addr; - // movabs rax, target_function int offset = 0; +#ifdef __aarch64__ + u64 tramp; + memcpy(&tramp, trampoline, 8); + offset = emit_arm64_c_stub(mem.c(), (u64)func, tramp, false); +#else + // movabs rax, target_function mem.c()[offset++] = 0x48; mem.c()[offset++] = 0xb8; for (int i = 0; i < 8; i++) { @@ -435,8 +446,9 @@ Ptr make_stack_arg_function_from_c_systemv(void* func) { // jmp rax mem.c()[offset++] = 0xff; mem.c()[offset++] = 0xe0; +#endif - // CacheFlush(mem, 0x34); + flush_icache_goal(mem.offset, offset); return mem.cast(); } @@ -517,9 +529,9 @@ Ptr make_nothing_func() { auto mem = Ptr(alloc_heap_object(s7.offset + FIX_SYM_GLOBAL_HEAP, u32_in_fixed_sym(FIX_SYM_FUNCTION_TYPE), 0x14, UNKNOWN_PP)); - // a single x86-64 ret. - mem.c()[0] = 0xc3; - // CacheFlush(mem, 8); + const int written = emit_return_stub(mem.c()); + // flush the executable view + flush_icache_goal(mem.offset, written); return mem.cast(); } @@ -529,12 +541,8 @@ Ptr make_nothing_func() { Ptr make_zero_func() { auto mem = Ptr(alloc_heap_object(s7.offset + FIX_SYM_GLOBAL_HEAP, u32_in_fixed_sym(FIX_SYM_FUNCTION_TYPE), 0x14, UNKNOWN_PP)); - // xor eax, eax - mem.c()[0] = 0x31; - mem.c()[1] = 0xc0; - // ret - mem.c()[2] = 0xc3; - // CacheFlush(mem, 8); + const int written = emit_zero_stub(mem.c()); + flush_icache_goal(mem.offset, written); return mem.cast(); } diff --git a/game/mips2c/mips2c_private.h b/game/mips2c/mips2c_private.h index f3124154e0..6ad75f7236 100644 --- a/game/mips2c/mips2c_private.h +++ b/game/mips2c/mips2c_private.h @@ -18,9 +18,28 @@ // This is only useful for extern u8* g_ee_main_mem; +extern u8* g_ee_main_mem_exec; extern "C" { -#ifdef __linux__ +#ifdef __aarch64__ +#ifdef __APPLE__ +u64 _call_goal8_asm_arm64(void* func, + u64* arg_array, + u64 zero, + u64 pp, + u64 st, + void* off, + void* exec_off) asm("_call_goal8_asm_arm64"); +#else +u64 _call_goal8_asm_arm64(void* func, + u64* arg_array, + u64 zero, + u64 pp, + u64 st, + void* off, + void* exec_off); +#endif +#elif defined __linux__ u64 _call_goal8_asm_systemv(void* func, u64* arg_array, u64 zero, u64 pp, u64 st, void* off); #elif defined __APPLE__ && defined __x86_64__ u64 _call_goal8_asm_systemv(void* func, u64* arg_array, u64 zero, u64 pp, u64 st, void* off) asm( @@ -353,7 +372,11 @@ struct ExecutionContext { u64 args[8] = {gprs[a0].du64[0], gprs[a1].du64[0], gprs[a2].du64[0], gprs[a3].du64[0], gprs[t0].du64[0], gprs[t1].du64[0], gprs[t2].du64[0], gprs[t3].du64[0]}; ASSERT(addr); -#ifdef __linux__ +#ifdef __aarch64__ + // use the executable mapping for the function and GOAL base + gprs[v0].du64[0] = _call_goal8_asm_arm64(g_ee_main_mem_exec + addr, args, 0, gprs[s6].du64[0], + gprs[s7].du64[0], g_ee_main_mem, g_ee_main_mem_exec); +#elif defined __linux__ gprs[v0].du64[0] = _call_goal8_asm_systemv(g_ee_main_mem + addr, args, 0, gprs[s6].du64[0], gprs[s7].du64[0], g_ee_main_mem); #elif defined __APPLE__ && defined __x86_64__ @@ -362,6 +385,9 @@ struct ExecutionContext { #elif _WIN32 gprs[v0].du64[0] = _call_goal8_asm_win32(g_ee_main_mem + addr, args, 0, gprs[s6].du64[0], gprs[s7].du64[0], g_ee_main_mem); +#else + ASSERT_MSG(false, "jalr: no GOAL call trampoline for this architecture or platform"); + return; #endif } diff --git a/game/mips2c/mips2c_table.cpp b/game/mips2c/mips2c_table.cpp index 018b0d3316..02841b22da 100644 --- a/game/mips2c/mips2c_table.cpp +++ b/game/mips2c/mips2c_table.cpp @@ -3,6 +3,7 @@ #include "common/log/log.h" #include "common/symbols.h" +#include "game/kernel/common/codegen.h" #include "game/kernel/common/kmalloc.h" #include "game/kernel/common/kscheme.h" #include "game/kernel/jak1/kscheme.h" @@ -11,7 +12,13 @@ #include "game/runtime.h" extern "C" { -#ifdef __linux__ +#ifdef __aarch64__ +#ifdef __APPLE__ +void _mips2c_call_arm64() asm("_mips2c_call_arm64"); +#else +void _mips2c_call_arm64(); +#endif +#elif defined __linux__ void _mips2c_call_systemv(); #elif defined __APPLE__ && defined __x86_64__ void _mips2c_call_systemv() asm("_mips2c_call_systemv"); @@ -692,10 +699,20 @@ void LinkedFunctionTable::reg(const std::string& name, u64 (*exec)(void*), u32 s it.first->second.goal_trampoline = jump_to_asm; u8* ptr = jump_to_asm.c(); + int written; +#ifdef __aarch64__ + // x9 is the target, x10 is the stack size, x17 jumps to the trampoline + written = emit_arm64_mov64(ptr, 9, (u64)exec); + written += emit_arm64_mov64(ptr + written, 10, (u64)stack_size); + written += emit_arm64_mov64(ptr + written, 17, (u64)_mips2c_call_arm64); + { + u32 br = 0xd61f0000 | (17 << 5); // br x17 + memcpy(ptr + written, &br, 4); + written += 4; + } +#else { - // linux - // push the function u64 addr = (u64)exec; *ptr = 0x48; @@ -725,6 +742,9 @@ void LinkedFunctionTable::reg(const std::string& name, u64 (*exec)(void*), u32 s addr = (u64)_mips2c_call_systemv; #elif _WIN32 addr = (u64)_mips2c_call_windows; +#else + ASSERT_MSG(false, "mips2c trampoline: no implementation for this architecture or platform"); + return; #endif *ptr = 0x48; @@ -738,7 +758,15 @@ void LinkedFunctionTable::reg(const std::string& name, u64 (*exec)(void*), u32 s *ptr = 0xff; ptr++; *ptr = 0xe0; + ptr++; + written = int(ptr - jump_to_asm.c()); } +#endif + + ASSERT_MSG(written <= 0x40, "mips2c stub exceeds its allocation"); + + // flush the executable view + flush_icache_goal(jump_to_asm.offset, written); } u32 LinkedFunctionTable::get(const std::string& name) { diff --git a/game/runtime.cpp b/game/runtime.cpp index 4f7cc5e181..b740ab3916 100644 --- a/game/runtime.cpp +++ b/game/runtime.cpp @@ -8,6 +8,10 @@ #include #include +#if defined(__APPLE__) && defined(__aarch64__) +#include +#include +#endif #elif _WIN32 #include @@ -87,6 +91,7 @@ #include "system/SystemThread.h" u8* g_ee_main_mem = nullptr; +u8* g_ee_main_mem_exec = nullptr; std::thread::id g_main_thread_id = std::thread::id(); GameVersion g_game_version = GameVersion::Jak1; BackgroundWorker g_background_worker; @@ -149,13 +154,7 @@ void deci2_runner(SystemThreadInterface& iface) { void ee_runner(SystemThreadInterface& iface) { prof().root_event(); // Allocate Main RAM. Must have execute enabled. - // TODO Apple Silicon - You cannot make a page be RWX, - // or more specifically it can't be both writable and executable at the same time - // - // https://github.com/zherczeg/sljit/issues/99 - // - // The solution to this is to flip-flop between permissions, or perhaps have two threads - // one that has writing permission, and another with executable permission + // Apple Silicon uses separate writable and executable mappings for EE memory. if (EE_MEM_LOW_MAP) { g_ee_main_mem = (u8*)mmap((void*)0x10000000, EE_MAIN_MEM_SIZE, PROT_EXEC | PROT_READ | PROT_WRITE, @@ -166,9 +165,15 @@ void ee_runner(SystemThreadInterface& iface) { MAP_ANONYMOUS | MAP_32BIT | MAP_PRIVATE | MAP_POPULATE, 0, 0); #endif } else { +#if defined(__APPLE__) && defined(__aarch64__) + // mach_vm_remap needs an anonymous writable mapping + g_ee_main_mem = (u8*)mmap((void*)EE_MAIN_MEM_MAP, EE_MAIN_MEM_SIZE, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); +#else g_ee_main_mem = (u8*)mmap((void*)EE_MAIN_MEM_MAP, EE_MAIN_MEM_SIZE, PROT_EXEC | PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); +#endif } if (g_ee_main_mem == (u8*)(-1)) { @@ -177,6 +182,35 @@ void ee_runner(SystemThreadInterface& iface) { return; } + g_ee_main_mem_exec = g_ee_main_mem; + +#if defined(__APPLE__) && defined(__aarch64__) + // map the same pages again with read and execute access + { + mach_vm_address_t exec_addr = 0; + vm_prot_t cur_prot = 0, max_prot = 0; + kern_return_t kr = mach_vm_remap( + mach_task_self(), &exec_addr, EE_MAIN_MEM_SIZE, 0, VM_FLAGS_ANYWHERE, mach_task_self(), + (mach_vm_address_t)g_ee_main_mem, false, &cur_prot, &max_prot, VM_INHERIT_NONE); + if (kr != KERN_SUCCESS) { + lg::error("Cannot map the executable view of main memory: mach_vm_remap returned {}", + (int)kr); + iface.initialization_complete(); + return; + } + kr = mach_vm_protect(mach_task_self(), exec_addr, EE_MAIN_MEM_SIZE, false, + VM_PROT_READ | VM_PROT_EXECUTE); + if (kr != KERN_SUCCESS) { + lg::error("Cannot make main memory executable: mach_vm_protect returned {}", (int)kr); + iface.initialization_complete(); + return; + } + g_ee_main_mem_exec = (u8*)exec_addr; + lg::debug("Main memory executable view at 0x{:016x} (delta 0x{:x})", (u64)g_ee_main_mem_exec, + (u64)(g_ee_main_mem_exec - g_ee_main_mem)); + } +#endif + lg::debug("Main memory mapped at 0x{:016x}", (u64)(g_ee_main_mem)); lg::debug("Main memory size 0x{:x} bytes ({:.3f} MB)", EE_MAIN_MEM_SIZE, (double)EE_MAIN_MEM_SIZE / (1 << 20)); diff --git a/game/runtime.h b/game/runtime.h index 89cb4b398a..3762273f40 100644 --- a/game/runtime.h +++ b/game/runtime.h @@ -15,6 +15,8 @@ #include "system/background_worker.h" extern u8* g_ee_main_mem; +// executable view of g_ee_main_mem for GOAL calls +extern u8* g_ee_main_mem_exec; extern GameVersion g_game_version; extern BackgroundWorker g_background_worker; extern int g_server_port; diff --git a/game/system/IOP_Kernel.cpp b/game/system/IOP_Kernel.cpp index 38acbe6abc..04e6f9538b 100644 --- a/game/system/IOP_Kernel.cpp +++ b/game/system/IOP_Kernel.cpp @@ -46,7 +46,7 @@ IOP_Kernel::IOP_Kernel() { u32 IOP_Kernel::GetSystemTimeLow() { auto delta_time = time_point_cast(steady_clock::now()) - m_start_time; - return delta_time.count() * 36.864; + return static_cast(static_cast(delta_time.count() * 36.864)); } /*! diff --git a/game/system/IOP_Kernel.h b/game/system/IOP_Kernel.h index 065c7caa53..cf3e9fde36 100644 --- a/game/system/IOP_Kernel.h +++ b/game/system/IOP_Kernel.h @@ -21,7 +21,7 @@ class IOP_Kernel; namespace iop { struct sceSifQueueData; -} +} // namespace iop using time_stamp = std::chrono::time_point; diff --git a/goal_src/goal-lib.gc b/goal_src/goal-lib.gc index cfe67746c3..7eeafd8a2a 100644 --- a/goal_src/goal-lib.gc +++ b/goal_src/goal-lib.gc @@ -1,5 +1,7 @@ ;;-*-Lisp-*- +(seval (define INSTRUCTION_SET 'x86)) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; BUILD SYSTEM ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/goal_src/jak1/kernel/gkernel-h.gc b/goal_src/jak1/kernel/gkernel-h.gc index a583af51bf..f358eb9a66 100644 --- a/goal_src/jak1/kernel/gkernel-h.gc +++ b/goal_src/jak1/kernel/gkernel-h.gc @@ -119,7 +119,8 @@ ;; trigger an exception. (GOAL used lw r0, 2(r0)) (defmacro break () - `(/ 0 0)) + "trigger an exception." + `(begin (.break) 0)) (defmacro msg-err (&rest args) "Print a message to stdout immediately. This won't appear in the compiler. diff --git a/goal_src/jak1/kernel/gkernel.gc b/goal_src/jak1/kernel/gkernel.gc index 73a056e45a..74a94160ff 100644 --- a/goal_src/jak1/kernel/gkernel.gc +++ b/goal_src/jak1/kernel/gkernel.gc @@ -426,27 +426,51 @@ (declare (asm-func none) ;(print-asm) ) - (rlet ((sp :reg rsp :type uint) - (off :reg r15 :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint)) - ;; get the kernel stack pointer as a GOAL pointer (won't use a temp reg) - (.load-sym :sext #f sp *kernel-sp*) - ;; convert it back to a real pointer - (.add sp off) - ;; restore saved registers... - ;; without coloring system because this is "cheating" and modifying saved registers without backing up. - (.pop :color #f s4) - (.pop :color #f s3) - (.pop :color #f s2) - (.pop :color #f s1) - (.pop :color #f s0) - ;; return to the kernel function that called the user code - ;; rax should still contain the return value. - (.ret))) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + ;; keep x0 intact while restoring the kernel return address + (rlet ((sp :reg sp :type uint) + (off :reg off :type uint) + (temp :reg x9 :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint)) + ;; build sp in temp because it cannot be loaded directly + (.load-sym :sext #f temp *kernel-sp*) + (.add temp off) + (set! sp temp) + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + (.pop :color #f ra) + (.ret))) + (#t + (rlet ((sp :reg rsp :type uint) + (off :reg r15 :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint)) + ;; get the kernel stack pointer as a GOAL pointer (won't use a temp reg) + (.load-sym :sext #f sp *kernel-sp*) + ;; convert it back to a real pointer + (.add sp off) + ;; restore saved registers... + ;; without coloring system because this is "cheating" and modifying saved registers without backing up. + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + ;; return to the kernel function that called the user code + ;; rax should still contain the return value. + (.ret))))) (defun return-from-thread-dead () "Like return from thread, but we clean up our process with deactivate first. @@ -454,29 +478,54 @@ (declare (asm-func none) ;(print-asm) ) - (rlet ((pp :reg r13 :type process) - (sp :reg rsp :type uint) - (off :reg r15 :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint)) - ;; first call the deactivate method. - (deactivate pp) - ;; get the kernel stack pointer as a GOAL pointer - (.load-sym :sext #f sp *kernel-sp*) - ;; convert it back to a real pointer - (.add sp off) - ;; restore saved registers... - ;; without coloring system because this is "cheating". - (.pop :color #f s4) - (.pop :color #f s3) - (.pop :color #f s2) - (.pop :color #f s1) - (.pop :color #f s0) - ;; return to the kernel function that called the user code - (.ret))) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + ;; deactivate can clobber lr, so restore it last + (rlet ((pp :reg pp :type process) + (sp :reg sp :type uint) + (off :reg off :type uint) + (temp :reg x9 :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint)) + (deactivate pp) + (.load-sym :sext #f temp *kernel-sp*) + (.add temp off) + (set! sp temp) + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + (.pop :color #f ra) + (.ret))) + (#t + (rlet ((pp :reg r13 :type process) + (sp :reg rsp :type uint) + (off :reg r15 :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint)) + ;; first call the deactivate method. + (deactivate pp) + ;; get the kernel stack pointer as a GOAL pointer + (.load-sym :sext #f sp *kernel-sp*) + ;; convert it back to a real pointer + (.add sp off) + ;; restore saved registers... + ;; without coloring system because this is "cheating". + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + ;; return to the kernel function that called the user code + (.ret))))) (defmacro abandon-thread () ;; abandon this one too. @@ -485,12 +534,22 @@ ;; but we don't because I don't have an easy way to find where to stick this. ;; I can't see how this makes a difference, as all non-main threads seem ;; temporary, but if this turns out to be false, we will need to change this. - `(rlet ((temp) - (off :reg r15 :type uint :reset-here #t)) - (.mov temp return-from-thread) ;; could probably just call this... - (.add temp off) - (.push temp) - (.ret))) + ;; keep #cond inside the quasiquote so it remains in the macro expansion + `(#cond + ((eq? INSTRUCTION_SET 'arm64) + ;; branch to return-from-thread through the executable alias + (rlet ((temp) + (xoff :reg exec-off :type uint :reset-here #t)) + (.mov temp return-from-thread) + (.add temp xoff) + (.jr temp))) + (#t + (rlet ((temp) + (off :reg r15 :type uint :reset-here #t)) + (.mov temp return-from-thread) ;; could probably just call this... + (.add temp off) + (.push temp) + (.ret))))) (defun reset-and-call ((this thread) (func function)) "Make the given thread the top thread, reset the stack, and call the function. @@ -499,40 +558,80 @@ NOTE: this should only be done from the kernel, running on the kernel's stack." (declare (asm-func object)) - (rlet ((pp :reg r13 :type process) - (sp :reg rsp :type uint) - (off :reg r15 :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (temp :reg rax :type uint)) - ;; set up the process pointer - (set! pp (-> this process)) - ;; mark the process as running and set its top thread - (set! (-> pp status) 'running) - (set! (-> pp top-thread) this) - ;; save the current kernel regs - (.push :color #f s0) - (.push :color #f s1) - (.push :color #f s2) - (.push :color #f s3) - (.push :color #f s4) - ;; make rsp a GOAL pointer - (.sub sp off) - ;; and store it - (set! *kernel-sp* (the pointer sp)) ;; todo, asm form here? - ;; setup the rsp for the new thread - (set! sp (the uint (-> this stack-top))) - (.add sp off) - ;; push the return trampoline to the stack for the user code to return to - (set! temp (the uint return-from-thread)) - (.add temp off) - (.push temp) ;; stack now 16 + 8 aligned - ;; and call the function! - (.add func off) - (.jr func))) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((pp :reg pp :type process) + (sp :reg sp :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (temp :reg x9 :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint)) + ;; set up the process pointer + (set! pp (-> this process)) + ;; mark the process as running and set its top thread + (set! (-> pp status) 'running) + (set! (-> pp top-thread) this) + ;; save the link register before the other kernel registers to match the restore order + (.push :color #f ra) + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) + ;; store the kernel stack pointer as a GOAL address + (set! temp sp) + (.sub temp off) + (set! *kernel-sp* (the pointer temp)) + ;; move onto the thread's stack + (set! temp (the uint (-> this stack-top))) + (.add temp off) + (set! sp temp) + ;; put return-from-thread in the link register before calling func + (set! ra (the uint return-from-thread)) + (.add ra xoff) + ;; convert func to the executable alias before branching + (.add func xoff) + (.jr func))) + (#t + (rlet ((pp :reg r13 :type process) + (sp :reg rsp :type uint) + (off :reg r15 :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (temp :reg rax :type uint)) + ;; set up the process pointer + (set! pp (-> this process)) + ;; mark the process as running and set its top thread + (set! (-> pp status) 'running) + (set! (-> pp top-thread) this) + ;; save the current kernel regs + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) + ;; make rsp a GOAL pointer + (.sub sp off) + ;; and store it + (set! *kernel-sp* (the pointer sp)) + ;; setup the rsp for the new thread + (set! sp (the uint (-> this stack-top))) + (.add sp off) + ;; push the return trampoline to the stack for the user code to return to + (set! temp (the uint return-from-thread)) + (.add temp off) + (.push temp) ;; stack now 16 + 8 aligned + ;; and call the function! + (.add func off) + (.jr func))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Context Suspend And Resume - Thread @@ -546,91 +645,176 @@ ;; we begin this function with the thread object in pp. ;; not sure why we do this, maybe at one point suspending didn't clobber ;; temp registers? - (rlet ((this :reg r13 :type cpu-thread) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr)) - ;; get the return address pushed by "call" in the suspend. - (.pop temp) - ;; convert to a GOAL address - (.sub temp off) - ;; store return address in thread - (set! (-> this pc) (the pointer temp)) - ;; convert our stack pointer to a GOAL address - (.sub sp off) - ;; store in thread. - (set! (-> this sp) (the pointer sp)) - ;; back up registers - (.mov :color #f temp s0) - (set! (-> this rreg 0) temp) - (.mov :color #f temp s1) - (set! (-> this rreg 1) temp) - (.mov :color #f temp s2) - (set! (-> this rreg 2) temp) - (.mov :color #f temp s3) - (set! (-> this rreg 3) temp) - (.mov :color #f temp s4) - (set! (-> this rreg 4) temp) - ;; back up fprs - (.mov :color #f temp xmm8) - (set! (-> this freg 0) (the-as float temp)) - (.mov :color #f temp xmm9) - (set! (-> this freg 1) (the-as float temp)) - (.mov :color #f temp xmm10) - (set! (-> this freg 2) (the-as float temp)) - (.mov :color #f temp xmm11) - (set! (-> this freg 3) (the-as float temp)) - (.mov :color #f temp xmm12) - (set! (-> this freg 4) (the-as float temp)) - (.mov :color #f temp xmm13) - (set! (-> this freg 5) (the-as float temp)) - (.mov :color #f temp xmm14) - (set! (-> this freg 6) (the-as float temp)) - (.mov :color #f temp xmm15) - (set! (-> this freg 7) (the-as float temp)) - ;; get our process - (let ((proc (-> this process))) - (when (> (process-stack-used proc) (-> this stack-size)) - (break) ;; too much stack has been used and we can't suspend! - ;; if you hit this, try with DEBUG_PRINT_SUSPEND_FAIL set to #t (see gkernel-h.gc) - ;; it will print more info before reaching here. - ) - ;; mark the process as suspended and copy the stack - (set! (-> proc status) 'suspended) - (let ((cur (the (pointer uint64) (-> this stack-top))) - (save (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size)))) - (while (> (the int cur) (the int sp)) - (set! cur (the (pointer uint64) (&- cur 8))) - (set! save (the (pointer uint64) (&- save 8))) - (set! (-> save) (-> cur))))) - ;; actually setting pp to 0 - (set! this (the cpu-thread 0)) - ;; get the kernel stack pointer as a GOAL pointer - (.load-sym :sext #f sp *kernel-sp*) - ;; convert it back to a real pointer - (.add sp off) - ;; restore saved registers... - ;; without coloring system because this is "cheating". - (.pop :color #f s4) - (.pop :color #f s3) - (.pop :color #f s2) - (.pop :color #f s1) - (.pop :color #f s0) - ;; return to the kernel function that called the user code - (.ret)) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((this :reg pp :type cpu-thread) + (temp :reg x9 :type uint) + ;; copy sp because these ARM64 store and compare forms cannot use it as a value + (goal-sp :reg x10 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; copy the resume address from lr because blr does not push it + (set! temp ra) + ;; subtract exec-off from the resume address to store a GOAL address + (.sub temp xoff) + (set! (-> this pc) (the pointer temp)) + ;; store sp as a GOAL address + (set! goal-sp sp) + (.sub goal-sp off) + (set! (-> this sp) (the pointer goal-sp)) + ;; back up registers + (.mov :color #f temp s0) + (set! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set! (-> this rreg 4) temp) + ;; back up fprs + (.mov :color #f temp v8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp v9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp v10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp v11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp v12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp v13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp v14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp v15) + (set! (-> this freg 7) (the-as float temp)) + (let ((proc (-> this process))) + (when (> (process-stack-used proc) (-> this stack-size)) + (break) ;; stack overflow prevents suspension + ) + ;; mark the process as suspended and copy the stack + (set! (-> proc status) 'suspended) + (let ((cur (the (pointer uint64) (-> this stack-top))) + (save (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size)))) + (while (> (the int cur) (the int goal-sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! save (the (pointer uint64) (&- save 8))) + (set! (-> save) (-> cur))))) + (set! this (the cpu-thread 0)) + ;; restore the kernel context in reverse save order + (.load-sym :sext #f temp *kernel-sp*) + (.add temp off) + (set! sp temp) + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + (.pop :color #f ra) + (.ret))) + (#t + (rlet ((this :reg r13 :type cpu-thread) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr)) + ;; get the return address pushed by "call" in the suspend. + (.pop temp) + ;; convert to a GOAL address + (.sub temp off) + ;; store return address in thread + (set! (-> this pc) (the pointer temp)) + ;; convert our stack pointer to a GOAL address + (.sub sp off) + ;; store in thread. + (set! (-> this sp) (the pointer sp)) + ;; back up registers + (.mov :color #f temp s0) + (set! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set! (-> this rreg 4) temp) + ;; back up fprs + (.mov :color #f temp xmm8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp xmm9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp xmm10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp xmm11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp xmm12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp xmm13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp xmm14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp xmm15) + (set! (-> this freg 7) (the-as float temp)) + ;; get our process + (let ((proc (-> this process))) + (when (> (process-stack-used proc) (-> this stack-size)) + (break) ;; too much stack has been used and we can't suspend! + ;; if you hit this, try with DEBUG_PRINT_SUSPEND_FAIL set to #t (see gkernel-h.gc) + ;; it will print more info before reaching here. + ) + ;; mark the process as suspended and copy the stack + (set! (-> proc status) 'suspended) + (let ((cur (the (pointer uint64) (-> this stack-top))) + (save (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size)))) + (while (> (the int cur) (the int sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! save (the (pointer uint64) (&- save 8))) + (set! (-> save) (-> cur))))) + ;; actually setting pp to 0 + (set! this (the cpu-thread 0)) + ;; get the kernel stack pointer as a GOAL pointer + (.load-sym :sext #f sp *kernel-sp*) + ;; convert it back to a real pointer + (.add sp off) + ;; restore saved registers... + ;; without coloring system because this is "cheating". + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + ;; return to the kernel function that called the user code + (.ret)))) (none)) (defmethod thread-resume ((thread-to-resume cpu-thread)) @@ -640,101 +824,196 @@ (declare (asm-func none) ;;(print-asm) ) - (rlet ((this :reg r13 :type cpu-thread) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (a4 :reg r8 :type uint) - (a5 :reg r9 :type uint) - (temp-float :reg xmm0 :class fpr) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr)) - ;; save the current kernel regs - (.push :color #f s0) - (.push :color #f s1) - (.push :color #f s2) - (.push :color #f s3) - (.push :color #f s4) - ;; make rsp a GOAL pointer - (.sub sp off) - ;; and store it - (set! *kernel-sp* (the pointer sp)) ;; todo, asm form here? - ;; temp, stash thread in process-pointer - (set! this thread-to-resume) - ;; set stack pointer for the thread. leave it as a GOAL pointer for now.. - (set! sp (the uint (-> this sp))) - ;; restore the stack (sp is a GOAL pointer) - (let ((cur (the (pointer uint64) (-> this stack-top))) - (restore (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size)))) - (while (> (the int cur) (the int sp)) - (set! cur (the (pointer uint64) (&- cur 8))) - (set! restore (the (pointer uint64) (&- restore 8))) - (set! (-> cur) (-> restore)))) - ;; offset sp after we're done using it as a GOAL pointer. - (.add sp off) - ;; setup process - (set! (-> (-> this process) top-thread) this) - (set! (-> (-> this process) status) 'running) - ;; restore reg - (set! temp (-> this rreg 0)) - (.mov :color #f s0 temp) - (set! temp (-> this rreg 1)) - (.mov :color #f s1 temp) - (set! temp (-> this rreg 2)) - (.mov :color #f s2 temp) - (set! temp (-> this rreg 3)) - (.mov :color #f s3 temp) - (set! temp (-> this rreg 4)) - (.mov :color #f s4 temp) - (set! temp-float (-> this freg 0)) - (.mov :color #f xmm8 temp-float) - (set! temp-float (-> this freg 1)) - (.mov :color #f xmm9 temp-float) - (set! temp-float (-> this freg 2)) - (.mov :color #f xmm10 temp-float) - (set! temp-float (-> this freg 3)) - (.mov :color #f xmm11 temp-float) - (set! temp-float (-> this freg 4)) - (.mov :color #f xmm12 temp-float) - (set! temp-float (-> this freg 5)) - (.mov :color #f xmm13 temp-float) - (set! temp-float (-> this freg 6)) - (.mov :color #f xmm14 temp-float) - (set! temp-float (-> this freg 7)) - (.mov :color #f xmm15 temp-float) - ;; hack for set-to-run-bootstrap. The set-to-run-bootstrap in MIPS - ;; expects to receive 7 values from the cpu thread's rregs. - ;; usually rreg holds saved registers, but on the first resume after - ;; a set-to-run, they hold arguments, and set-to-run-bootstrap copies them. - ;; We only have 5 saved regs, so we need to cheat and directly pass - ;; two values in other registers - ;; so we load the a4/a5 argument registers with rreg 5 and rreg 6 - ;; In the case where we are doing a normal resume, the - ;; compiler should assume that these registers are overwritten anyway. - (set! temp (-> this rreg 5)) - (.mov a4 temp) - (set! temp (-> this rreg 6)) - (.mov a5 temp) - ;; get the resume address - (set! temp (the uint (-> this pc))) - (.add temp off) - ;; setup the process - (set! this (the cpu-thread (-> this process))) - ;; resume! - (.jr temp) - (.add a4 a4) - (.add a5 a5)) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((this :reg pp :type cpu-thread) + (temp :reg x9 :type uint) + ;; keep the GOAL sp in x10 because the comparison cannot use sp + (goal-sp :reg x10 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (a4 :reg carg4 :type uint) + (a5 :reg carg5 :type uint) + (temp-float :reg v0 :class fpr) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; save lr before the other kernel registers to match the restore order + (.push :color #f ra) + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) + ;; store the kernel stack pointer as a GOAL address + (set! goal-sp sp) + (.sub goal-sp off) + (set! *kernel-sp* (the pointer goal-sp)) + (set! this thread-to-resume) + ;; keep the thread stack pointer as a GOAL address until the stack copy finishes + (set! goal-sp (the uint (-> this sp))) + ;; restore the stack + (let ((cur (the (pointer uint64) (-> this stack-top))) + (restore (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size)))) + (while (> (the int cur) (the int goal-sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! restore (the (pointer uint64) (&- restore 8))) + (set! (-> cur) (-> restore)))) + ;; convert the restored thread stack pointer to a native address + (.add goal-sp off) + (set! sp goal-sp) + ;; mark the process as running and set its top thread + (set! (-> (-> this process) top-thread) this) + (set! (-> (-> this process) status) 'running) + ;; restore the saved registers + (set! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set! temp (-> this rreg 4)) + (.mov :color #f s4 temp) + (set! temp-float (-> this freg 0)) + (.mov :color #f v8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f v9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f v10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f v11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f v12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f v13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f v14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f v15 temp-float) + ;; restore rreg 5 and rreg 6 into argument registers a4 and a5 + (set! temp (-> this rreg 5)) + (.mov a4 temp) + (set! temp (-> this rreg 6)) + (.mov a5 temp) + ;; convert the resume address to the executable alias before branching + (set! temp (the uint (-> this pc))) + (.add temp xoff) + ;; load the thread process into pp + (set! this (the cpu-thread (-> this process))) + ;; the resumed frame restores lr from its stack + (.jr temp) + (.add a4 a4) + (.add a5 a5))) + (#t + (rlet ((this :reg r13 :type cpu-thread) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (a4 :reg r8 :type uint) + (a5 :reg r9 :type uint) + (temp-float :reg xmm0 :class fpr) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr)) + ;; save the current kernel regs + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) + ;; make rsp a GOAL pointer + (.sub sp off) + ;; and store it + (set! *kernel-sp* (the pointer sp)) + ;; temp, stash thread in process-pointer + (set! this thread-to-resume) + ;; set stack pointer for the thread. leave it as a GOAL pointer for now.. + (set! sp (the uint (-> this sp))) + ;; restore the stack (sp is a GOAL pointer) + (let ((cur (the (pointer uint64) (-> this stack-top))) + (restore (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size)))) + (while (> (the int cur) (the int sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! restore (the (pointer uint64) (&- restore 8))) + (set! (-> cur) (-> restore)))) + ;; offset sp after we're done using it as a GOAL pointer. + (.add sp off) + ;; setup process + (set! (-> (-> this process) top-thread) this) + (set! (-> (-> this process) status) 'running) + ;; restore reg + (set! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set! temp (-> this rreg 4)) + (.mov :color #f s4 temp) + (set! temp-float (-> this freg 0)) + (.mov :color #f xmm8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f xmm9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f xmm10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f xmm11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f xmm12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f xmm13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f xmm14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f xmm15 temp-float) + ;; hack for set-to-run-bootstrap. The set-to-run-bootstrap in MIPS + ;; expects to receive 7 values from the cpu thread's rregs. + ;; usually rreg holds saved registers, but on the first resume after + ;; a set-to-run, they hold arguments, and set-to-run-bootstrap copies them. + ;; We only have 5 saved regs, so we need to cheat and directly pass + ;; two values in other registers + ;; so we load the a4/a5 argument registers with rreg 5 and rreg 6 + ;; In the case where we are doing a normal resume, the + ;; compiler should assume that these registers are overwritten anyway. + (set! temp (-> this rreg 5)) + (.mov a4 temp) + (set! temp (-> this rreg 6)) + (.mov a5 temp) + ;; get the resume address + (set! temp (the uint (-> this pc))) + (.add temp off) + ;; setup the process + (set! this (the cpu-thread (-> this process))) + ;; resume! + (.jr temp) + (.add a4 a4) + (.add a5 a5)))) (none)) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1449,147 +1728,289 @@ (declare (asm-func object) (allow-saved-regs) ;; very dangerous! ) - (rlet ((pp :reg r13 :type process) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type (pointer uint64)) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr)) - ;; we treat the allocation as an address. - (let ((this (the catch-frame (&+ allocation *gtype-basic-offset*)))) - ;; setup catch frame - (set! (-> this type) type-to-make) - (set! (-> this name) name) - ;; get the return address (the compiler won't touch the stack because we're an asm-func) - (.pop temp) - (.push temp) - ;; make it a GOAL address so it fits in 32 bits - (.sub temp off) - ;; store it - (set! (-> this ra) (the int temp)) - ;; todo, do we need a stack offset here? - ;; remember the stack pointer - (set! temp sp) - (.sub temp off) - (set! (-> this sp) (the int temp)) - ;; back up registers we care about - (.mov :color #f temp s0) - (set-u128-as-u64! (-> this rreg 0) temp) - (.mov :color #f temp s1) - (set-u128-as-u64! (-> this rreg 1) temp) - (.mov :color #f temp s2) - (set-u128-as-u64! (-> this rreg 2) temp) - (.mov :color #f temp s3) - (set-u128-as-u64! (-> this rreg 3) temp) - (.mov :color #f temp s4) - (set-u128-as-u64! (-> this rreg 4) temp) - (.mov :color #f temp xmm8) - (set! (-> this freg 0) (the-as float temp)) - (.mov :color #f temp xmm9) - (set! (-> this freg 1) (the-as float temp)) - (.mov :color #f temp xmm10) - (set! (-> this freg 2) (the-as float temp)) - (.mov :color #f temp xmm11) - (set! (-> this freg 3) (the-as float temp)) - (.mov :color #f temp xmm12) - (set! (-> this freg 4) (the-as float temp)) - (.mov :color #f temp xmm13) - (set! (-> this freg 5) (the-as float temp)) - (.mov :color #f temp xmm14) - (set! (-> this freg 6) (the-as float temp)) - (.mov :color #f temp xmm15) - (set! (-> this freg 7) (the-as float temp)) - ;; push this stack frame - (set! (-> this next) (-> pp stack-frame-top)) - (set! (-> pp stack-frame-top) this) - ;; help coloring, it isn't smart enough to realize it's "safe" to use these registers. - (.push :color #f s3) - (.push :color #f s2) - (.push :color #f s2) - (set! s3 (the uint func)) - (set! s2 param-block) - ;; todo - are we aligned correctly here? - (let ((ret ((the-super-u64-fucntion s3) (-> s2 0) (-> s2 1) (-> s2 2) (-> s2 3) (-> s2 4) (-> s2 5)))) - (.pop :color #f s2) - (.pop :color #f s2) - (.pop :color #f s3) - (set! (-> pp stack-frame-top) (-> pp stack-frame-top next)) - (.ret) - (the object ret))))) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((pp :reg pp :type process) + (temp :reg x9 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type (pointer uint64)) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; treat allocation as an address + (let ((this (the catch-frame (&+ allocation *gtype-basic-offset*)))) + ;; initialize the catch frame + (set! (-> this type) type-to-make) + (set! (-> this name) name) + ;; subtract exec-off from lr to store the return address as a GOAL address + (set! temp ra) + (.sub temp xoff) + (set! (-> this ra) (the int temp)) + ;; copy sp through temp because str cannot use sp as data + (set! temp sp) + (.sub temp off) + (set! (-> this sp) (the int temp)) + ;; save the thread registers in the catch frame + (.mov :color #f temp s0) + (set-u128-as-u64! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set-u128-as-u64! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set-u128-as-u64! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set-u128-as-u64! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set-u128-as-u64! (-> this rreg 4) temp) + (.mov :color #f temp v8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp v9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp v10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp v11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp v12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp v13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp v14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp v15) + (set! (-> this freg 7) (the-as float temp)) + ;; add the catch frame to the process frame stack + (set! (-> this next) (-> pp stack-frame-top)) + (set! (-> pp stack-frame-top) this) + ;; save lr before the call clobbers it + (.push :color #f ra) + ;; save s3 and s2 before using them as call temporaries + (.push :color #f s3) + (.push :color #f s2) + (set! s3 (the uint func)) + (set! s2 param-block) + (let ((ret ((the-super-u64-fucntion s3) (-> s2 0) (-> s2 1) (-> s2 2) (-> s2 3) (-> s2 4) (-> s2 5)))) + (.pop :color #f s2) + (.pop :color #f s3) + (.pop :color #f ra) + (set! (-> pp stack-frame-top) (-> pp stack-frame-top next)) + (.ret) + (the object ret))))) + (#t + (rlet ((pp :reg r13 :type process) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type (pointer uint64)) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr)) + ;; we treat the allocation as an address. + (let ((this (the catch-frame (&+ allocation *gtype-basic-offset*)))) + ;; setup catch frame + (set! (-> this type) type-to-make) + (set! (-> this name) name) + ;; get the return address (the compiler won't touch the stack because we're an asm-func) + (.pop temp) + (.push temp) + ;; make it a GOAL address so it fits in 32 bits + (.sub temp off) + ;; store it + (set! (-> this ra) (the int temp)) + ;; store the stack pointer as a GOAL address + (set! temp sp) + (.sub temp off) + (set! (-> this sp) (the int temp)) + ;; back up registers we care about + (.mov :color #f temp s0) + (set-u128-as-u64! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set-u128-as-u64! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set-u128-as-u64! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set-u128-as-u64! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set-u128-as-u64! (-> this rreg 4) temp) + (.mov :color #f temp xmm8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp xmm9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp xmm10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp xmm11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp xmm12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp xmm13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp xmm14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp xmm15) + (set! (-> this freg 7) (the-as float temp)) + ;; push this stack frame + (set! (-> this next) (-> pp stack-frame-top)) + (set! (-> pp stack-frame-top) this) + ;; help coloring, it isn't smart enough to realize it's "safe" to use these registers. + (.push :color #f s3) + (.push :color #f s2) + ;; keep the stack aligned for the call + (.push :color #f s2) + (set! s3 (the uint func)) + (set! s2 param-block) + (let ((ret ((the-super-u64-fucntion s3) (-> s2 0) (-> s2 1) (-> s2 2) (-> s2 3) (-> s2 4) (-> s2 5)))) + (.pop :color #f s2) + (.pop :color #f s2) + (.pop :color #f s3) + (set! (-> pp stack-frame-top) (-> pp stack-frame-top next)) + (.ret) + (the object ret))))))) (defun throw-dispatch ((this catch-frame) value) "Throw the given value to the catch frame. Only can throw a 64-bit value. The original could throw 128 bits." (declare (asm-func none)) - (rlet ((pp :reg r13 :type process) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type (pointer uint64)) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (temp-float :reg xmm0 :class fpr) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr)) - ;; pop everything we threw past - (set! (-> pp stack-frame-top) (-> this next)) - ;; restore regs we care about. - (set-u64-from-u128! temp (-> this rreg 0)) - (.mov :color #f s0 temp) - (set-u64-from-u128! temp (-> this rreg 1)) - (.mov :color #f s1 temp) - (set-u64-from-u128! temp (-> this rreg 2)) - (.mov :color #f s2 temp) - (set-u64-from-u128! temp (-> this rreg 3)) - (.mov :color #f s3 temp) - (set-u64-from-u128! temp (-> this rreg 4)) - (.mov :color #f s4 temp) - (set! temp-float (-> this freg 0)) - (.mov :color #f xmm8 temp-float) - (set! temp-float (-> this freg 1)) - (.mov :color #f xmm9 temp-float) - (set! temp-float (-> this freg 2)) - (.mov :color #f xmm10 temp-float) - (set! temp-float (-> this freg 3)) - (.mov :color #f xmm11 temp-float) - (set! temp-float (-> this freg 4)) - (.mov :color #f xmm12 temp-float) - (set! temp-float (-> this freg 5)) - (.mov :color #f xmm13 temp-float) - (set! temp-float (-> this freg 6)) - (.mov :color #f xmm14 temp-float) - (set! temp-float (-> this freg 7)) - (.mov :color #f xmm15 temp-float) - ;; set stack pointer - (set! sp (the uint (-> this sp))) - (.add sp off) - ;; overwrite our return address - (.pop temp) - (set! temp (the uint (-> this ra))) - (.add temp off) - (.push temp) - ;; load the return register - (.mov temp value) - (.ret))) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((pp :reg pp :type process) + (temp :reg x9 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + ;; reuse x0 for retval after loading the catch frame return address + (retval :reg carg0 :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type (pointer uint64)) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (temp-float :reg v0 :class fpr) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; discard catch frames through this frame + (set! (-> pp stack-frame-top) (-> this next)) + ;; restore the saved registers + (set-u64-from-u128! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set-u64-from-u128! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set-u64-from-u128! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set-u64-from-u128! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set-u64-from-u128! temp (-> this rreg 4)) + (.mov :color #f s4 temp) + (set! temp-float (-> this freg 0)) + (.mov :color #f v8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f v9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f v10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f v11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f v12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f v13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f v14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f v15 temp-float) + ;; rebuild sp through temp because ldr cannot target sp + (set! temp (the uint (-> this sp))) + (.add temp off) + (set! sp temp) + ;; add exec-off to restore the catch frame return address in lr + (set! ra (the uint (-> this ra))) + (.add ra xoff) + ;; load the return register + (.mov retval value) + (.ret))) + (#t + (rlet ((pp :reg r13 :type process) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type (pointer uint64)) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (temp-float :reg xmm0 :class fpr) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr)) + ;; pop everything we threw past + (set! (-> pp stack-frame-top) (-> this next)) + ;; restore regs we care about. + (set-u64-from-u128! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set-u64-from-u128! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set-u64-from-u128! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set-u64-from-u128! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set-u64-from-u128! temp (-> this rreg 4)) + (.mov :color #f s4 temp) + (set! temp-float (-> this freg 0)) + (.mov :color #f xmm8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f xmm9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f xmm10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f xmm11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f xmm12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f xmm13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f xmm14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f xmm15 temp-float) + ;; set stack pointer + (set! sp (the uint (-> this sp))) + (.add sp off) + ;; overwrite our return address + (.pop temp) + (set! temp (the uint (-> this ra))) + (.add temp off) + (.push temp) + ;; load the return register + (.mov temp value) + (.ret))))) (defun throw ((name symbol) value) "Dynamic throw." @@ -1833,31 +2254,59 @@ (declare (asm-func none) ;;(print-asm) ) - (rlet ((s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (a0 :reg rdi :type uint) ; ok - (a1 :reg rsi :type uint) ; ok - (a2 :reg rdx :type uint) ; ok - (a3 :reg rcx :type uint) ; ok - (off :reg r15 :type uint) - (a4 :reg r8 :type uint) - (a5 :reg r9 :type uint) - (temp :reg rax)) - (.mov temp return-from-thread-dead) - (.add temp off) - (.push temp) - ;; stack is 16 + 8 aligned now - (.mov :color #f a0 s1) - (.mov :color #f a1 s2) - (.mov :color #f a2 s3) - (.mov :color #f a3 s4) - (.add :color #f s0 off) - (.jr :color #f s0) - (.add a4 a4) - (.add a5 a5))) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (a0 :reg carg0 :type uint) + (a1 :reg carg1 :type uint) + (a2 :reg carg2 :type uint) + (a3 :reg carg3 :type uint) + (xoff :reg exec-off :type uint) + (a4 :reg carg4 :type uint) + (a5 :reg carg5 :type uint) + (ra :reg lr :type uint)) + ;; set lr before the argument registers are live + (set! ra (the uint return-from-thread-dead)) + (.add ra xoff) + (.mov :color #f a0 s1) + (.mov :color #f a1 s2) + (.mov :color #f a2 s3) + (.mov :color #f a3 s4) + ;; branch targets go through the exec base + (.add :color #f s0 xoff) + (.jr :color #f s0) + (.add a4 a4) + (.add a5 a5))) + (#t + (rlet ((s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (a0 :reg rdi :type uint) ; ok + (a1 :reg rsi :type uint) ; ok + (a2 :reg rdx :type uint) ; ok + (a3 :reg rcx :type uint) ; ok + (off :reg r15 :type uint) + (a4 :reg r8 :type uint) + (a5 :reg r9 :type uint) + (temp :reg rax)) + (.mov temp return-from-thread-dead) + (.add temp off) + (.push temp) + ;; stack is 16 + 8 aligned now + (.mov :color #f a0 s1) + (.mov :color #f a1 s2) + (.mov :color #f a2 s3) + (.mov :color #f a3 s4) + (.add :color #f s0 off) + (.jr :color #f s0) + (.add a4 a4) + (.add a5 a5))))) (defun set-to-run ((thread cpu-thread) (func function) a0 a1 a2 a3 a4 a5) "Set the given thread to call the given function with the given arguments next time it resumes. @@ -1952,7 +2401,11 @@ (set! (-> this status) 'dead) ;; and return (with no deactivate) ;; TODO: replace with abandon. - (let ((temp (the uint return-from-thread))) (rlet ((off :reg r15 :type uint)) (+! temp off) (.push temp) (.ret)))) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (let ((temp (the uint return-from-thread))) (rlet ((xoff :reg exec-off :type uint)) (+! temp xoff) (.jr temp)))) + (#t + (let ((temp (the uint return-from-thread))) (rlet ((off :reg r15 :type uint)) (+! temp off) (.push temp) (.ret)))))) ;; second case - we deactivated while initializing. ((eq? (-> this status) 'initialize) (set! (-> this status) 'dead) diff --git a/goal_src/jak1/kernel/gstate.gc b/goal_src/jak1/kernel/gstate.gc index 29ee50f60d..3993c9f99d 100644 --- a/goal_src/jak1/kernel/gstate.gc +++ b/goal_src/jak1/kernel/gstate.gc @@ -343,38 +343,70 @@ It type checks the arguments for the entry function. ;; we need to: ;; - make sure that when this code returns, we do a deactivate ;; - reset the stack to the top, so we can't just call the code. - (rlet ((temp) - (func) - (sp :reg rsp :type uint) - (off :reg r15 :type uint) - (carg0 :reg rdi) - (carg1 :reg rsi) - (carg2 :reg rdx) - (carg3 :reg rcx)) - ;; prepare args - ;; compiler will likely have these on the stack, we need to get them in regs - ;; before messing with the stack. - (.mov carg0 arg0) - (.mov carg1 arg1) - (.mov carg2 arg2) - (.mov carg3 arg3) - ;; get the main code as an x86-64 pointer - (.mov func (-> new-state code)) - (.add func off) - ;; reset the stack (scary) - (.mov sp (-> pp main-thread stack-top)) - (.add sp off) - ;; push the return trampoline for when code returns. - (.mov temp return-from-thread-dead) ;; will deactivate - (.add temp off) - (.push temp) - ;; and call! - (.jr func) - ;; stupid hack so the compiler doesn't throw away these registers. - (.add carg0 carg1) - (.add carg2 carg3) - #f ;; can't get here - ))) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((func) + (sp :reg sp :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (ra :reg lr :type uint) + (carg0 :reg carg0) + (carg1 :reg carg1) + (carg2 :reg carg2) + (carg3 :reg carg3)) + ;; load the arguments before discarding their stack slots + (.mov carg0 arg0) + (.mov carg1 arg1) + (.mov carg2 arg2) + (.mov carg3 arg3) + ;; convert the code pointer to the executable alias before branching + (.mov func (-> new-state code)) + (.add func xoff) + ;; reset sp through the writable alias + (.mov sp (-> pp main-thread stack-top)) + (.add sp off) + ;; return through deactivate + (.mov ra return-from-thread-dead) + (.add ra xoff) + (.jr func) + ;; keep the argument registers live until the branch + (.add carg0 carg1) + (.add carg2 carg3) + #f ;; can't get here + )) + (#t + (rlet ((temp) + (func) + (sp :reg rsp :type uint) + (off :reg r15 :type uint) + (carg0 :reg rdi) + (carg1 :reg rsi) + (carg2 :reg rdx) + (carg3 :reg rcx)) + ;; prepare args + ;; compiler will likely have these on the stack, we need to get them in regs + ;; before messing with the stack. + (.mov carg0 arg0) + (.mov carg1 arg1) + (.mov carg2 arg2) + (.mov carg3 arg3) + ;; get the main code as an x86-64 pointer + (.mov func (-> new-state code)) + (.add func off) + ;; reset the stack (scary) + (.mov sp (-> pp main-thread stack-top)) + (.add sp off) + ;; push the return trampoline for when code returns. + (.mov temp return-from-thread-dead) ;; will deactivate + (.add temp off) + (.push temp) + ;; and call! + (.jr func) + ;; stupid hack so the compiler doesn't throw away these registers. + (.add carg0 carg1) + (.add carg2 carg3) + #f ;; can't get here + ))))) (else ;; not in the main-thread. ;; so we set up the main thread to try again. @@ -387,14 +419,25 @@ It type checks the arguments for the entry function. ;; but we don't because I don't have an easy way to find where to stick this. ;; I can't see how this makes a difference, as all non-main threads seem ;; temporary, but if this turns out to be false, we will need to change this. - (rlet ((temp) - (off :reg r15 :type uint :reset-here #t)) - (.mov temp return-from-thread) ;; could probably just call this... - (.add temp off) - (.push temp) - (.ret) - #f ;; can't get here - )))))) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + ;; branch to return-from-thread through the executable alias + (rlet ((temp) + (xoff :reg exec-off :type uint :reset-here #t)) + (.mov temp return-from-thread) + (.add temp xoff) + (.jr temp) + #f ;; can't get here + )) + (#t + (rlet ((temp) + (off :reg r15 :type uint :reset-here #t)) + (.mov temp return-from-thread) ;; could probably just call this... + (.add temp off) + (.push temp) + (.ret) + #f ;; can't get here + )))))))) (defun send-event-function ((proc process-tree) (msg event-message-block)) "Function to send an event to a process. Please use the send-event macros when possible" diff --git a/goal_src/jak2/kernel/gkernel-h.gc b/goal_src/jak2/kernel/gkernel-h.gc index ddb4e64e24..dad97392f3 100644 --- a/goal_src/jak2/kernel/gkernel-h.gc +++ b/goal_src/jak2/kernel/gkernel-h.gc @@ -600,8 +600,8 @@ ) (defmacro break () - "crash the game by dividing by 0." - `(/ 0 0) + "trigger an exception." + `(begin (.break) 0) ) (defmacro with-pp (&rest body) @@ -726,4 +726,4 @@ ) (defmacro suspend-for (time &rest body) - `(let ((time (current-time))) (until (time-elapsed? time ,time) ,@body (suspend)))) \ No newline at end of file + `(let ((time (current-time))) (until (time-elapsed? time ,time) ,@body (suspend)))) diff --git a/goal_src/jak2/kernel/gkernel.gc b/goal_src/jak2/kernel/gkernel.gc index 8b500c9169..78bc343a05 100644 --- a/goal_src/jak2/kernel/gkernel.gc +++ b/goal_src/jak2/kernel/gkernel.gc @@ -450,30 +450,55 @@ (declare (asm-func none) ;(print-asm) ) - (rlet ((sp :reg rsp :type uint) - (off :reg r15 :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - ) - ;; get the kernel stack pointer as a GOAL pointer (won't use a temp reg) - (.load-sym :sext #f sp *kernel-sp*) - ;; convert it back to a real pointer - (.add sp off) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + ;; keep x0 intact while restoring the kernel return address + (rlet ((sp :reg sp :type uint) + (off :reg off :type uint) + (temp :reg x9 :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint)) + ;; build sp in temp because it cannot be loaded directly + (.load-sym :sext #f temp *kernel-sp*) + (.add temp off) + (set! sp temp) + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + (.pop :color #f ra) + (.ret))) + (#t + (rlet ((sp :reg rsp :type uint) + (off :reg r15 :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + ) + ;; get the kernel stack pointer as a GOAL pointer (won't use a temp reg) + (.load-sym :sext #f sp *kernel-sp*) + ;; convert it back to a real pointer + (.add sp off) - ;; restore saved registers... - ;; without coloring system because this is "cheating" and modifying saved registers without backing up. - (.pop :color #f s4) - (.pop :color #f s3) - (.pop :color #f s2) - (.pop :color #f s1) - (.pop :color #f s0) - ;; return to the kernel function that called the user code - ;; rax should still contain the return value. - (.ret) - ) + ;; restore saved registers... + ;; without coloring system because this is "cheating" and modifying saved registers without backing up. + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + ;; return to the kernel function that called the user code + ;; rax should still contain the return value. + (.ret) + ) + )) ) (defun return-from-thread-dead () @@ -482,33 +507,59 @@ (declare (asm-func none) ;(print-asm) ) - (rlet ((pp :reg r13 :type process) - (sp :reg rsp :type uint) - (off :reg r15 :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - ) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + ;; deactivate can clobber lr, so restore it last + (rlet ((pp :reg pp :type process) + (sp :reg sp :type uint) + (off :reg off :type uint) + (temp :reg x9 :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint)) + (deactivate pp) + (.load-sym :sext #f temp *kernel-sp*) + (.add temp off) + (set! sp temp) + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + (.pop :color #f ra) + (.ret))) + (#t + (rlet ((pp :reg r13 :type process) + (sp :reg rsp :type uint) + (off :reg r15 :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + ) - ;; first call the deactivate method. - (deactivate pp) - ;; get the kernel stack pointer as a GOAL pointer - (.load-sym :sext #f sp *kernel-sp*) - ;; convert it back to a real pointer - (.add sp off) + ;; first call the deactivate method. + (deactivate pp) + ;; get the kernel stack pointer as a GOAL pointer + (.load-sym :sext #f sp *kernel-sp*) + ;; convert it back to a real pointer + (.add sp off) - ;; restore saved registers... - ;; without coloring system because this is "cheating". - (.pop :color #f s4) - (.pop :color #f s3) - (.pop :color #f s2) - (.pop :color #f s1) - (.pop :color #f s0) - ;; return to the kernel function that called the user code - (.ret) - ) + ;; restore saved registers... + ;; without coloring system because this is "cheating". + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + ;; return to the kernel function that called the user code + (.ret) + ) + )) ) (defmacro abandon-thread () @@ -518,14 +569,22 @@ ;; but we don't because I don't have an easy way to find where to stick this. ;; I can't see how this makes a difference, as all non-main threads seem ;; temporary, but if this turns out to be false, we will need to change this. - `(rlet ((temp) - (off :reg r15 :type uint :reset-here #t)) - (.mov temp return-from-thread) ;; could probably just call this... - (.add temp off) - (.push temp) - (.ret) - ) - ) + ;; keep #cond inside the quasiquote so it remains in the macro expansion + `(#cond + ((eq? INSTRUCTION_SET 'arm64) + ;; branch to return-from-thread through the executable alias + (rlet ((temp) + (xoff :reg exec-off :type uint :reset-here #t)) + (.mov temp return-from-thread) + (.add temp xoff) + (.jr temp))) + (#t + (rlet ((temp) + (off :reg r15 :type uint :reset-here #t)) + (.mov temp return-from-thread) ;; could probably just call this... + (.add temp off) + (.push temp) + (.ret))))) (defun reset-and-call ((this thread) (func function)) "Make the given thread the top thread, reset the stack, and call the function. @@ -536,47 +595,88 @@ (declare (asm-func object) ) - (rlet ((pp :reg r13 :type process) - (sp :reg rsp :type uint) - (off :reg r15 :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (temp :reg rax :type uint) - ) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((pp :reg pp :type process) + (sp :reg sp :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (temp :reg x9 :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint)) + ;; set up the process pointer + (set! pp (-> this process)) + ;; mark the process as running and set its top thread + (set! (-> pp status) 'running) + (set! (-> pp top-thread) (the cpu-thread this)) + ;; save the link register before the other kernel registers to match the restore order + (.push :color #f ra) + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) + ;; store the kernel stack pointer as a GOAL address + (set! temp sp) + (.sub temp off) + (set! *kernel-sp* (the pointer temp)) + ;; move onto the thread's stack + (set! temp (the uint (-> this stack-top))) + (.add temp off) + (set! sp temp) + ;; put return-from-thread in the link register before calling func + (set! ra (the uint return-from-thread)) + (.add ra xoff) + ;; convert func to the executable alias before branching + (.add func xoff) + (.jr func))) + (#t + (rlet ((pp :reg r13 :type process) + (sp :reg rsp :type uint) + (off :reg r15 :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (temp :reg rax :type uint) + ) - ;; set up the process pointer - (set! pp (-> this process)) - ;; mark the process as running and set its top thread - (set! (-> pp status) 'running) - (set! (-> pp top-thread) (the cpu-thread this)) + ;; set up the process pointer + (set! pp (-> this process)) + ;; mark the process as running and set its top thread + (set! (-> pp status) 'running) + (set! (-> pp top-thread) (the cpu-thread this)) - ;; save the current kernel regs - (.push :color #f s0) - (.push :color #f s1) - (.push :color #f s2) - (.push :color #f s3) - (.push :color #f s4) + ;; save the current kernel regs + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) - ;; make rsp a GOAL pointer - (.sub sp off) - ;; and store it - (set! *kernel-sp* (the pointer sp)) ;; todo, asm form here? + ;; make rsp a GOAL pointer + (.sub sp off) + ;; and store it + (set! *kernel-sp* (the pointer sp)) - ;; setup the rsp for the new thread - (set! sp (the uint (-> this stack-top))) - (.add sp off) + ;; setup the rsp for the new thread + (set! sp (the uint (-> this stack-top))) + (.add sp off) - ;; push the return trampoline to the stack for the user code to return to - (set! temp (the uint return-from-thread)) - (.add temp off) - (.push temp) ;; stack now 16 + 8 aligned - ;; and call the function! - (.add func off) - (.jr func) - ) + ;; push the return trampoline to the stack for the user code to return to + (set! temp (the uint return-from-thread)) + (.add temp off) + (.push temp) ;; stack now 16 + 8 aligned + ;; and call the function! + (.add func off) + (.jr func) + ) + )) ) (defmethod thread-suspend ((unused cpu-thread)) @@ -587,109 +687,195 @@ ;; we begin this function with the thread object in pp. ;; not sure why we do this, maybe at one point suspending didn't clobber ;; temp registers? - (rlet ((this :reg r13 :type cpu-thread) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((this :reg pp :type cpu-thread) + (temp :reg x9 :type uint) + ;; copy sp because these ARM64 store and compare forms cannot use it as a value + (goal-sp :reg x10 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; copy the resume address from lr because blr does not push it + (set! temp ra) + ;; subtract exec-off from the resume address to store a GOAL address + (.sub temp xoff) + (set! (-> this pc) (the pointer temp)) + ;; store sp as a GOAL address + (set! goal-sp sp) + (.sub goal-sp off) + (set! (-> this sp) (the pointer goal-sp)) + ;; back up registers + (.mov :color #f temp s0) + (set! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set! (-> this rreg 4) temp) + ;; back up fprs + (.mov :color #f temp v8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp v9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp v10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp v11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp v12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp v13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp v14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp v15) + (set! (-> this freg 7) (the-as float temp)) + (let ((proc (-> this process))) + (when (> (process-stack-used proc) (-> this stack-size)) + (break) ;; stack overflow prevents suspension + ) + ;; mark the process as suspended and copy the stack + (set! (-> proc status) 'suspended) + (let ((cur (the (pointer uint64) (-> this stack-top))) + (save (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size)))) + (while (> (the int cur) (the int goal-sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! save (the (pointer uint64) (&- save 8))) + (set! (-> save) (-> cur))))) + (set! this (the cpu-thread 0)) + ;; restore the kernel context in reverse save order + (.load-sym :sext #f temp *kernel-sp*) + (.add temp off) + (set! sp temp) + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + (.pop :color #f ra) + (.ret))) + (#t + (rlet ((this :reg r13 :type cpu-thread) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr) + ) + + ;; get the return address pushed by "call" in the suspend. + (.pop temp) + ;; convert to a GOAL address + (.sub temp off) + ;; store return address in thread + (set! (-> this pc) (the pointer temp)) + + ;; convert our stack pointer to a GOAL address + (.sub sp off) + ;; store in thread. + (set! (-> this sp) (the pointer sp)) + + ;; back up registers + (.mov :color #f temp s0) + (set! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set! (-> this rreg 4) temp) + + ;; back up fprs + (.mov :color #f temp xmm8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp xmm9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp xmm10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp xmm11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp xmm12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp xmm13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp xmm14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp xmm15) + (set! (-> this freg 7) (the-as float temp)) + + + + ;; get our process + (let ((proc (-> this process))) + (when (> (process-stack-used proc) (-> this stack-size)) + (break) ;; too much stack has been used and we can't suspend! + ;; if you hit this, try with DEBUG_PRINT_SUSPEND_FAIL set to #t (see gkernel-h.gc) + ;; it will print more info before reaching here. ) - ;; get the return address pushed by "call" in the suspend. - (.pop temp) - ;; convert to a GOAL address - (.sub temp off) - ;; store return address in thread - (set! (-> this pc) (the pointer temp)) + ;; mark the process as suspended and copy the stack + (set! (-> proc status) 'suspended) + (let ((cur (the (pointer uint64) (-> this stack-top))) + (save (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size))) + ) + (while (> (the int cur) (the int sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! save (the (pointer uint64) (&- save 8))) + (set! (-> save) (-> cur)) + ) + ) + ) - ;; convert our stack pointer to a GOAL address - (.sub sp off) - ;; store in thread. - (set! (-> this sp) (the pointer sp)) + ;; actually setting pp to 0 + (set! this (the cpu-thread 0)) - ;; back up registers - (.mov :color #f temp s0) - (set! (-> this rreg 0) temp) - (.mov :color #f temp s1) - (set! (-> this rreg 1) temp) - (.mov :color #f temp s2) - (set! (-> this rreg 2) temp) - (.mov :color #f temp s3) - (set! (-> this rreg 3) temp) - (.mov :color #f temp s4) - (set! (-> this rreg 4) temp) + ;; get the kernel stack pointer as a GOAL pointer + (.load-sym :sext #f sp *kernel-sp*) + ;; convert it back to a real pointer + (.add sp off) - ;; back up fprs - (.mov :color #f temp xmm8) - (set! (-> this freg 0) (the-as float temp)) - (.mov :color #f temp xmm9) - (set! (-> this freg 1) (the-as float temp)) - (.mov :color #f temp xmm10) - (set! (-> this freg 2) (the-as float temp)) - (.mov :color #f temp xmm11) - (set! (-> this freg 3) (the-as float temp)) - (.mov :color #f temp xmm12) - (set! (-> this freg 4) (the-as float temp)) - (.mov :color #f temp xmm13) - (set! (-> this freg 5) (the-as float temp)) - (.mov :color #f temp xmm14) - (set! (-> this freg 6) (the-as float temp)) - (.mov :color #f temp xmm15) - (set! (-> this freg 7) (the-as float temp)) - - - - ;; get our process - (let ((proc (-> this process))) - (when (> (process-stack-used proc) (-> this stack-size)) - (break) ;; too much stack has been used and we can't suspend! - ;; if you hit this, try with DEBUG_PRINT_SUSPEND_FAIL set to #t (see gkernel-h.gc) - ;; it will print more info before reaching here. - ) - - ;; mark the process as suspended and copy the stack - (set! (-> proc status) 'suspended) - (let ((cur (the (pointer uint64) (-> this stack-top))) - (save (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size))) - ) - (while (> (the int cur) (the int sp)) - (set! cur (the (pointer uint64) (&- cur 8))) - (set! save (the (pointer uint64) (&- save 8))) - (set! (-> save) (-> cur)) - ) - ) - ) - - ;; actually setting pp to 0 - (set! this (the cpu-thread 0)) - - ;; get the kernel stack pointer as a GOAL pointer - (.load-sym :sext #f sp *kernel-sp*) - ;; convert it back to a real pointer - (.add sp off) - - ;; restore saved registers... - ;; without coloring system because this is "cheating". - (.pop :color #f s4) - (.pop :color #f s3) - (.pop :color #f s2) - (.pop :color #f s1) - (.pop :color #f s0) - ;; return to the kernel function that called the user code - (.ret) - ) + ;; restore saved registers... + ;; without coloring system because this is "cheating". + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + ;; return to the kernel function that called the user code + (.ret) + ) + )) (none) ) @@ -701,119 +887,215 @@ ;;(print-asm) ) - (rlet ((this :reg r13 :type cpu-thread) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (a4 :reg r8 :type uint) - (a5 :reg r9 :type uint) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((this :reg pp :type cpu-thread) + (temp :reg x9 :type uint) + ;; keep the GOAL sp in x10 because the comparison cannot use sp + (goal-sp :reg x10 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (a4 :reg carg4 :type uint) + (a5 :reg carg5 :type uint) + (temp-float :reg v0 :class fpr) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; save lr before the other kernel registers to match the restore order + (.push :color #f ra) + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) + ;; store the kernel stack pointer as a GOAL address + (set! goal-sp sp) + (.sub goal-sp off) + (set! *kernel-sp* (the pointer goal-sp)) + (set! this thread-to-resume) + ;; keep the thread stack pointer as a GOAL address until the stack copy finishes + (set! goal-sp (the uint (-> this sp))) + ;; restore the stack + (let ((cur (the (pointer uint64) (-> this stack-top))) + (restore (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size)))) + (while (> (the int cur) (the int goal-sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! restore (the (pointer uint64) (&- restore 8))) + (set! (-> cur) (-> restore)))) + ;; convert the restored thread stack pointer to a native address + (.add goal-sp off) + (set! sp goal-sp) + ;; mark the process as running and set its top thread + (set! (-> (-> this process) top-thread) this) + (set! (-> (-> this process) status) 'running) + ;; restore the saved registers + (set! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set! temp (-> this rreg 4)) + (.mov :color #f s4 temp) + (set! temp-float (-> this freg 0)) + (.mov :color #f v8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f v9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f v10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f v11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f v12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f v13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f v14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f v15 temp-float) + ;; restore rreg 5 and rreg 6 into argument registers a4 and a5 + (set! temp (-> this rreg 5)) + (.mov a4 temp) + (set! temp (-> this rreg 6)) + (.mov a5 temp) + ;; convert the resume address to the executable alias before branching + (set! temp (the uint (-> this pc))) + (.add temp xoff) + ;; load the thread process into pp + (set! this (the cpu-thread (-> this process))) + ;; the resumed frame restores lr from its stack + (.jr temp) + (.add a4 a4) + (.add a5 a5))) + (#t + (rlet ((this :reg r13 :type cpu-thread) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (a4 :reg r8 :type uint) + (a5 :reg r9 :type uint) - (temp-float :reg xmm0 :class fpr) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr) - ) - - ;; save the current kernel regs - (.push :color #f s0) - (.push :color #f s1) - (.push :color #f s2) - (.push :color #f s3) - (.push :color #f s4) - - ;; make rsp a GOAL pointer - (.sub sp off) - ;; and store it - (set! *kernel-sp* (the pointer sp)) ;; todo, asm form here? - - ;; temp, stash thread in process-pointer - (set! this thread-to-resume) - - ;; set stack pointer for the thread. leave it as a GOAL pointer for now.. - (set! sp (the uint (-> this sp))) - - ;; restore the stack (sp is a GOAL pointer) - (let ((cur (the (pointer uint64) (-> this stack-top))) - (restore (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size))) + (temp-float :reg xmm0 :class fpr) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr) ) - (while (> (the int cur) (the int sp)) - (set! cur (the (pointer uint64) (&- cur 8))) - (set! restore (the (pointer uint64) (&- restore 8))) - (set! (-> cur) (-> restore)) - ) - ) - ;; offset sp after we're done using it as a GOAL pointer. - (.add sp off) + ;; save the current kernel regs + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) - ;; setup process - (set! (-> (-> this process) top-thread) this) - (set! (-> (-> this process) status) 'running) + ;; make rsp a GOAL pointer + (.sub sp off) + ;; and store it + (set! *kernel-sp* (the pointer sp)) - ;; restore reg - (set! temp (-> this rreg 0)) - (.mov :color #f s0 temp) - (set! temp (-> this rreg 1)) - (.mov :color #f s1 temp) - (set! temp (-> this rreg 2)) - (.mov :color #f s2 temp) - (set! temp (-> this rreg 3)) - (.mov :color #f s3 temp) - (set! temp (-> this rreg 4)) - (.mov :color #f s4 temp) - (set! temp-float (-> this freg 0)) - (.mov :color #f xmm8 temp-float) - (set! temp-float (-> this freg 1)) - (.mov :color #f xmm9 temp-float) - (set! temp-float (-> this freg 2)) - (.mov :color #f xmm10 temp-float) - (set! temp-float (-> this freg 3)) - (.mov :color #f xmm11 temp-float) - (set! temp-float (-> this freg 4)) - (.mov :color #f xmm12 temp-float) - (set! temp-float (-> this freg 5)) - (.mov :color #f xmm13 temp-float) - (set! temp-float (-> this freg 6)) - (.mov :color #f xmm14 temp-float) - (set! temp-float (-> this freg 7)) - (.mov :color #f xmm15 temp-float) + ;; temp, stash thread in process-pointer + (set! this thread-to-resume) - ;; hack for set-to-run-bootstrap. The set-to-run-bootstrap in MIPS - ;; expects to receive 7 values from the cpu thread's rregs. - ;; usually rreg holds saved registers, but on the first resume after - ;; a set-to-run, they hold arguments, and set-to-run-bootstrap copies them. + ;; set stack pointer for the thread. leave it as a GOAL pointer for now.. + (set! sp (the uint (-> this sp))) - ;; We only have 5 saved regs, so we need to cheat and directly pass - ;; two values in other registers - ;; so we load the a4/a5 argument registers with rreg 5 and rreg 6 - ;; In the case where we are doing a normal resume, the - ;; compiler should assume that these registers are overwritten anyway. - (set! temp (-> this rreg 5)) - (.mov a4 temp) - (set! temp (-> this rreg 6)) - (.mov a5 temp) + ;; restore the stack (sp is a GOAL pointer) + (let ((cur (the (pointer uint64) (-> this stack-top))) + (restore (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size))) + ) + (while (> (the int cur) (the int sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! restore (the (pointer uint64) (&- restore 8))) + (set! (-> cur) (-> restore)) + ) + ) - ;; get the resume address - (set! temp (the uint (-> this pc))) - (.add temp off) + ;; offset sp after we're done using it as a GOAL pointer. + (.add sp off) - ;; setup the process - (set! this (the cpu-thread (-> this process))) - ;; resume! - (.jr temp) - (.add a4 a4) - (.add a5 a5) - ) + ;; setup process + (set! (-> (-> this process) top-thread) this) + (set! (-> (-> this process) status) 'running) + + ;; restore reg + (set! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set! temp (-> this rreg 4)) + (.mov :color #f s4 temp) + (set! temp-float (-> this freg 0)) + (.mov :color #f xmm8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f xmm9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f xmm10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f xmm11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f xmm12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f xmm13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f xmm14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f xmm15 temp-float) + + ;; hack for set-to-run-bootstrap. The set-to-run-bootstrap in MIPS + ;; expects to receive 7 values from the cpu thread's rregs. + ;; usually rreg holds saved registers, but on the first resume after + ;; a set-to-run, they hold arguments, and set-to-run-bootstrap copies them. + + ;; We only have 5 saved regs, so we need to cheat and directly pass + ;; two values in other registers + ;; so we load the a4/a5 argument registers with rreg 5 and rreg 6 + ;; In the case where we are doing a normal resume, the + ;; compiler should assume that these registers are overwritten anyway. + (set! temp (-> this rreg 5)) + (.mov a4 temp) + (set! temp (-> this rreg 6)) + (.mov a5 temp) + + ;; get the resume address + (set! temp (the uint (-> this pc))) + (.add temp off) + + ;; setup the process + (set! this (the cpu-thread (-> this process))) + ;; resume! + (.jr temp) + (.add a4 a4) + (.add a5 a5) + ) + )) (none) ) ;; decomp deviation @@ -1782,104 +2064,183 @@ (allow-saved-regs) ;; very dangerous! ) - (rlet ((pp :reg r13 :type process) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type (pointer uint64)) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((pp :reg pp :type process) + (temp :reg x9 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type (pointer uint64)) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; treat allocation as an address + (let ((this (the catch-frame (&+ (the pointer allocation) *gtype-basic-offset*)))) + ;; initialize the catch frame + (set! (-> this type) type-to-make) + (set! (-> this name) name) + ;; subtract exec-off from lr to store the return address as a GOAL address + (set! temp ra) + (.sub temp xoff) + (set! (-> this ra) (the int temp)) + ;; copy sp through temp because str cannot use sp as data + (set! temp sp) + (.sub temp off) + (set! (-> this sp) (the int temp)) + ;; save the thread registers in the catch frame + (.mov :color #f temp s0) + (set-u128-as-u64! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set-u128-as-u64! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set-u128-as-u64! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set-u128-as-u64! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set-u128-as-u64! (-> this rreg 4) temp) + (.mov :color #f temp v8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp v9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp v10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp v11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp v12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp v13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp v14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp v15) + (set! (-> this freg 7) (the-as float temp)) + ;; add the catch frame to the process frame stack + (set! (-> this next) (-> pp stack-frame-top)) + (set! (-> pp stack-frame-top) this) + ;; save lr before the call clobbers it + (.push :color #f ra) + ;; save s3 and s2 before using them as call temporaries + (.push :color #f s3) + (.push :color #f s2) + (set! s3 (the uint func)) + (set! s2 param-block) + (let ((ret ((the-super-u64-fucntion s3) (-> s2 0) (-> s2 1) (-> s2 2) (-> s2 3) (-> s2 4) (-> s2 5)))) + (.pop :color #f s2) + (.pop :color #f s3) + (.pop :color #f ra) + (set! (-> pp stack-frame-top) (-> pp stack-frame-top next)) + (.ret) + (the object ret))))) + (#t + (rlet ((pp :reg r13 :type process) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type (pointer uint64)) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr) + ) + + ;; we treat the allocation as an address. + (let ((this (the catch-frame (&+ (the pointer allocation) *gtype-basic-offset*)))) + ;; setup catch frame + (set! (-> this type) type-to-make) + (set! (-> this name) name) + ;; get the return address (the compiler won't touch the stack because we're an asm-func) + (.pop temp) + (.push temp) + ;; make it a GOAL address so it fits in 32 bits + (.sub temp off) + ;; store it + (set! (-> this ra) (the int temp)) + + ;; store the stack pointer as a GOAL address + (set! temp sp) + (.sub temp off) + (set! (-> this sp) (the int temp)) + + ;; back up registers we care about + (.mov :color #f temp s0) + (set-u128-as-u64! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set-u128-as-u64! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set-u128-as-u64! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set-u128-as-u64! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set-u128-as-u64! (-> this rreg 4) temp) + + (.mov :color #f temp xmm8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp xmm9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp xmm10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp xmm11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp xmm12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp xmm13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp xmm14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp xmm15) + (set! (-> this freg 7) (the-as float temp)) + + ;; push this stack frame + (set! (-> this next) (-> pp stack-frame-top)) + (set! (-> pp stack-frame-top) this) + + ;; help coloring, it isn't smart enough to realize it's "safe" to use these registers. + (.push :color #f s3) + (.push :color #f s2) + ;; keep the stack aligned for the call + (.push :color #f s2) + (set! s3 (the uint func)) + (set! s2 param-block) + + (let ((ret ((the-super-u64-fucntion s3) + (-> s2 0) + (-> s2 1) + (-> s2 2) + (-> s2 3) + (-> s2 4) + (-> s2 5) + )) + ) + (.pop :color #f s2) + (.pop :color #f s2) + (.pop :color #f s3) + (set! (-> pp stack-frame-top) (-> pp stack-frame-top next)) + (.ret) + (the object ret) ) - - ;; we treat the allocation as an address. - (let ((this (the catch-frame (&+ (the pointer allocation) *gtype-basic-offset*)))) - ;; setup catch frame - (set! (-> this type) type-to-make) - (set! (-> this name) name) - ;; get the return address (the compiler won't touch the stack because we're an asm-func) - (.pop temp) - (.push temp) - ;; make it a GOAL address so it fits in 32 bits - (.sub temp off) - ;; store it - (set! (-> this ra) (the int temp)) - - ;; todo, do we need a stack offset here? - ;; remember the stack pointer - (set! temp sp) - (.sub temp off) - (set! (-> this sp) (the int temp)) - - ;; back up registers we care about - (.mov :color #f temp s0) - (set-u128-as-u64! (-> this rreg 0) temp) - (.mov :color #f temp s1) - (set-u128-as-u64! (-> this rreg 1) temp) - (.mov :color #f temp s2) - (set-u128-as-u64! (-> this rreg 2) temp) - (.mov :color #f temp s3) - (set-u128-as-u64! (-> this rreg 3) temp) - (.mov :color #f temp s4) - (set-u128-as-u64! (-> this rreg 4) temp) - - (.mov :color #f temp xmm8) - (set! (-> this freg 0) (the-as float temp)) - (.mov :color #f temp xmm9) - (set! (-> this freg 1) (the-as float temp)) - (.mov :color #f temp xmm10) - (set! (-> this freg 2) (the-as float temp)) - (.mov :color #f temp xmm11) - (set! (-> this freg 3) (the-as float temp)) - (.mov :color #f temp xmm12) - (set! (-> this freg 4) (the-as float temp)) - (.mov :color #f temp xmm13) - (set! (-> this freg 5) (the-as float temp)) - (.mov :color #f temp xmm14) - (set! (-> this freg 6) (the-as float temp)) - (.mov :color #f temp xmm15) - (set! (-> this freg 7) (the-as float temp)) - - ;; push this stack frame - (set! (-> this next) (-> pp stack-frame-top)) - (set! (-> pp stack-frame-top) this) - - ;; help coloring, it isn't smart enough to realize it's "safe" to use these registers. - (.push :color #f s3) - (.push :color #f s2) - (.push :color #f s2) - (set! s3 (the uint func)) - (set! s2 param-block) - - ;; todo - are we aligned correctly here? - (let ((ret ((the-super-u64-fucntion s3) - (-> s2 0) - (-> s2 1) - (-> s2 2) - (-> s2 3) - (-> s2 4) - (-> s2 5) - )) - ) - (.pop :color #f s2) - (.pop :color #f s2) - (.pop :color #f s3) - (set! (-> pp stack-frame-top) (-> pp stack-frame-top next)) - (.ret) - (the object ret) - ) - ) - ) + ) + ) + )) ) (defun throw-dispatch ((this catch-frame) value) @@ -1887,73 +2248,138 @@ Only can throw a 64-bit value. The original could throw 128 bits." (declare (asm-func none)) - (rlet ((pp :reg r13 :type process) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type (pointer uint64)) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((pp :reg pp :type process) + (temp :reg x9 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + ;; reuse x0 for retval after loading the catch frame return address + (retval :reg carg0 :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type (pointer uint64)) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (temp-float :reg v0 :class fpr) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; discard catch frames through this frame + (set! (-> pp stack-frame-top) (-> this next)) + ;; restore the saved registers + (set-u64-from-u128! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set-u64-from-u128! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set-u64-from-u128! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set-u64-from-u128! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set-u64-from-u128! temp (-> this rreg 4)) + (.mov :color #f s4 temp) + (set! temp-float (-> this freg 0)) + (.mov :color #f v8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f v9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f v10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f v11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f v12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f v13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f v14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f v15 temp-float) + ;; rebuild sp through temp because ldr cannot target sp + (set! temp (the uint (-> this sp))) + (.add temp off) + (set! sp temp) + ;; add exec-off to restore the catch frame return address in lr + (set! ra (the uint (-> this ra))) + (.add ra xoff) + ;; load the return register + (.mov retval value) + (.ret))) + (#t + (rlet ((pp :reg r13 :type process) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type (pointer uint64)) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) - (temp-float :reg xmm0 :class fpr) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr) - ) + (temp-float :reg xmm0 :class fpr) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr) + ) - ;; pop everything we threw past - (set! (-> pp stack-frame-top) (-> this next)) + ;; pop everything we threw past + (set! (-> pp stack-frame-top) (-> this next)) - ;; restore regs we care about. - (set-u64-from-u128! temp (-> this rreg 0)) - (.mov :color #f s0 temp) - (set-u64-from-u128! temp (-> this rreg 1)) - (.mov :color #f s1 temp) - (set-u64-from-u128! temp (-> this rreg 2)) - (.mov :color #f s2 temp) - (set-u64-from-u128! temp (-> this rreg 3)) - (.mov :color #f s3 temp) - (set-u64-from-u128! temp (-> this rreg 4)) - (.mov :color #f s4 temp) + ;; restore regs we care about. + (set-u64-from-u128! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set-u64-from-u128! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set-u64-from-u128! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set-u64-from-u128! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set-u64-from-u128! temp (-> this rreg 4)) + (.mov :color #f s4 temp) - (set! temp-float (-> this freg 0)) - (.mov :color #f xmm8 temp-float) - (set! temp-float (-> this freg 1)) - (.mov :color #f xmm9 temp-float) - (set! temp-float (-> this freg 2)) - (.mov :color #f xmm10 temp-float) - (set! temp-float (-> this freg 3)) - (.mov :color #f xmm11 temp-float) - (set! temp-float (-> this freg 4)) - (.mov :color #f xmm12 temp-float) - (set! temp-float (-> this freg 5)) - (.mov :color #f xmm13 temp-float) - (set! temp-float (-> this freg 6)) - (.mov :color #f xmm14 temp-float) - (set! temp-float (-> this freg 7)) - (.mov :color #f xmm15 temp-float) + (set! temp-float (-> this freg 0)) + (.mov :color #f xmm8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f xmm9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f xmm10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f xmm11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f xmm12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f xmm13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f xmm14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f xmm15 temp-float) - ;; set stack pointer - (set! sp (the uint (-> this sp))) - (.add sp off) + ;; set stack pointer + (set! sp (the uint (-> this sp))) + (.add sp off) - ;; overwrite our return address - (.pop temp) - (set! temp (the uint (-> this ra))) - (.add temp off) - (.push temp) + ;; overwrite our return address + (.pop temp) + (set! temp (the uint (-> this ra))) + (.add temp off) + (.push temp) - ;; load the return register - (.mov temp value) - (.ret) - ) + ;; load the return register + (.mov temp value) + (.ret) + ) + )) ) (defun throw ((name symbol) value) @@ -2255,39 +2681,68 @@ ;;(print-asm) ) - (rlet ((s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (a0 :reg rdi :type uint) ; ok - (a1 :reg rsi :type uint) ; ok - (a2 :reg rdx :type uint) ; ok - (a3 :reg rcx :type uint) ; ok - (off :reg r15 :type uint) - (a4 :reg r8 :type uint) - (a5 :reg r9 :type uint) - (temp :reg rax) - ) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (a0 :reg carg0 :type uint) + (a1 :reg carg1 :type uint) + (a2 :reg carg2 :type uint) + (a3 :reg carg3 :type uint) + (xoff :reg exec-off :type uint) + (a4 :reg carg4 :type uint) + (a5 :reg carg5 :type uint) + (ra :reg lr :type uint)) + ;; set lr before the argument registers are live + (set! ra (the uint return-from-thread-dead)) + (.add ra xoff) + (.mov :color #f a0 s1) + (.mov :color #f a1 s2) + (.mov :color #f a2 s3) + (.mov :color #f a3 s4) + ;; branch targets go through the exec base + (.add :color #f s0 xoff) + (.jr :color #f s0) + (.add a4 a4) + (.add a5 a5))) + (#t + (rlet ((s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (a0 :reg rdi :type uint) ; ok + (a1 :reg rsi :type uint) ; ok + (a2 :reg rdx :type uint) ; ok + (a3 :reg rcx :type uint) ; ok + (off :reg r15 :type uint) + (a4 :reg r8 :type uint) + (a5 :reg r9 :type uint) + (temp :reg rax) + ) - (.mov temp return-from-thread-dead) - (.add temp off) - (.push temp) + (.mov temp return-from-thread-dead) + (.add temp off) + (.push temp) - ;; stack is 16 + 8 aligned now + ;; stack is 16 + 8 aligned now - (.mov :color #f a0 s1) - (.mov :color #f a1 s2) - (.mov :color #f a2 s3) - (.mov :color #f a3 s4) + (.mov :color #f a0 s1) + (.mov :color #f a1 s2) + (.mov :color #f a2 s3) + (.mov :color #f a3 s4) - (.add :color #f s0 off) - (.jr :color #f s0) + (.add :color #f s0 off) + (.jr :color #f s0) - (.add a4 a4) - (.add a5 a5) - ) + (.add a4 a4) + (.add a5 a5) + ) + )) ) ;; decomp deviation @@ -2499,5 +2954,3 @@ (defmacro ps (&key (detail #f)) `(inspect-process-tree *active-pool* 0 0 ,detail) ) - - diff --git a/goal_src/jak2/kernel/gstate.gc b/goal_src/jak2/kernel/gstate.gc index c00c37af62..6797cc15e9 100644 --- a/goal_src/jak2/kernel/gstate.gc +++ b/goal_src/jak2/kernel/gstate.gc @@ -432,39 +432,72 @@ It type checks the arguments for the entry function. ;; we need to: ;; - make sure that when this code returns, we do a deactivate ;; - reset the stack to the top, so we can't just call the code. - (rlet ((temp) - (func) - (sp :reg rsp :type uint) - (off :reg r15 :type uint) - (carg0 :reg rdi) - (carg1 :reg rsi) - (carg2 :reg rdx) - (carg3 :reg rcx)) - ;; prepare args - ;; compiler will likely have these on the stack, we need to get them in regs - ;; before messing with the stack. - (.mov carg0 arg0) - (.mov carg1 arg1) - (.mov carg2 arg2) - (.mov carg3 arg3) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((func) + (sp :reg sp :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (ra :reg lr :type uint) + (carg0 :reg carg0) + (carg1 :reg carg1) + (carg2 :reg carg2) + (carg3 :reg carg3)) + ;; load the arguments before discarding their stack slots + (.mov carg0 arg0) + (.mov carg1 arg1) + (.mov carg2 arg2) + (.mov carg3 arg3) + ;; convert the code pointer to the executable alias before branching + (.mov func (-> new-state code)) + (.add func xoff) + ;; reset sp through the writable alias + (.mov sp (-> pp main-thread stack-top)) + (.add sp off) + ;; return through deactivate + (.mov ra return-from-thread-dead) + (.add ra xoff) + (.jr func) + ;; keep the argument registers live until the branch + (.add carg0 carg1) + (.add carg2 carg3) + #f ;; can't get here + )) + (#t + (rlet ((temp) + (func) + (sp :reg rsp :type uint) + (off :reg r15 :type uint) + (carg0 :reg rdi) + (carg1 :reg rsi) + (carg2 :reg rdx) + (carg3 :reg rcx)) + ;; prepare args + ;; compiler will likely have these on the stack, we need to get them in regs + ;; before messing with the stack. + (.mov carg0 arg0) + (.mov carg1 arg1) + (.mov carg2 arg2) + (.mov carg3 arg3) - ;; get the main code as an x86-64 pointer - (.mov func (-> new-state code)) - (.add func off) - ;; reset the stack (scary) - (.mov sp (-> pp main-thread stack-top)) - (.add sp off) - ;; push the return trampoline for when code returns. - (.mov temp return-from-thread-dead) ;; will deactivate - (.add temp off) - (.push temp) - ;; and call! - (.jr func) - ;; stupid hack so the compiler doesn't throw away these registers. - (.add carg0 carg1) - (.add carg2 carg3) - #f ;; can't get here - ) + ;; get the main code as an x86-64 pointer + (.mov func (-> new-state code)) + (.add func off) + ;; reset the stack (scary) + (.mov sp (-> pp main-thread stack-top)) + (.add sp off) + ;; push the return trampoline for when code returns. + (.mov temp return-from-thread-dead) ;; will deactivate + (.add temp off) + (.push temp) + ;; and call! + (.jr func) + ;; stupid hack so the compiler doesn't throw away these registers. + (.add carg0 carg1) + (.add carg2 carg3) + #f ;; can't get here + ) + )) ) ) (else @@ -552,4 +585,3 @@ It type checks the arguments for the entry function. (send-event-function ,proc event-data) ) ) - diff --git a/goal_src/jak3/kernel/gkernel-h.gc b/goal_src/jak3/kernel/gkernel-h.gc index a869e439c6..547c88d577 100644 --- a/goal_src/jak3/kernel/gkernel-h.gc +++ b/goal_src/jak3/kernel/gkernel-h.gc @@ -566,8 +566,8 @@ ) (defmacro break () - "crash the game by dividing by 0." - `(/ 0 0) + "trigger an exception." + `(begin (.break) 0) ) (defmacro with-pp (&rest body) @@ -692,4 +692,4 @@ ) (defmacro suspend-for (time &rest body) - `(let ((time (current-time))) (until (time-elapsed? time ,time) ,@body (suspend)))) \ No newline at end of file + `(let ((time (current-time))) (until (time-elapsed? time ,time) ,@body (suspend)))) diff --git a/goal_src/jak3/kernel/gkernel.gc b/goal_src/jak3/kernel/gkernel.gc index ccfa176365..a2104bd763 100644 --- a/goal_src/jak3/kernel/gkernel.gc +++ b/goal_src/jak3/kernel/gkernel.gc @@ -486,30 +486,55 @@ (declare (asm-func none) ;; (print-asm) ) - (rlet ((sp :reg rsp :type uint) - (off :reg r15 :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - ) - ;; get the kernel stack pointer as a GOAL pointer (won't use a temp reg) - (.load-sym :sext #f sp *kernel-sp*) - ;; convert it back to a real pointer - (.add sp off) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + ;; keep x0 intact while restoring the kernel return address + (rlet ((sp :reg sp :type uint) + (off :reg off :type uint) + (temp :reg x9 :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint)) + ;; build sp in temp because it cannot be loaded directly + (.load-sym :sext #f temp *kernel-sp*) + (.add temp off) + (set! sp temp) + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + (.pop :color #f ra) + (.ret))) + (#t + (rlet ((sp :reg rsp :type uint) + (off :reg r15 :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + ) + ;; get the kernel stack pointer as a GOAL pointer (won't use a temp reg) + (.load-sym :sext #f sp *kernel-sp*) + ;; convert it back to a real pointer + (.add sp off) - ;; restore saved registers... - ;; without coloring system because this is "cheating" and modifying saved registers without backing up. - (.pop :color #f s4) - (.pop :color #f s3) - (.pop :color #f s2) - (.pop :color #f s1) - (.pop :color #f s0) - ;; return to the kernel function that called the user code - ;; rax should still contain the return value. - (.ret) - ) + ;; restore saved registers... + ;; without coloring system because this is "cheating" and modifying saved registers without backing up. + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + ;; return to the kernel function that called the user code + ;; rax should still contain the return value. + (.ret) + ) + )) ) (defun return-from-thread-dead () @@ -518,33 +543,59 @@ (declare (asm-func none) ;; (print-asm) ) - (rlet ((pp :reg r13 :type process) - (sp :reg rsp :type uint) - (off :reg r15 :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - ) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + ;; deactivate can clobber lr, so restore it last + (rlet ((pp :reg pp :type process) + (sp :reg sp :type uint) + (off :reg off :type uint) + (temp :reg x9 :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint)) + (deactivate pp) + (.load-sym :sext #f temp *kernel-sp*) + (.add temp off) + (set! sp temp) + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + (.pop :color #f ra) + (.ret))) + (#t + (rlet ((pp :reg r13 :type process) + (sp :reg rsp :type uint) + (off :reg r15 :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + ) - ;; first call the deactivate method. - (deactivate pp) - ;; get the kernel stack pointer as a GOAL pointer - (.load-sym :sext #f sp *kernel-sp*) - ;; convert it back to a real pointer - (.add sp off) + ;; first call the deactivate method. + (deactivate pp) + ;; get the kernel stack pointer as a GOAL pointer + (.load-sym :sext #f sp *kernel-sp*) + ;; convert it back to a real pointer + (.add sp off) - ;; restore saved registers... - ;; without coloring system because this is "cheating". - (.pop :color #f s4) - (.pop :color #f s3) - (.pop :color #f s2) - (.pop :color #f s1) - (.pop :color #f s0) - ;; return to the kernel function that called the user code - (.ret) - ) + ;; restore saved registers... + ;; without coloring system because this is "cheating". + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + ;; return to the kernel function that called the user code + (.ret) + ) + )) ) (defmacro abandon-thread () @@ -554,14 +605,22 @@ ;; but we don't because I don't have an easy way to find where to stick this. ;; I can't see how this makes a difference, as all non-main threads seem ;; temporary, but if this turns out to be false, we will need to change this. - `(rlet ((temp) - (off :reg r15 :type uint :reset-here #t)) - (.mov temp return-from-thread) ;; could probably just call this... - (.add temp off) - (.push temp) - (.ret) - ) - ) + ;; keep #cond inside the quasiquote so it remains in the macro expansion + `(#cond + ((eq? INSTRUCTION_SET 'arm64) + ;; branch to return-from-thread through the executable alias + (rlet ((temp) + (xoff :reg exec-off :type uint :reset-here #t)) + (.mov temp return-from-thread) + (.add temp xoff) + (.jr temp))) + (#t + (rlet ((temp) + (off :reg r15 :type uint :reset-here #t)) + (.mov temp return-from-thread) ;; could probably just call this... + (.add temp off) + (.push temp) + (.ret))))) (defun reset-and-call ((this thread) (func function)) "Make the given thread the top thread, reset the stack, and call the function. @@ -572,47 +631,88 @@ (declare (asm-func object) ) - (rlet ((pp :reg r13 :type process) - (sp :reg rsp :type uint) - (off :reg r15 :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (temp :reg rax :type uint) - ) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((pp :reg pp :type process) + (sp :reg sp :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (temp :reg x9 :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint)) + ;; set up the process pointer + (set! pp (-> this process)) + ;; mark the process as running and set its top thread + (set! (-> pp status) 'running) + (set! (-> pp top-thread) (the cpu-thread this)) + ;; save the link register before the other kernel registers to match the restore order + (.push :color #f ra) + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) + ;; store the kernel stack pointer as a GOAL address + (set! temp sp) + (.sub temp off) + (set! *kernel-sp* (the pointer temp)) + ;; move onto the thread's stack + (set! temp (the uint (-> this stack-top))) + (.add temp off) + (set! sp temp) + ;; put return-from-thread in the link register before calling func + (set! ra (the uint return-from-thread)) + (.add ra xoff) + ;; convert func to the executable alias before branching + (.add func xoff) + (.jr func))) + (#t + (rlet ((pp :reg r13 :type process) + (sp :reg rsp :type uint) + (off :reg r15 :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (temp :reg rax :type uint) + ) - ;; set up the process pointer - (set! pp (-> this process)) - ;; mark the process as running and set its top thread - (set! (-> pp status) 'running) - (set! (-> pp top-thread) (the cpu-thread this)) + ;; set up the process pointer + (set! pp (-> this process)) + ;; mark the process as running and set its top thread + (set! (-> pp status) 'running) + (set! (-> pp top-thread) (the cpu-thread this)) - ;; save the current kernel regs - (.push :color #f s0) - (.push :color #f s1) - (.push :color #f s2) - (.push :color #f s3) - (.push :color #f s4) + ;; save the current kernel regs + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) - ;; make rsp a GOAL pointer - (.sub sp off) - ;; and store it - (set! *kernel-sp* (the pointer sp)) ;; todo, asm form here? + ;; make rsp a GOAL pointer + (.sub sp off) + ;; and store it + (set! *kernel-sp* (the pointer sp)) - ;; setup the rsp for the new thread - (set! sp (the uint (-> this stack-top))) - (.add sp off) + ;; setup the rsp for the new thread + (set! sp (the uint (-> this stack-top))) + (.add sp off) - ;; push the return trampoline to the stack for the user code to return to - (set! temp (the uint return-from-thread)) - (.add temp off) - (.push temp) ;; stack now 16 + 8 aligned - ;; and call the function! - (.add func off) - (.jr func) - ) + ;; push the return trampoline to the stack for the user code to return to + (set! temp (the uint return-from-thread)) + (.add temp off) + (.push temp) ;; stack now 16 + 8 aligned + ;; and call the function! + (.add func off) + (.jr func) + ) + )) ) (defmethod thread-suspend ((unused cpu-thread)) @@ -623,107 +723,193 @@ ;; we begin this function with the thread object in pp. ;; not sure why we do this, maybe at one point suspending didn't clobber ;; temp registers? - (rlet ((this :reg r13 :type cpu-thread) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((this :reg pp :type cpu-thread) + (temp :reg x9 :type uint) + ;; copy sp because these ARM64 store and compare forms cannot use it as a value + (goal-sp :reg x10 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; copy the resume address from lr because blr does not push it + (set! temp ra) + ;; subtract exec-off from the resume address to store a GOAL address + (.sub temp xoff) + (set! (-> this pc) (the pointer temp)) + ;; store sp as a GOAL address + (set! goal-sp sp) + (.sub goal-sp off) + (set! (-> this sp) (the pointer goal-sp)) + ;; back up registers + (.mov :color #f temp s0) + (set! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set! (-> this rreg 4) temp) + ;; back up fprs + (.mov :color #f temp v8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp v9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp v10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp v11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp v12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp v13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp v14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp v15) + (set! (-> this freg 7) (the-as float temp)) + (let ((proc (-> this process))) + (when (> (process-stack-used proc) (-> this stack-size)) + (break) ;; stack overflow prevents suspension + ) + ;; mark the process as suspended and copy the stack + (set! (-> proc status) 'suspended) + (let ((cur (the (pointer uint64) (-> this stack-top))) + (save (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size)))) + (while (> (the int cur) (the int goal-sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! save (the (pointer uint64) (&- save 8))) + (set! (-> save) (-> cur))))) + (set! this (the cpu-thread 0)) + ;; restore the kernel context in reverse save order + (.load-sym :sext #f temp *kernel-sp*) + (.add temp off) + (set! sp temp) + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + (.pop :color #f ra) + (.ret))) + (#t + (rlet ((this :reg r13 :type cpu-thread) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr) + ) + + ;; get the return address pushed by "call" in the suspend. + (.pop temp) + ;; convert to a GOAL address + (.sub temp off) + ;; store return address in thread + (set! (-> this pc) (the pointer temp)) + + ;; convert our stack pointer to a GOAL address + (.sub sp off) + ;; store in thread. + (set! (-> this sp) (the pointer sp)) + + ;; back up registers + (.mov :color #f temp s0) + (set! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set! (-> this rreg 4) temp) + + ;; back up fprs + (.mov :color #f temp xmm8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp xmm9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp xmm10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp xmm11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp xmm12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp xmm13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp xmm14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp xmm15) + (set! (-> this freg 7) (the-as float temp)) + + ;; get our process + (let ((proc (-> this process))) + (when (> (process-stack-used proc) (-> this stack-size)) + (break) ;; too much stack has been used and we can't suspend! + ;; if you hit this, try with DEBUG_PRINT_SUSPEND_FAIL set to #t (see gkernel-h.gc) + ;; it will print more info before reaching here. ) - ;; get the return address pushed by "call" in the suspend. - (.pop temp) - ;; convert to a GOAL address - (.sub temp off) - ;; store return address in thread - (set! (-> this pc) (the pointer temp)) + ;; mark the process as suspended and copy the stack + (set! (-> proc status) 'suspended) + (let ((cur (the (pointer uint64) (-> this stack-top))) + (save (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size))) + ) + (while (> (the int cur) (the int sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! save (the (pointer uint64) (&- save 8))) + (set! (-> save) (-> cur)) + ) + ) + ) - ;; convert our stack pointer to a GOAL address - (.sub sp off) - ;; store in thread. - (set! (-> this sp) (the pointer sp)) + ;; actually setting pp to 0 + (set! this (the cpu-thread 0)) - ;; back up registers - (.mov :color #f temp s0) - (set! (-> this rreg 0) temp) - (.mov :color #f temp s1) - (set! (-> this rreg 1) temp) - (.mov :color #f temp s2) - (set! (-> this rreg 2) temp) - (.mov :color #f temp s3) - (set! (-> this rreg 3) temp) - (.mov :color #f temp s4) - (set! (-> this rreg 4) temp) + ;; get the kernel stack pointer as a GOAL pointer + (.load-sym :sext #f sp *kernel-sp*) + ;; convert it back to a real pointer + (.add sp off) - ;; back up fprs - (.mov :color #f temp xmm8) - (set! (-> this freg 0) (the-as float temp)) - (.mov :color #f temp xmm9) - (set! (-> this freg 1) (the-as float temp)) - (.mov :color #f temp xmm10) - (set! (-> this freg 2) (the-as float temp)) - (.mov :color #f temp xmm11) - (set! (-> this freg 3) (the-as float temp)) - (.mov :color #f temp xmm12) - (set! (-> this freg 4) (the-as float temp)) - (.mov :color #f temp xmm13) - (set! (-> this freg 5) (the-as float temp)) - (.mov :color #f temp xmm14) - (set! (-> this freg 6) (the-as float temp)) - (.mov :color #f temp xmm15) - (set! (-> this freg 7) (the-as float temp)) - - ;; get our process - (let ((proc (-> this process))) - (when (> (process-stack-used proc) (-> this stack-size)) - (break) ;; too much stack has been used and we can't suspend! - ;; if you hit this, try with DEBUG_PRINT_SUSPEND_FAIL set to #t (see gkernel-h.gc) - ;; it will print more info before reaching here. - ) - - ;; mark the process as suspended and copy the stack - (set! (-> proc status) 'suspended) - (let ((cur (the (pointer uint64) (-> this stack-top))) - (save (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size))) - ) - (while (> (the int cur) (the int sp)) - (set! cur (the (pointer uint64) (&- cur 8))) - (set! save (the (pointer uint64) (&- save 8))) - (set! (-> save) (-> cur)) - ) - ) - ) - - ;; actually setting pp to 0 - (set! this (the cpu-thread 0)) - - ;; get the kernel stack pointer as a GOAL pointer - (.load-sym :sext #f sp *kernel-sp*) - ;; convert it back to a real pointer - (.add sp off) - - ;; restore saved registers... - ;; without coloring system because this is "cheating". - (.pop :color #f s4) - (.pop :color #f s3) - (.pop :color #f s2) - (.pop :color #f s1) - (.pop :color #f s0) - ;; return to the kernel function that called the user code - (.ret) - ) + ;; restore saved registers... + ;; without coloring system because this is "cheating". + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + ;; return to the kernel function that called the user code + (.ret) + ) + )) (none) ) @@ -735,119 +921,215 @@ ;;(print-asm) ) - (rlet ((this :reg r13 :type cpu-thread) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (a4 :reg r8 :type uint) - (a5 :reg r9 :type uint) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((this :reg pp :type cpu-thread) + (temp :reg x9 :type uint) + ;; keep the GOAL sp in x10 because the comparison cannot use sp + (goal-sp :reg x10 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (a4 :reg carg4 :type uint) + (a5 :reg carg5 :type uint) + (temp-float :reg v0 :class fpr) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; save lr before the other kernel registers to match the restore order + (.push :color #f ra) + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) + ;; store the kernel stack pointer as a GOAL address + (set! goal-sp sp) + (.sub goal-sp off) + (set! *kernel-sp* (the pointer goal-sp)) + (set! this thread-to-resume) + ;; keep the thread stack pointer as a GOAL address until the stack copy finishes + (set! goal-sp (the uint (-> this sp))) + ;; restore the stack + (let ((cur (the (pointer uint64) (-> this stack-top))) + (restore (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size)))) + (while (> (the int cur) (the int goal-sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! restore (the (pointer uint64) (&- restore 8))) + (set! (-> cur) (-> restore)))) + ;; convert the restored thread stack pointer to a native address + (.add goal-sp off) + (set! sp goal-sp) + ;; mark the process as running and set its top thread + (set! (-> (-> this process) top-thread) this) + (set! (-> (-> this process) status) 'running) + ;; restore the saved registers + (set! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set! temp (-> this rreg 4)) + (.mov :color #f s4 temp) + (set! temp-float (-> this freg 0)) + (.mov :color #f v8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f v9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f v10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f v11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f v12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f v13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f v14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f v15 temp-float) + ;; restore rreg 5 and rreg 6 into argument registers a4 and a5 + (set! temp (-> this rreg 5)) + (.mov a4 temp) + (set! temp (-> this rreg 6)) + (.mov a5 temp) + ;; convert the resume address to the executable alias before branching + (set! temp (the uint (-> this pc))) + (.add temp xoff) + ;; load the thread process into pp + (set! this (the cpu-thread (-> this process))) + ;; the resumed frame restores lr from its stack + (.jr temp) + (.add a4 a4) + (.add a5 a5))) + (#t + (rlet ((this :reg r13 :type cpu-thread) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (a4 :reg r8 :type uint) + (a5 :reg r9 :type uint) - (temp-float :reg xmm0 :class fpr) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr) - ) - - ;; save the current kernel regs - (.push :color #f s0) - (.push :color #f s1) - (.push :color #f s2) - (.push :color #f s3) - (.push :color #f s4) - - ;; make rsp a GOAL pointer - (.sub sp off) - ;; and store it - (set! *kernel-sp* (the pointer sp)) ;; todo, asm form here? - - ;; temp, stash thread in process-pointer - (set! this thread-to-resume) - - ;; set stack pointer for the thread. leave it as a GOAL pointer for now.. - (set! sp (the uint (-> this sp))) - - ;; restore the stack (sp is a GOAL pointer) - (let ((cur (the (pointer uint64) (-> this stack-top))) - (restore (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size))) + (temp-float :reg xmm0 :class fpr) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr) ) - (while (> (the int cur) (the int sp)) - (set! cur (the (pointer uint64) (&- cur 8))) - (set! restore (the (pointer uint64) (&- restore 8))) - (set! (-> cur) (-> restore)) - ) - ) - ;; offset sp after we're done using it as a GOAL pointer. - (.add sp off) + ;; save the current kernel regs + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) - ;; setup process - (set! (-> (-> this process) top-thread) this) - (set! (-> (-> this process) status) 'running) + ;; make rsp a GOAL pointer + (.sub sp off) + ;; and store it + (set! *kernel-sp* (the pointer sp)) - ;; restore reg - (set! temp (-> this rreg 0)) - (.mov :color #f s0 temp) - (set! temp (-> this rreg 1)) - (.mov :color #f s1 temp) - (set! temp (-> this rreg 2)) - (.mov :color #f s2 temp) - (set! temp (-> this rreg 3)) - (.mov :color #f s3 temp) - (set! temp (-> this rreg 4)) - (.mov :color #f s4 temp) - (set! temp-float (-> this freg 0)) - (.mov :color #f xmm8 temp-float) - (set! temp-float (-> this freg 1)) - (.mov :color #f xmm9 temp-float) - (set! temp-float (-> this freg 2)) - (.mov :color #f xmm10 temp-float) - (set! temp-float (-> this freg 3)) - (.mov :color #f xmm11 temp-float) - (set! temp-float (-> this freg 4)) - (.mov :color #f xmm12 temp-float) - (set! temp-float (-> this freg 5)) - (.mov :color #f xmm13 temp-float) - (set! temp-float (-> this freg 6)) - (.mov :color #f xmm14 temp-float) - (set! temp-float (-> this freg 7)) - (.mov :color #f xmm15 temp-float) + ;; temp, stash thread in process-pointer + (set! this thread-to-resume) - ;; hack for set-to-run-bootstrap. The set-to-run-bootstrap in MIPS - ;; expects to receive 7 values from the cpu thread's rregs. - ;; usually rreg holds saved registers, but on the first resume after - ;; a set-to-run, they hold arguments, and set-to-run-bootstrap copies them. + ;; set stack pointer for the thread. leave it as a GOAL pointer for now.. + (set! sp (the uint (-> this sp))) - ;; We only have 5 saved regs, so we need to cheat and directly pass - ;; two values in other registers - ;; so we load the a4/a5 argument registers with rreg 5 and rreg 6 - ;; In the case where we are doing a normal resume, the - ;; compiler should assume that these registers are overwritten anyway. - (set! temp (-> this rreg 5)) - (.mov a4 temp) - (set! temp (-> this rreg 6)) - (.mov a5 temp) + ;; restore the stack (sp is a GOAL pointer) + (let ((cur (the (pointer uint64) (-> this stack-top))) + (restore (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size))) + ) + (while (> (the int cur) (the int sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! restore (the (pointer uint64) (&- restore 8))) + (set! (-> cur) (-> restore)) + ) + ) - ;; get the resume address - (set! temp (the uint (-> this pc))) - (.add temp off) + ;; offset sp after we're done using it as a GOAL pointer. + (.add sp off) - ;; setup the process - (set! this (the cpu-thread (-> this process))) - ;; resume! - (.jr temp) - (.add a4 a4) - (.add a5 a5) - ) + ;; setup process + (set! (-> (-> this process) top-thread) this) + (set! (-> (-> this process) status) 'running) + + ;; restore reg + (set! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set! temp (-> this rreg 4)) + (.mov :color #f s4 temp) + (set! temp-float (-> this freg 0)) + (.mov :color #f xmm8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f xmm9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f xmm10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f xmm11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f xmm12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f xmm13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f xmm14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f xmm15 temp-float) + + ;; hack for set-to-run-bootstrap. The set-to-run-bootstrap in MIPS + ;; expects to receive 7 values from the cpu thread's rregs. + ;; usually rreg holds saved registers, but on the first resume after + ;; a set-to-run, they hold arguments, and set-to-run-bootstrap copies them. + + ;; We only have 5 saved regs, so we need to cheat and directly pass + ;; two values in other registers + ;; so we load the a4/a5 argument registers with rreg 5 and rreg 6 + ;; In the case where we are doing a normal resume, the + ;; compiler should assume that these registers are overwritten anyway. + (set! temp (-> this rreg 5)) + (.mov a4 temp) + (set! temp (-> this rreg 6)) + (.mov a5 temp) + + ;; get the resume address + (set! temp (the uint (-> this pc))) + (.add temp off) + + ;; setup the process + (set! this (the cpu-thread (-> this process))) + ;; resume! + (.jr temp) + (.add a4 a4) + (.add a5 a5) + ) + )) (none) ) @@ -1699,104 +1981,183 @@ (allow-saved-regs) ;; very dangerous! ) - (rlet ((pp :reg r13 :type process) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type (pointer uint64)) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((pp :reg pp :type process) + (temp :reg x9 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type (pointer uint64)) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; treat allocation as an address + (let ((this (the catch-frame (&+ (the pointer allocation) *gtype-basic-offset*)))) + ;; initialize the catch frame + (set! (-> this type) type-to-make) + (set! (-> this name) name) + ;; subtract exec-off from lr to store the return address as a GOAL address + (set! temp ra) + (.sub temp xoff) + (set! (-> this ra) (the int temp)) + ;; copy sp through temp because str cannot use sp as data + (set! temp sp) + (.sub temp off) + (set! (-> this sp) (the int temp)) + ;; save the thread registers in the catch frame + (.mov :color #f temp s0) + (set-u128-as-u64! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set-u128-as-u64! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set-u128-as-u64! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set-u128-as-u64! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set-u128-as-u64! (-> this rreg 4) temp) + (.mov :color #f temp v8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp v9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp v10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp v11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp v12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp v13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp v14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp v15) + (set! (-> this freg 7) (the-as float temp)) + ;; add the catch frame to the process frame stack + (set! (-> this next) (-> pp stack-frame-top)) + (set! (-> pp stack-frame-top) this) + ;; save lr before the call clobbers it + (.push :color #f ra) + ;; save s3 and s2 before using them as call temporaries + (.push :color #f s3) + (.push :color #f s2) + (set! s3 (the uint func)) + (set! s2 param-block) + (let ((ret ((the-super-u64-fucntion s3) (-> s2 0) (-> s2 1) (-> s2 2) (-> s2 3) (-> s2 4) (-> s2 5)))) + (.pop :color #f s2) + (.pop :color #f s3) + (.pop :color #f ra) + (set! (-> pp stack-frame-top) (-> pp stack-frame-top next)) + (.ret) + (the object ret))))) + (#t + (rlet ((pp :reg r13 :type process) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type (pointer uint64)) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr) + ) + + ;; we treat the allocation as an address. + (let ((this (the catch-frame (&+ (the pointer allocation) *gtype-basic-offset*)))) + ;; setup catch frame + (set! (-> this type) type-to-make) + (set! (-> this name) name) + ;; get the return address (the compiler won't touch the stack because we're an asm-func) + (.pop temp) + (.push temp) + ;; make it a GOAL address so it fits in 32 bits + (.sub temp off) + ;; store it + (set! (-> this ra) (the int temp)) + + ;; store the stack pointer as a GOAL address + (set! temp sp) + (.sub temp off) + (set! (-> this sp) (the int temp)) + + ;; back up registers we care about + (.mov :color #f temp s0) + (set-u128-as-u64! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set-u128-as-u64! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set-u128-as-u64! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set-u128-as-u64! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set-u128-as-u64! (-> this rreg 4) temp) + + (.mov :color #f temp xmm8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp xmm9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp xmm10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp xmm11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp xmm12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp xmm13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp xmm14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp xmm15) + (set! (-> this freg 7) (the-as float temp)) + + ;; push this stack frame + (set! (-> this next) (-> pp stack-frame-top)) + (set! (-> pp stack-frame-top) this) + + ;; help coloring, it isn't smart enough to realize it's "safe" to use these registers. + (.push :color #f s3) + (.push :color #f s2) + ;; keep the stack aligned for the call + (.push :color #f s2) + (set! s3 (the uint func)) + (set! s2 param-block) + + (let ((ret ((the-super-u64-fucntion s3) + (-> s2 0) + (-> s2 1) + (-> s2 2) + (-> s2 3) + (-> s2 4) + (-> s2 5) + )) + ) + (.pop :color #f s2) + (.pop :color #f s2) + (.pop :color #f s3) + (set! (-> pp stack-frame-top) (-> pp stack-frame-top next)) + (.ret) + (the object ret) ) - - ;; we treat the allocation as an address. - (let ((this (the catch-frame (&+ (the pointer allocation) *gtype-basic-offset*)))) - ;; setup catch frame - (set! (-> this type) type-to-make) - (set! (-> this name) name) - ;; get the return address (the compiler won't touch the stack because we're an asm-func) - (.pop temp) - (.push temp) - ;; make it a GOAL address so it fits in 32 bits - (.sub temp off) - ;; store it - (set! (-> this ra) (the int temp)) - - ;; todo, do we need a stack offset here? - ;; remember the stack pointer - (set! temp sp) - (.sub temp off) - (set! (-> this sp) (the int temp)) - - ;; back up registers we care about - (.mov :color #f temp s0) - (set-u128-as-u64! (-> this rreg 0) temp) - (.mov :color #f temp s1) - (set-u128-as-u64! (-> this rreg 1) temp) - (.mov :color #f temp s2) - (set-u128-as-u64! (-> this rreg 2) temp) - (.mov :color #f temp s3) - (set-u128-as-u64! (-> this rreg 3) temp) - (.mov :color #f temp s4) - (set-u128-as-u64! (-> this rreg 4) temp) - - (.mov :color #f temp xmm8) - (set! (-> this freg 0) (the-as float temp)) - (.mov :color #f temp xmm9) - (set! (-> this freg 1) (the-as float temp)) - (.mov :color #f temp xmm10) - (set! (-> this freg 2) (the-as float temp)) - (.mov :color #f temp xmm11) - (set! (-> this freg 3) (the-as float temp)) - (.mov :color #f temp xmm12) - (set! (-> this freg 4) (the-as float temp)) - (.mov :color #f temp xmm13) - (set! (-> this freg 5) (the-as float temp)) - (.mov :color #f temp xmm14) - (set! (-> this freg 6) (the-as float temp)) - (.mov :color #f temp xmm15) - (set! (-> this freg 7) (the-as float temp)) - - ;; push this stack frame - (set! (-> this next) (-> pp stack-frame-top)) - (set! (-> pp stack-frame-top) this) - - ;; help coloring, it isn't smart enough to realize it's "safe" to use these registers. - (.push :color #f s3) - (.push :color #f s2) - (.push :color #f s2) - (set! s3 (the uint func)) - (set! s2 param-block) - - ;; todo - are we aligned correctly here? - (let ((ret ((the-super-u64-fucntion s3) - (-> s2 0) - (-> s2 1) - (-> s2 2) - (-> s2 3) - (-> s2 4) - (-> s2 5) - )) - ) - (.pop :color #f s2) - (.pop :color #f s2) - (.pop :color #f s3) - (set! (-> pp stack-frame-top) (-> pp stack-frame-top next)) - (.ret) - (the object ret) - ) - ) - ) + ) + ) + )) ) (defun throw-dispatch ((this catch-frame) value) @@ -1804,73 +2165,138 @@ Only can throw a 64-bit value. The original could throw 128 bits." (declare (asm-func none)) - (rlet ((pp :reg r13 :type process) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type (pointer uint64)) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((pp :reg pp :type process) + (temp :reg x9 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + ;; reuse x0 for retval after loading the catch frame return address + (retval :reg carg0 :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type (pointer uint64)) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (temp-float :reg v0 :class fpr) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; discard catch frames through this frame + (set! (-> pp stack-frame-top) (-> this next)) + ;; restore the saved registers + (set-u64-from-u128! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set-u64-from-u128! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set-u64-from-u128! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set-u64-from-u128! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set-u64-from-u128! temp (-> this rreg 4)) + (.mov :color #f s4 temp) + (set! temp-float (-> this freg 0)) + (.mov :color #f v8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f v9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f v10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f v11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f v12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f v13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f v14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f v15 temp-float) + ;; rebuild sp through temp because ldr cannot target sp + (set! temp (the uint (-> this sp))) + (.add temp off) + (set! sp temp) + ;; add exec-off to restore the catch frame return address in lr + (set! ra (the uint (-> this ra))) + (.add ra xoff) + ;; load the return register + (.mov retval value) + (.ret))) + (#t + (rlet ((pp :reg r13 :type process) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type (pointer uint64)) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) - (temp-float :reg xmm0 :class fpr) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr) - ) + (temp-float :reg xmm0 :class fpr) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr) + ) - ;; pop everything we threw past - (set! (-> pp stack-frame-top) (-> this next)) + ;; pop everything we threw past + (set! (-> pp stack-frame-top) (-> this next)) - ;; restore regs we care about. - (set-u64-from-u128! temp (-> this rreg 0)) - (.mov :color #f s0 temp) - (set-u64-from-u128! temp (-> this rreg 1)) - (.mov :color #f s1 temp) - (set-u64-from-u128! temp (-> this rreg 2)) - (.mov :color #f s2 temp) - (set-u64-from-u128! temp (-> this rreg 3)) - (.mov :color #f s3 temp) - (set-u64-from-u128! temp (-> this rreg 4)) - (.mov :color #f s4 temp) + ;; restore regs we care about. + (set-u64-from-u128! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set-u64-from-u128! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set-u64-from-u128! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set-u64-from-u128! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set-u64-from-u128! temp (-> this rreg 4)) + (.mov :color #f s4 temp) - (set! temp-float (-> this freg 0)) - (.mov :color #f xmm8 temp-float) - (set! temp-float (-> this freg 1)) - (.mov :color #f xmm9 temp-float) - (set! temp-float (-> this freg 2)) - (.mov :color #f xmm10 temp-float) - (set! temp-float (-> this freg 3)) - (.mov :color #f xmm11 temp-float) - (set! temp-float (-> this freg 4)) - (.mov :color #f xmm12 temp-float) - (set! temp-float (-> this freg 5)) - (.mov :color #f xmm13 temp-float) - (set! temp-float (-> this freg 6)) - (.mov :color #f xmm14 temp-float) - (set! temp-float (-> this freg 7)) - (.mov :color #f xmm15 temp-float) + (set! temp-float (-> this freg 0)) + (.mov :color #f xmm8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f xmm9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f xmm10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f xmm11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f xmm12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f xmm13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f xmm14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f xmm15 temp-float) - ;; set stack pointer - (set! sp (the uint (-> this sp))) - (.add sp off) + ;; set stack pointer + (set! sp (the uint (-> this sp))) + (.add sp off) - ;; overwrite our return address - (.pop temp) - (set! temp (the uint (-> this ra))) - (.add temp off) - (.push temp) + ;; overwrite our return address + (.pop temp) + (set! temp (the uint (-> this ra))) + (.add temp off) + (.push temp) - ;; load the return register - (.mov temp value) - (.ret) - ) + ;; load the return register + (.mov temp value) + (.ret) + ) + )) ) (defun throw ((name symbol) value) @@ -2172,39 +2598,68 @@ ;;(print-asm) ) - (rlet ((s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (a0 :reg rdi :type uint) ; ok - (a1 :reg rsi :type uint) ; ok - (a2 :reg rdx :type uint) ; ok - (a3 :reg rcx :type uint) ; ok - (off :reg r15 :type uint) - (a4 :reg r8 :type uint) - (a5 :reg r9 :type uint) - (temp :reg rax) - ) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (a0 :reg carg0 :type uint) + (a1 :reg carg1 :type uint) + (a2 :reg carg2 :type uint) + (a3 :reg carg3 :type uint) + (xoff :reg exec-off :type uint) + (a4 :reg carg4 :type uint) + (a5 :reg carg5 :type uint) + (ra :reg lr :type uint)) + ;; set lr before the argument registers are live + (set! ra (the uint return-from-thread-dead)) + (.add ra xoff) + (.mov :color #f a0 s1) + (.mov :color #f a1 s2) + (.mov :color #f a2 s3) + (.mov :color #f a3 s4) + ;; branch targets go through the exec base + (.add :color #f s0 xoff) + (.jr :color #f s0) + (.add a4 a4) + (.add a5 a5))) + (#t + (rlet ((s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (a0 :reg rdi :type uint) ; ok + (a1 :reg rsi :type uint) ; ok + (a2 :reg rdx :type uint) ; ok + (a3 :reg rcx :type uint) ; ok + (off :reg r15 :type uint) + (a4 :reg r8 :type uint) + (a5 :reg r9 :type uint) + (temp :reg rax) + ) - (.mov temp return-from-thread-dead) - (.add temp off) - (.push temp) + (.mov temp return-from-thread-dead) + (.add temp off) + (.push temp) - ;; stack is 16 + 8 aligned now + ;; stack is 16 + 8 aligned now - (.mov :color #f a0 s1) - (.mov :color #f a1 s2) - (.mov :color #f a2 s3) - (.mov :color #f a3 s4) + (.mov :color #f a0 s1) + (.mov :color #f a1 s2) + (.mov :color #f a2 s3) + (.mov :color #f a3 s4) - (.add :color #f s0 off) - (.jr :color #f s0) + (.add :color #f s0 off) + (.jr :color #f s0) - (.add a4 a4) - (.add a5 a5) - ) + (.add a4 a4) + (.add a5 a5) + ) + )) ) ;; definition for function set-to-run diff --git a/goal_src/jak3/kernel/gstate.gc b/goal_src/jak3/kernel/gstate.gc index f2b0e6c027..51c61b4907 100644 --- a/goal_src/jak3/kernel/gstate.gc +++ b/goal_src/jak3/kernel/gstate.gc @@ -385,39 +385,72 @@ (t9-5) ) ) - (rlet ((temp) - (func) - (sp :reg rsp :type uint) - (off :reg r15 :type uint) - (carg0 :reg rdi) - (carg1 :reg rsi) - (carg2 :reg rdx) - (carg3 :reg rcx)) - ;; prepare args - ;; compiler will likely have these on the stack, we need to get them in regs - ;; before messing with the stack. - (.mov carg0 arg0) - (.mov carg1 arg1) - (.mov carg2 arg2) - (.mov carg3 arg3) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((func) + (sp :reg sp :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (ra :reg lr :type uint) + (carg0 :reg carg0) + (carg1 :reg carg1) + (carg2 :reg carg2) + (carg3 :reg carg3)) + ;; load the arguments before discarding their stack slots + (.mov carg0 arg0) + (.mov carg1 arg1) + (.mov carg2 arg2) + (.mov carg3 arg3) + ;; convert the code pointer to the executable alias before branching + (.mov func (-> s0-2 code)) + (.add func xoff) + ;; reset sp through the writable alias + (.mov sp (-> pp main-thread stack-top)) + (.add sp off) + ;; return through deactivate + (.mov ra return-from-thread-dead) + (.add ra xoff) + (.jr func) + ;; keep the argument registers live until the branch + (.add carg0 carg1) + (.add carg2 carg3) + #f ;; can't get here + )) + (#t + (rlet ((temp) + (func) + (sp :reg rsp :type uint) + (off :reg r15 :type uint) + (carg0 :reg rdi) + (carg1 :reg rsi) + (carg2 :reg rdx) + (carg3 :reg rcx)) + ;; prepare args + ;; compiler will likely have these on the stack, we need to get them in regs + ;; before messing with the stack. + (.mov carg0 arg0) + (.mov carg1 arg1) + (.mov carg2 arg2) + (.mov carg3 arg3) - ;; get the main code as an x86-64 pointer - (.mov func (-> s0-2 code)) - (.add func off) - ;; reset the stack (scary) - (.mov sp (-> pp main-thread stack-top)) - (.add sp off) - ;; push the return trampoline for when code returns. - (.mov temp return-from-thread-dead) ;; will deactivate - (.add temp off) - (.push temp) - ;; and call! - (.jr func) - ;; stupid hack so the compiler doesn't throw away these registers. - (.add carg0 carg1) - (.add carg2 carg3) - #f ;; can't get here - ) + ;; get the main code as an x86-64 pointer + (.mov func (-> s0-2 code)) + (.add func off) + ;; reset the stack (scary) + (.mov sp (-> pp main-thread stack-top)) + (.add sp off) + ;; push the return trampoline for when code returns. + (.mov temp return-from-thread-dead) ;; will deactivate + (.add temp off) + (.push temp) + ;; and call! + (.jr func) + ;; stupid hack so the compiler doesn't throw away these registers. + (.add carg0 carg1) + (.add carg2 carg3) + #f ;; can't get here + ) + )) ; (let ((v1-29 (-> pp main-thread))) ; (.lwu sp-0 28 v1-29) ; ) @@ -510,4 +543,3 @@ (send-event-function ,proc event-data) ) ) - diff --git a/goal_src/jakx/kernel/gkernel-h.gc b/goal_src/jakx/kernel/gkernel-h.gc index 589ab72cc0..c758160ccf 100644 --- a/goal_src/jakx/kernel/gkernel-h.gc +++ b/goal_src/jakx/kernel/gkernel-h.gc @@ -581,8 +581,8 @@ ) (defmacro break () - "crash the game by dividing by 0." - `(/ 0 0) + "trigger an exception." + `(begin (.break) 0) ) (defmacro with-pp (&rest body) @@ -707,4 +707,4 @@ ) (defmacro suspend-for (time &rest body) - `(let ((time (current-time))) (until (time-elapsed? time ,time) ,@body (suspend)))) \ No newline at end of file + `(let ((time (current-time))) (until (time-elapsed? time ,time) ,@body (suspend)))) diff --git a/goal_src/jakx/kernel/gkernel.gc b/goal_src/jakx/kernel/gkernel.gc index 14a3e5ca7d..930100e86c 100644 --- a/goal_src/jakx/kernel/gkernel.gc +++ b/goal_src/jakx/kernel/gkernel.gc @@ -488,30 +488,55 @@ (declare (asm-func none) ;; (print-asm) ) - (rlet ((sp :reg rsp :type uint) - (off :reg r15 :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - ) - ;; get the kernel stack pointer as a GOAL pointer (won't use a temp reg) - (.load-sym :sext #f sp *kernel-sp*) - ;; convert it back to a real pointer - (.add sp off) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + ;; keep x0 intact while restoring the kernel return address + (rlet ((sp :reg sp :type uint) + (off :reg off :type uint) + (temp :reg x9 :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint)) + ;; build sp in temp because it cannot be loaded directly + (.load-sym :sext #f temp *kernel-sp*) + (.add temp off) + (set! sp temp) + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + (.pop :color #f ra) + (.ret))) + (#t + (rlet ((sp :reg rsp :type uint) + (off :reg r15 :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + ) + ;; get the kernel stack pointer as a GOAL pointer (won't use a temp reg) + (.load-sym :sext #f sp *kernel-sp*) + ;; convert it back to a real pointer + (.add sp off) - ;; restore saved registers... - ;; without coloring system because this is "cheating" and modifying saved registers without backing up. - (.pop :color #f s4) - (.pop :color #f s3) - (.pop :color #f s2) - (.pop :color #f s1) - (.pop :color #f s0) - ;; return to the kernel function that called the user code - ;; rax should still contain the return value. - (.ret) - ) + ;; restore saved registers... + ;; without coloring system because this is "cheating" and modifying saved registers without backing up. + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + ;; return to the kernel function that called the user code + ;; rax should still contain the return value. + (.ret) + ) + )) ) (defun return-from-thread-dead () @@ -520,33 +545,59 @@ (declare (asm-func none) ;; (print-asm) ) - (rlet ((pp :reg r13 :type process) - (sp :reg rsp :type uint) - (off :reg r15 :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - ) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + ;; deactivate can clobber lr, so restore it last + (rlet ((pp :reg pp :type process) + (sp :reg sp :type uint) + (off :reg off :type uint) + (temp :reg x9 :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint)) + (deactivate pp) + (.load-sym :sext #f temp *kernel-sp*) + (.add temp off) + (set! sp temp) + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + (.pop :color #f ra) + (.ret))) + (#t + (rlet ((pp :reg r13 :type process) + (sp :reg rsp :type uint) + (off :reg r15 :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + ) - ;; first call the deactivate method. - (deactivate pp) - ;; get the kernel stack pointer as a GOAL pointer - (.load-sym :sext #f sp *kernel-sp*) - ;; convert it back to a real pointer - (.add sp off) + ;; first call the deactivate method. + (deactivate pp) + ;; get the kernel stack pointer as a GOAL pointer + (.load-sym :sext #f sp *kernel-sp*) + ;; convert it back to a real pointer + (.add sp off) - ;; restore saved registers... - ;; without coloring system because this is "cheating". - (.pop :color #f s4) - (.pop :color #f s3) - (.pop :color #f s2) - (.pop :color #f s1) - (.pop :color #f s0) - ;; return to the kernel function that called the user code - (.ret) - ) + ;; restore saved registers... + ;; without coloring system because this is "cheating". + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + ;; return to the kernel function that called the user code + (.ret) + ) + )) ) (defmacro abandon-thread () @@ -556,14 +607,22 @@ ;; but we don't because I don't have an easy way to find where to stick this. ;; I can't see how this makes a difference, as all non-main threads seem ;; temporary, but if this turns out to be false, we will need to change this. - `(rlet ((temp) - (off :reg r15 :type uint :reset-here #t)) - (.mov temp return-from-thread) ;; could probably just call this... - (.add temp off) - (.push temp) - (.ret) - ) - ) + ;; keep #cond inside the quasiquote so it remains in the macro expansion + `(#cond + ((eq? INSTRUCTION_SET 'arm64) + ;; branch to return-from-thread through the executable alias + (rlet ((temp) + (xoff :reg exec-off :type uint :reset-here #t)) + (.mov temp return-from-thread) + (.add temp xoff) + (.jr temp))) + (#t + (rlet ((temp) + (off :reg r15 :type uint :reset-here #t)) + (.mov temp return-from-thread) ;; could probably just call this... + (.add temp off) + (.push temp) + (.ret))))) (defun reset-and-call ((this thread) (func function)) "Make the given thread the top thread, reset the stack, and call the function. @@ -574,47 +633,88 @@ (declare (asm-func object) ) - (rlet ((pp :reg r13 :type process) - (sp :reg rsp :type uint) - (off :reg r15 :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (temp :reg rax :type uint) - ) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((pp :reg pp :type process) + (sp :reg sp :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (temp :reg x9 :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint)) + ;; set up the process pointer + (set! pp (-> this process)) + ;; mark the process as running and set its top thread + (set! (-> pp status) 'running) + (set! (-> pp top-thread) (the cpu-thread this)) + ;; save the link register before the other kernel registers to match the restore order + (.push :color #f ra) + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) + ;; store the kernel stack pointer as a GOAL address + (set! temp sp) + (.sub temp off) + (set! *kernel-sp* (the pointer temp)) + ;; move onto the thread's stack + (set! temp (the uint (-> this stack-top))) + (.add temp off) + (set! sp temp) + ;; put return-from-thread in the link register before calling func + (set! ra (the uint return-from-thread)) + (.add ra xoff) + ;; convert func to the executable alias before branching + (.add func xoff) + (.jr func))) + (#t + (rlet ((pp :reg r13 :type process) + (sp :reg rsp :type uint) + (off :reg r15 :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (temp :reg rax :type uint) + ) - ;; set up the process pointer - (set! pp (-> this process)) - ;; mark the process as running and set its top thread - (set! (-> pp status) 'running) - (set! (-> pp top-thread) (the cpu-thread this)) + ;; set up the process pointer + (set! pp (-> this process)) + ;; mark the process as running and set its top thread + (set! (-> pp status) 'running) + (set! (-> pp top-thread) (the cpu-thread this)) - ;; save the current kernel regs - (.push :color #f s0) - (.push :color #f s1) - (.push :color #f s2) - (.push :color #f s3) - (.push :color #f s4) + ;; save the current kernel regs + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) - ;; make rsp a GOAL pointer - (.sub sp off) - ;; and store it - (set! *kernel-sp* (the pointer sp)) ;; todo, asm form here? + ;; make rsp a GOAL pointer + (.sub sp off) + ;; and store it + (set! *kernel-sp* (the pointer sp)) - ;; setup the rsp for the new thread - (set! sp (the uint (-> this stack-top))) - (.add sp off) + ;; setup the rsp for the new thread + (set! sp (the uint (-> this stack-top))) + (.add sp off) - ;; push the return trampoline to the stack for the user code to return to - (set! temp (the uint return-from-thread)) - (.add temp off) - (.push temp) ;; stack now 16 + 8 aligned - ;; and call the function! - (.add func off) - (.jr func) - ) + ;; push the return trampoline to the stack for the user code to return to + (set! temp (the uint return-from-thread)) + (.add temp off) + (.push temp) ;; stack now 16 + 8 aligned + ;; and call the function! + (.add func off) + (.jr func) + ) + )) ) (defmethod thread-suspend ((unused cpu-thread)) @@ -625,107 +725,193 @@ ;; we begin this function with the thread object in pp. ;; not sure why we do this, maybe at one point suspending didn't clobber ;; temp registers? - (rlet ((this :reg r13 :type cpu-thread) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((this :reg pp :type cpu-thread) + (temp :reg x9 :type uint) + ;; copy sp because these ARM64 store and compare forms cannot use it as a value + (goal-sp :reg x10 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; copy the resume address from lr because blr does not push it + (set! temp ra) + ;; subtract exec-off from the resume address to store a GOAL address + (.sub temp xoff) + (set! (-> this pc) (the pointer temp)) + ;; store sp as a GOAL address + (set! goal-sp sp) + (.sub goal-sp off) + (set! (-> this sp) (the pointer goal-sp)) + ;; back up registers + (.mov :color #f temp s0) + (set! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set! (-> this rreg 4) temp) + ;; back up fprs + (.mov :color #f temp v8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp v9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp v10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp v11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp v12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp v13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp v14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp v15) + (set! (-> this freg 7) (the-as float temp)) + (let ((proc (-> this process))) + (when (> (process-stack-used proc) (-> this stack-size)) + (break) ;; stack overflow prevents suspension + ) + ;; mark the process as suspended and copy the stack + (set! (-> proc status) 'suspended) + (let ((cur (the (pointer uint64) (-> this stack-top))) + (save (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size)))) + (while (> (the int cur) (the int goal-sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! save (the (pointer uint64) (&- save 8))) + (set! (-> save) (-> cur))))) + (set! this (the cpu-thread 0)) + ;; restore the kernel context in reverse save order + (.load-sym :sext #f temp *kernel-sp*) + (.add temp off) + (set! sp temp) + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + (.pop :color #f ra) + (.ret))) + (#t + (rlet ((this :reg r13 :type cpu-thread) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr) + ) + + ;; get the return address pushed by "call" in the suspend. + (.pop temp) + ;; convert to a GOAL address + (.sub temp off) + ;; store return address in thread + (set! (-> this pc) (the pointer temp)) + + ;; convert our stack pointer to a GOAL address + (.sub sp off) + ;; store in thread. + (set! (-> this sp) (the pointer sp)) + + ;; back up registers + (.mov :color #f temp s0) + (set! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set! (-> this rreg 4) temp) + + ;; back up fprs + (.mov :color #f temp xmm8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp xmm9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp xmm10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp xmm11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp xmm12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp xmm13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp xmm14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp xmm15) + (set! (-> this freg 7) (the-as float temp)) + + ;; get our process + (let ((proc (-> this process))) + (when (> (process-stack-used proc) (-> this stack-size)) + (break) ;; too much stack has been used and we can't suspend! + ;; if you hit this, try with DEBUG_PRINT_SUSPEND_FAIL set to #t (see gkernel-h.gc) + ;; it will print more info before reaching here. ) - ;; get the return address pushed by "call" in the suspend. - (.pop temp) - ;; convert to a GOAL address - (.sub temp off) - ;; store return address in thread - (set! (-> this pc) (the pointer temp)) + ;; mark the process as suspended and copy the stack + (set! (-> proc status) 'suspended) + (let ((cur (the (pointer uint64) (-> this stack-top))) + (save (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size))) + ) + (while (> (the int cur) (the int sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! save (the (pointer uint64) (&- save 8))) + (set! (-> save) (-> cur)) + ) + ) + ) - ;; convert our stack pointer to a GOAL address - (.sub sp off) - ;; store in thread. - (set! (-> this sp) (the pointer sp)) + ;; actually setting pp to 0 + (set! this (the cpu-thread 0)) - ;; back up registers - (.mov :color #f temp s0) - (set! (-> this rreg 0) temp) - (.mov :color #f temp s1) - (set! (-> this rreg 1) temp) - (.mov :color #f temp s2) - (set! (-> this rreg 2) temp) - (.mov :color #f temp s3) - (set! (-> this rreg 3) temp) - (.mov :color #f temp s4) - (set! (-> this rreg 4) temp) + ;; get the kernel stack pointer as a GOAL pointer + (.load-sym :sext #f sp *kernel-sp*) + ;; convert it back to a real pointer + (.add sp off) - ;; back up fprs - (.mov :color #f temp xmm8) - (set! (-> this freg 0) (the-as float temp)) - (.mov :color #f temp xmm9) - (set! (-> this freg 1) (the-as float temp)) - (.mov :color #f temp xmm10) - (set! (-> this freg 2) (the-as float temp)) - (.mov :color #f temp xmm11) - (set! (-> this freg 3) (the-as float temp)) - (.mov :color #f temp xmm12) - (set! (-> this freg 4) (the-as float temp)) - (.mov :color #f temp xmm13) - (set! (-> this freg 5) (the-as float temp)) - (.mov :color #f temp xmm14) - (set! (-> this freg 6) (the-as float temp)) - (.mov :color #f temp xmm15) - (set! (-> this freg 7) (the-as float temp)) - - ;; get our process - (let ((proc (-> this process))) - (when (> (process-stack-used proc) (-> this stack-size)) - (break) ;; too much stack has been used and we can't suspend! - ;; if you hit this, try with DEBUG_PRINT_SUSPEND_FAIL set to #t (see gkernel-h.gc) - ;; it will print more info before reaching here. - ) - - ;; mark the process as suspended and copy the stack - (set! (-> proc status) 'suspended) - (let ((cur (the (pointer uint64) (-> this stack-top))) - (save (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size))) - ) - (while (> (the int cur) (the int sp)) - (set! cur (the (pointer uint64) (&- cur 8))) - (set! save (the (pointer uint64) (&- save 8))) - (set! (-> save) (-> cur)) - ) - ) - ) - - ;; actually setting pp to 0 - (set! this (the cpu-thread 0)) - - ;; get the kernel stack pointer as a GOAL pointer - (.load-sym :sext #f sp *kernel-sp*) - ;; convert it back to a real pointer - (.add sp off) - - ;; restore saved registers... - ;; without coloring system because this is "cheating". - (.pop :color #f s4) - (.pop :color #f s3) - (.pop :color #f s2) - (.pop :color #f s1) - (.pop :color #f s0) - ;; return to the kernel function that called the user code - (.ret) - ) + ;; restore saved registers... + ;; without coloring system because this is "cheating". + (.pop :color #f s4) + (.pop :color #f s3) + (.pop :color #f s2) + (.pop :color #f s1) + (.pop :color #f s0) + ;; return to the kernel function that called the user code + (.ret) + ) + )) (none) ) @@ -737,119 +923,215 @@ ;;(print-asm) ) - (rlet ((this :reg r13 :type cpu-thread) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (a4 :reg r8 :type uint) - (a5 :reg r9 :type uint) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((this :reg pp :type cpu-thread) + (temp :reg x9 :type uint) + ;; keep the GOAL sp in x10 because the comparison cannot use sp + (goal-sp :reg x10 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (a4 :reg carg4 :type uint) + (a5 :reg carg5 :type uint) + (temp-float :reg v0 :class fpr) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; save lr before the other kernel registers to match the restore order + (.push :color #f ra) + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) + ;; store the kernel stack pointer as a GOAL address + (set! goal-sp sp) + (.sub goal-sp off) + (set! *kernel-sp* (the pointer goal-sp)) + (set! this thread-to-resume) + ;; keep the thread stack pointer as a GOAL address until the stack copy finishes + (set! goal-sp (the uint (-> this sp))) + ;; restore the stack + (let ((cur (the (pointer uint64) (-> this stack-top))) + (restore (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size)))) + (while (> (the int cur) (the int goal-sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! restore (the (pointer uint64) (&- restore 8))) + (set! (-> cur) (-> restore)))) + ;; convert the restored thread stack pointer to a native address + (.add goal-sp off) + (set! sp goal-sp) + ;; mark the process as running and set its top thread + (set! (-> (-> this process) top-thread) this) + (set! (-> (-> this process) status) 'running) + ;; restore the saved registers + (set! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set! temp (-> this rreg 4)) + (.mov :color #f s4 temp) + (set! temp-float (-> this freg 0)) + (.mov :color #f v8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f v9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f v10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f v11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f v12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f v13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f v14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f v15 temp-float) + ;; restore rreg 5 and rreg 6 into argument registers a4 and a5 + (set! temp (-> this rreg 5)) + (.mov a4 temp) + (set! temp (-> this rreg 6)) + (.mov a5 temp) + ;; convert the resume address to the executable alias before branching + (set! temp (the uint (-> this pc))) + (.add temp xoff) + ;; load the thread process into pp + (set! this (the cpu-thread (-> this process))) + ;; the resumed frame restores lr from its stack + (.jr temp) + (.add a4 a4) + (.add a5 a5))) + (#t + (rlet ((this :reg r13 :type cpu-thread) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (a4 :reg r8 :type uint) + (a5 :reg r9 :type uint) - (temp-float :reg xmm0 :class fpr) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr) - ) - - ;; save the current kernel regs - (.push :color #f s0) - (.push :color #f s1) - (.push :color #f s2) - (.push :color #f s3) - (.push :color #f s4) - - ;; make rsp a GOAL pointer - (.sub sp off) - ;; and store it - (set! *kernel-sp* (the pointer sp)) ;; todo, asm form here? - - ;; temp, stash thread in process-pointer - (set! this thread-to-resume) - - ;; set stack pointer for the thread. leave it as a GOAL pointer for now.. - (set! sp (the uint (-> this sp))) - - ;; restore the stack (sp is a GOAL pointer) - (let ((cur (the (pointer uint64) (-> this stack-top))) - (restore (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size))) + (temp-float :reg xmm0 :class fpr) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr) ) - (while (> (the int cur) (the int sp)) - (set! cur (the (pointer uint64) (&- cur 8))) - (set! restore (the (pointer uint64) (&- restore 8))) - (set! (-> cur) (-> restore)) - ) - ) - ;; offset sp after we're done using it as a GOAL pointer. - (.add sp off) + ;; save the current kernel regs + (.push :color #f s0) + (.push :color #f s1) + (.push :color #f s2) + (.push :color #f s3) + (.push :color #f s4) - ;; setup process - (set! (-> (-> this process) top-thread) this) - (set! (-> (-> this process) status) 'running) + ;; make rsp a GOAL pointer + (.sub sp off) + ;; and store it + (set! *kernel-sp* (the pointer sp)) - ;; restore reg - (set! temp (-> this rreg 0)) - (.mov :color #f s0 temp) - (set! temp (-> this rreg 1)) - (.mov :color #f s1 temp) - (set! temp (-> this rreg 2)) - (.mov :color #f s2 temp) - (set! temp (-> this rreg 3)) - (.mov :color #f s3 temp) - (set! temp (-> this rreg 4)) - (.mov :color #f s4 temp) - (set! temp-float (-> this freg 0)) - (.mov :color #f xmm8 temp-float) - (set! temp-float (-> this freg 1)) - (.mov :color #f xmm9 temp-float) - (set! temp-float (-> this freg 2)) - (.mov :color #f xmm10 temp-float) - (set! temp-float (-> this freg 3)) - (.mov :color #f xmm11 temp-float) - (set! temp-float (-> this freg 4)) - (.mov :color #f xmm12 temp-float) - (set! temp-float (-> this freg 5)) - (.mov :color #f xmm13 temp-float) - (set! temp-float (-> this freg 6)) - (.mov :color #f xmm14 temp-float) - (set! temp-float (-> this freg 7)) - (.mov :color #f xmm15 temp-float) + ;; temp, stash thread in process-pointer + (set! this thread-to-resume) - ;; hack for set-to-run-bootstrap. The set-to-run-bootstrap in MIPS - ;; expects to receive 7 values from the cpu thread's rregs. - ;; usually rreg holds saved registers, but on the first resume after - ;; a set-to-run, they hold arguments, and set-to-run-bootstrap copies them. + ;; set stack pointer for the thread. leave it as a GOAL pointer for now.. + (set! sp (the uint (-> this sp))) - ;; We only have 5 saved regs, so we need to cheat and directly pass - ;; two values in other registers - ;; so we load the a4/a5 argument registers with rreg 5 and rreg 6 - ;; In the case where we are doing a normal resume, the - ;; compiler should assume that these registers are overwritten anyway. - (set! temp (-> this rreg 5)) - (.mov a4 temp) - (set! temp (-> this rreg 6)) - (.mov a5 temp) + ;; restore the stack (sp is a GOAL pointer) + (let ((cur (the (pointer uint64) (-> this stack-top))) + (restore (&+ (the (pointer uint64) (-> this stack)) (-> this stack-size))) + ) + (while (> (the int cur) (the int sp)) + (set! cur (the (pointer uint64) (&- cur 8))) + (set! restore (the (pointer uint64) (&- restore 8))) + (set! (-> cur) (-> restore)) + ) + ) - ;; get the resume address - (set! temp (the uint (-> this pc))) - (.add temp off) + ;; offset sp after we're done using it as a GOAL pointer. + (.add sp off) - ;; setup the process - (set! this (the cpu-thread (-> this process))) - ;; resume! - (.jr temp) - (.add a4 a4) - (.add a5 a5) - ) + ;; setup process + (set! (-> (-> this process) top-thread) this) + (set! (-> (-> this process) status) 'running) + + ;; restore reg + (set! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set! temp (-> this rreg 4)) + (.mov :color #f s4 temp) + (set! temp-float (-> this freg 0)) + (.mov :color #f xmm8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f xmm9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f xmm10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f xmm11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f xmm12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f xmm13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f xmm14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f xmm15 temp-float) + + ;; hack for set-to-run-bootstrap. The set-to-run-bootstrap in MIPS + ;; expects to receive 7 values from the cpu thread's rregs. + ;; usually rreg holds saved registers, but on the first resume after + ;; a set-to-run, they hold arguments, and set-to-run-bootstrap copies them. + + ;; We only have 5 saved regs, so we need to cheat and directly pass + ;; two values in other registers + ;; so we load the a4/a5 argument registers with rreg 5 and rreg 6 + ;; In the case where we are doing a normal resume, the + ;; compiler should assume that these registers are overwritten anyway. + (set! temp (-> this rreg 5)) + (.mov a4 temp) + (set! temp (-> this rreg 6)) + (.mov a5 temp) + + ;; get the resume address + (set! temp (the uint (-> this pc))) + (.add temp off) + + ;; setup the process + (set! this (the cpu-thread (-> this process))) + ;; resume! + (.jr temp) + (.add a4 a4) + (.add a5 a5) + ) + )) (none) ) @@ -1701,104 +1983,183 @@ (allow-saved-regs) ;; very dangerous! ) - (rlet ((pp :reg r13 :type process) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type (pointer uint64)) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((pp :reg pp :type process) + (temp :reg x9 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type (pointer uint64)) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; treat allocation as an address + (let ((this (the catch-frame (&+ (the pointer allocation) *gtype-basic-offset*)))) + ;; initialize the catch frame + (set! (-> this type) type-to-make) + (set! (-> this name) name) + ;; subtract exec-off from lr to store the return address as a GOAL address + (set! temp ra) + (.sub temp xoff) + (set! (-> this ra) (the int temp)) + ;; copy sp through temp because str cannot use sp as data + (set! temp sp) + (.sub temp off) + (set! (-> this sp) (the int temp)) + ;; save the thread registers in the catch frame + (.mov :color #f temp s0) + (set-u128-as-u64! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set-u128-as-u64! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set-u128-as-u64! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set-u128-as-u64! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set-u128-as-u64! (-> this rreg 4) temp) + (.mov :color #f temp v8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp v9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp v10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp v11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp v12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp v13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp v14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp v15) + (set! (-> this freg 7) (the-as float temp)) + ;; add the catch frame to the process frame stack + (set! (-> this next) (-> pp stack-frame-top)) + (set! (-> pp stack-frame-top) this) + ;; save lr before the call clobbers it + (.push :color #f ra) + ;; save s3 and s2 before using them as call temporaries + (.push :color #f s3) + (.push :color #f s2) + (set! s3 (the uint func)) + (set! s2 param-block) + (let ((ret ((the-super-u64-fucntion s3) (-> s2 0) (-> s2 1) (-> s2 2) (-> s2 3) (-> s2 4) (-> s2 5)))) + (.pop :color #f s2) + (.pop :color #f s3) + (.pop :color #f ra) + (set! (-> pp stack-frame-top) (-> pp stack-frame-top next)) + (.ret) + (the object ret))))) + (#t + (rlet ((pp :reg r13 :type process) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type (pointer uint64)) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr) + ) + + ;; we treat the allocation as an address. + (let ((this (the catch-frame (&+ (the pointer allocation) *gtype-basic-offset*)))) + ;; setup catch frame + (set! (-> this type) type-to-make) + (set! (-> this name) name) + ;; get the return address (the compiler won't touch the stack because we're an asm-func) + (.pop temp) + (.push temp) + ;; make it a GOAL address so it fits in 32 bits + (.sub temp off) + ;; store it + (set! (-> this ra) (the int temp)) + + ;; store the stack pointer as a GOAL address + (set! temp sp) + (.sub temp off) + (set! (-> this sp) (the int temp)) + + ;; back up registers we care about + (.mov :color #f temp s0) + (set-u128-as-u64! (-> this rreg 0) temp) + (.mov :color #f temp s1) + (set-u128-as-u64! (-> this rreg 1) temp) + (.mov :color #f temp s2) + (set-u128-as-u64! (-> this rreg 2) temp) + (.mov :color #f temp s3) + (set-u128-as-u64! (-> this rreg 3) temp) + (.mov :color #f temp s4) + (set-u128-as-u64! (-> this rreg 4) temp) + + (.mov :color #f temp xmm8) + (set! (-> this freg 0) (the-as float temp)) + (.mov :color #f temp xmm9) + (set! (-> this freg 1) (the-as float temp)) + (.mov :color #f temp xmm10) + (set! (-> this freg 2) (the-as float temp)) + (.mov :color #f temp xmm11) + (set! (-> this freg 3) (the-as float temp)) + (.mov :color #f temp xmm12) + (set! (-> this freg 4) (the-as float temp)) + (.mov :color #f temp xmm13) + (set! (-> this freg 5) (the-as float temp)) + (.mov :color #f temp xmm14) + (set! (-> this freg 6) (the-as float temp)) + (.mov :color #f temp xmm15) + (set! (-> this freg 7) (the-as float temp)) + + ;; push this stack frame + (set! (-> this next) (-> pp stack-frame-top)) + (set! (-> pp stack-frame-top) this) + + ;; help coloring, it isn't smart enough to realize it's "safe" to use these registers. + (.push :color #f s3) + (.push :color #f s2) + ;; keep the stack aligned for the call + (.push :color #f s2) + (set! s3 (the uint func)) + (set! s2 param-block) + + (let ((ret ((the-super-u64-fucntion s3) + (-> s2 0) + (-> s2 1) + (-> s2 2) + (-> s2 3) + (-> s2 4) + (-> s2 5) + )) + ) + (.pop :color #f s2) + (.pop :color #f s2) + (.pop :color #f s3) + (set! (-> pp stack-frame-top) (-> pp stack-frame-top next)) + (.ret) + (the object ret) ) - - ;; we treat the allocation as an address. - (let ((this (the catch-frame (&+ (the pointer allocation) *gtype-basic-offset*)))) - ;; setup catch frame - (set! (-> this type) type-to-make) - (set! (-> this name) name) - ;; get the return address (the compiler won't touch the stack because we're an asm-func) - (.pop temp) - (.push temp) - ;; make it a GOAL address so it fits in 32 bits - (.sub temp off) - ;; store it - (set! (-> this ra) (the int temp)) - - ;; todo, do we need a stack offset here? - ;; remember the stack pointer - (set! temp sp) - (.sub temp off) - (set! (-> this sp) (the int temp)) - - ;; back up registers we care about - (.mov :color #f temp s0) - (set-u128-as-u64! (-> this rreg 0) temp) - (.mov :color #f temp s1) - (set-u128-as-u64! (-> this rreg 1) temp) - (.mov :color #f temp s2) - (set-u128-as-u64! (-> this rreg 2) temp) - (.mov :color #f temp s3) - (set-u128-as-u64! (-> this rreg 3) temp) - (.mov :color #f temp s4) - (set-u128-as-u64! (-> this rreg 4) temp) - - (.mov :color #f temp xmm8) - (set! (-> this freg 0) (the-as float temp)) - (.mov :color #f temp xmm9) - (set! (-> this freg 1) (the-as float temp)) - (.mov :color #f temp xmm10) - (set! (-> this freg 2) (the-as float temp)) - (.mov :color #f temp xmm11) - (set! (-> this freg 3) (the-as float temp)) - (.mov :color #f temp xmm12) - (set! (-> this freg 4) (the-as float temp)) - (.mov :color #f temp xmm13) - (set! (-> this freg 5) (the-as float temp)) - (.mov :color #f temp xmm14) - (set! (-> this freg 6) (the-as float temp)) - (.mov :color #f temp xmm15) - (set! (-> this freg 7) (the-as float temp)) - - ;; push this stack frame - (set! (-> this next) (-> pp stack-frame-top)) - (set! (-> pp stack-frame-top) this) - - ;; help coloring, it isn't smart enough to realize it's "safe" to use these registers. - (.push :color #f s3) - (.push :color #f s2) - (.push :color #f s2) - (set! s3 (the uint func)) - (set! s2 param-block) - - ;; todo - are we aligned correctly here? - (let ((ret ((the-super-u64-fucntion s3) - (-> s2 0) - (-> s2 1) - (-> s2 2) - (-> s2 3) - (-> s2 4) - (-> s2 5) - )) - ) - (.pop :color #f s2) - (.pop :color #f s2) - (.pop :color #f s3) - (set! (-> pp stack-frame-top) (-> pp stack-frame-top next)) - (.ret) - (the object ret) - ) - ) - ) + ) + ) + )) ) (defun throw-dispatch ((this catch-frame) value) @@ -1806,73 +2167,138 @@ Only can throw a 64-bit value. The original could throw 128 bits." (declare (asm-func none)) - (rlet ((pp :reg r13 :type process) - (temp :reg rax :type uint) - (off :reg r15 :type uint) - (sp :reg rsp :type uint) - (s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type (pointer uint64)) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((pp :reg pp :type process) + (temp :reg x9 :type uint) + (off :reg off :type uint) + (xoff :reg exec-off :type uint) + (sp :reg sp :type uint) + (ra :reg lr :type uint) + ;; reuse x0 for retval after loading the catch frame return address + (retval :reg carg0 :type uint) + (s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type (pointer uint64)) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (temp-float :reg v0 :class fpr) + (v8 :reg v8 :class fpr) + (v9 :reg v9 :class fpr) + (v10 :reg v10 :class fpr) + (v11 :reg v11 :class fpr) + (v12 :reg v12 :class fpr) + (v13 :reg v13 :class fpr) + (v14 :reg v14 :class fpr) + (v15 :reg v15 :class fpr)) + ;; discard catch frames through this frame + (set! (-> pp stack-frame-top) (-> this next)) + ;; restore the saved registers + (set-u64-from-u128! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set-u64-from-u128! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set-u64-from-u128! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set-u64-from-u128! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set-u64-from-u128! temp (-> this rreg 4)) + (.mov :color #f s4 temp) + (set! temp-float (-> this freg 0)) + (.mov :color #f v8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f v9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f v10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f v11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f v12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f v13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f v14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f v15 temp-float) + ;; rebuild sp through temp because ldr cannot target sp + (set! temp (the uint (-> this sp))) + (.add temp off) + (set! sp temp) + ;; add exec-off to restore the catch frame return address in lr + (set! ra (the uint (-> this ra))) + (.add ra xoff) + ;; load the return register + (.mov retval value) + (.ret))) + (#t + (rlet ((pp :reg r13 :type process) + (temp :reg rax :type uint) + (off :reg r15 :type uint) + (sp :reg rsp :type uint) + (s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type (pointer uint64)) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) - (temp-float :reg xmm0 :class fpr) - (xmm8 :reg xmm8 :class fpr) - (xmm9 :reg xmm9 :class fpr) - (xmm10 :reg xmm10 :class fpr) - (xmm11 :reg xmm11 :class fpr) - (xmm12 :reg xmm12 :class fpr) - (xmm13 :reg xmm13 :class fpr) - (xmm14 :reg xmm14 :class fpr) - (xmm15 :reg xmm15 :class fpr) - ) + (temp-float :reg xmm0 :class fpr) + (xmm8 :reg xmm8 :class fpr) + (xmm9 :reg xmm9 :class fpr) + (xmm10 :reg xmm10 :class fpr) + (xmm11 :reg xmm11 :class fpr) + (xmm12 :reg xmm12 :class fpr) + (xmm13 :reg xmm13 :class fpr) + (xmm14 :reg xmm14 :class fpr) + (xmm15 :reg xmm15 :class fpr) + ) - ;; pop everything we threw past - (set! (-> pp stack-frame-top) (-> this next)) + ;; pop everything we threw past + (set! (-> pp stack-frame-top) (-> this next)) - ;; restore regs we care about. - (set-u64-from-u128! temp (-> this rreg 0)) - (.mov :color #f s0 temp) - (set-u64-from-u128! temp (-> this rreg 1)) - (.mov :color #f s1 temp) - (set-u64-from-u128! temp (-> this rreg 2)) - (.mov :color #f s2 temp) - (set-u64-from-u128! temp (-> this rreg 3)) - (.mov :color #f s3 temp) - (set-u64-from-u128! temp (-> this rreg 4)) - (.mov :color #f s4 temp) + ;; restore regs we care about. + (set-u64-from-u128! temp (-> this rreg 0)) + (.mov :color #f s0 temp) + (set-u64-from-u128! temp (-> this rreg 1)) + (.mov :color #f s1 temp) + (set-u64-from-u128! temp (-> this rreg 2)) + (.mov :color #f s2 temp) + (set-u64-from-u128! temp (-> this rreg 3)) + (.mov :color #f s3 temp) + (set-u64-from-u128! temp (-> this rreg 4)) + (.mov :color #f s4 temp) - (set! temp-float (-> this freg 0)) - (.mov :color #f xmm8 temp-float) - (set! temp-float (-> this freg 1)) - (.mov :color #f xmm9 temp-float) - (set! temp-float (-> this freg 2)) - (.mov :color #f xmm10 temp-float) - (set! temp-float (-> this freg 3)) - (.mov :color #f xmm11 temp-float) - (set! temp-float (-> this freg 4)) - (.mov :color #f xmm12 temp-float) - (set! temp-float (-> this freg 5)) - (.mov :color #f xmm13 temp-float) - (set! temp-float (-> this freg 6)) - (.mov :color #f xmm14 temp-float) - (set! temp-float (-> this freg 7)) - (.mov :color #f xmm15 temp-float) + (set! temp-float (-> this freg 0)) + (.mov :color #f xmm8 temp-float) + (set! temp-float (-> this freg 1)) + (.mov :color #f xmm9 temp-float) + (set! temp-float (-> this freg 2)) + (.mov :color #f xmm10 temp-float) + (set! temp-float (-> this freg 3)) + (.mov :color #f xmm11 temp-float) + (set! temp-float (-> this freg 4)) + (.mov :color #f xmm12 temp-float) + (set! temp-float (-> this freg 5)) + (.mov :color #f xmm13 temp-float) + (set! temp-float (-> this freg 6)) + (.mov :color #f xmm14 temp-float) + (set! temp-float (-> this freg 7)) + (.mov :color #f xmm15 temp-float) - ;; set stack pointer - (set! sp (the uint (-> this sp))) - (.add sp off) + ;; set stack pointer + (set! sp (the uint (-> this sp))) + (.add sp off) - ;; overwrite our return address - (.pop temp) - (set! temp (the uint (-> this ra))) - (.add temp off) - (.push temp) + ;; overwrite our return address + (.pop temp) + (set! temp (the uint (-> this ra))) + (.add temp off) + (.push temp) - ;; load the return register - (.mov temp value) - (.ret) - ) + ;; load the return register + (.mov temp value) + (.ret) + ) + )) ) (defun throw ((name symbol) value) @@ -2174,39 +2600,68 @@ ;;(print-asm) ) - (rlet ((s0 :reg rbx :type uint) - (s1 :reg rbp :type uint) - (s2 :reg r10 :type uint) - (s3 :reg r11 :type uint) - (s4 :reg r12 :type uint) - (a0 :reg rdi :type uint) ; ok - (a1 :reg rsi :type uint) ; ok - (a2 :reg rdx :type uint) ; ok - (a3 :reg rcx :type uint) ; ok - (off :reg r15 :type uint) - (a4 :reg r8 :type uint) - (a5 :reg r9 :type uint) - (temp :reg rax) - ) + (#cond + ((eq? INSTRUCTION_SET 'arm64) + (rlet ((s0 :reg x19 :type uint) + (s1 :reg x23 :type uint) + (s2 :reg x24 :type uint) + (s3 :reg x25 :type uint) + (s4 :reg x26 :type uint) + (a0 :reg carg0 :type uint) + (a1 :reg carg1 :type uint) + (a2 :reg carg2 :type uint) + (a3 :reg carg3 :type uint) + (xoff :reg exec-off :type uint) + (a4 :reg carg4 :type uint) + (a5 :reg carg5 :type uint) + (ra :reg lr :type uint)) + ;; set lr before the argument registers are live + (set! ra (the uint return-from-thread-dead)) + (.add ra xoff) + (.mov :color #f a0 s1) + (.mov :color #f a1 s2) + (.mov :color #f a2 s3) + (.mov :color #f a3 s4) + ;; branch targets go through the exec base + (.add :color #f s0 xoff) + (.jr :color #f s0) + (.add a4 a4) + (.add a5 a5))) + (#t + (rlet ((s0 :reg rbx :type uint) + (s1 :reg rbp :type uint) + (s2 :reg r10 :type uint) + (s3 :reg r11 :type uint) + (s4 :reg r12 :type uint) + (a0 :reg rdi :type uint) ; ok + (a1 :reg rsi :type uint) ; ok + (a2 :reg rdx :type uint) ; ok + (a3 :reg rcx :type uint) ; ok + (off :reg r15 :type uint) + (a4 :reg r8 :type uint) + (a5 :reg r9 :type uint) + (temp :reg rax) + ) - (.mov temp return-from-thread-dead) - (.add temp off) - (.push temp) + (.mov temp return-from-thread-dead) + (.add temp off) + (.push temp) - ;; stack is 16 + 8 aligned now + ;; stack is 16 + 8 aligned now - (.mov :color #f a0 s1) - (.mov :color #f a1 s2) - (.mov :color #f a2 s3) - (.mov :color #f a3 s4) + (.mov :color #f a0 s1) + (.mov :color #f a1 s2) + (.mov :color #f a2 s3) + (.mov :color #f a3 s4) - (.add :color #f s0 off) - (.jr :color #f s0) + (.add :color #f s0 off) + (.jr :color #f s0) - (.add a4 a4) - (.add a5 a5) - ) + (.add a4 a4) + (.add a5 a5) + ) + )) ) ;; definition for function set-to-run diff --git a/goalc/compiler/CodeGenerator.cpp b/goalc/compiler/CodeGenerator.cpp index 3866e884be..4cfbef99d8 100644 --- a/goalc/compiler/CodeGenerator.cpp +++ b/goalc/compiler/CodeGenerator.cpp @@ -15,6 +15,7 @@ #include "goalc/debugger/DebugInfo.h" #include "goalc/emitter/IGen.h" +#include "goalc/emitter/IGenARM64.h" #include "fmt/format.h" @@ -165,13 +166,7 @@ std::vector CodeGenerator::run(const TypeSystem* ts) { void CodeGenerator::do_function(FunctionEnv* env, int f_idx) { if (env->is_asm_func) { - if (m_gen.instr_set() == InstructionSet::X86) { - do_asm_function_x86(env, f_idx, env->asm_func_saved_regs); - } else if (m_gen.instr_set() == InstructionSet::ARM64) { - do_asm_function_arm64(env, f_idx, env->asm_func_saved_regs); - } else { - throw std::runtime_error("CodeGenerator::do_function, instruction set not supported"); - } + do_asm_function(env, f_idx, env->asm_func_saved_regs); } else { if (m_gen.instr_set() == InstructionSet::X86) { do_goal_function_x86(env, f_idx); @@ -194,7 +189,7 @@ void CodeGenerator::do_goal_function_x86(FunctionEnv* env, int f_idx) { auto f_rec = m_gen.get_existing_function_record(f_idx); // todo, extra alignment settings - auto& ri = emitter::gRegInfo; + auto& ri = emitter::reg_info(m_gen.instr_set()); const auto& allocs = env->alloc_result(); // compute how much stack we will use @@ -209,7 +204,7 @@ void CodeGenerator::do_goal_function_x86(FunctionEnv* env, int f_idx) { } // only for new xmms. if n == 0, we don't use this at all. - int xmm_backup_stack_offset = 8 + XMM_SIZE * n_xmm_backups; + int xmm_backup_stack_offset = 8 + SIMD_SIZE * n_xmm_backups; if (use_new_xmms) { if (n_xmm_backups > 0) { @@ -221,9 +216,9 @@ void CodeGenerator::do_goal_function_x86(FunctionEnv* env, int f_idx) { int i = 0; for (auto& saved_reg : allocs.used_saved_regs) { if (saved_reg.is_xmm(m_gen.instr_set())) { - int offset = i * XMM_SIZE; + int offset = i * SIMD_SIZE; m_gen.add_instr_no_ir(f_rec, - IGen::store128_xmm128_reg_offset(m_gen, RSP, saved_reg, offset), + IGen::store128_simd128_reg_offset(m_gen, RSP, saved_reg, offset), InstructionInfo::Kind::PROLOGUE); i++; } @@ -233,11 +228,11 @@ void CodeGenerator::do_goal_function_x86(FunctionEnv* env, int f_idx) { // back up xmms (currently not aligned) for (auto& saved_reg : allocs.used_saved_regs) { if (saved_reg.is_xmm(m_gen.instr_set())) { - m_gen.add_instr_no_ir(f_rec, IGen::sub_gpr64_imm8s(m_gen, RSP, XMM_SIZE), + m_gen.add_instr_no_ir(f_rec, IGen::sub_gpr64_imm8s(m_gen, RSP, SIMD_SIZE), InstructionInfo::Kind::PROLOGUE); m_gen.add_instr_no_ir(f_rec, IGen::store128_gpr64_simd128(m_gen, RSP, saved_reg), InstructionInfo::Kind::PROLOGUE); - stack_offset += XMM_SIZE; + stack_offset += SIMD_SIZE; } } } @@ -302,12 +297,12 @@ void CodeGenerator::do_goal_function_x86(FunctionEnv* env, int f_idx) { i_rec); } else if (op.reg.is_xmm(m_gen.instr_set()) && op.reg_class == RegClass::FLOAT) { // load xmm32 off of the stack - m_gen.add_instr(IGen::load_reg_offset_xmm32( + m_gen.add_instr(IGen::load_reg_offset_simd32( m_gen, op.reg, RSP, allocs.get_slot_for_spill(op.slot) * GPR_SIZE), i_rec); } else if (op.reg.is_xmm(m_gen.instr_set()) && (op.reg_class == RegClass::VECTOR_FLOAT || op.reg_class == RegClass::INT_128)) { - m_gen.add_instr(IGen::load128_xmm128_reg_offset( + m_gen.add_instr(IGen::load128_simd128_reg_offset( m_gen, op.reg, RSP, allocs.get_slot_for_spill(op.slot) * GPR_SIZE), i_rec); } else { @@ -329,12 +324,12 @@ void CodeGenerator::do_goal_function_x86(FunctionEnv* env, int f_idx) { i_rec); } else if (op.reg.is_xmm(m_gen.instr_set()) && op.reg_class == RegClass::FLOAT) { // store xmm32 on the stack - m_gen.add_instr(IGen::store_reg_offset_xmm32( + m_gen.add_instr(IGen::store_reg_offset_simd32( m_gen, RSP, op.reg, allocs.get_slot_for_spill(op.slot) * GPR_SIZE), i_rec); } else if (op.reg.is_xmm(m_gen.instr_set()) && (op.reg_class == RegClass::VECTOR_FLOAT || op.reg_class == RegClass::INT_128)) { - m_gen.add_instr(IGen::store128_xmm128_reg_offset( + m_gen.add_instr(IGen::store128_simd128_reg_offset( m_gen, RSP, op.reg, allocs.get_slot_for_spill(op.slot) * GPR_SIZE), i_rec); } else { @@ -374,9 +369,9 @@ void CodeGenerator::do_goal_function_x86(FunctionEnv* env, int f_idx) { auto& saved_reg = allocs.used_saved_regs.at(i); if (saved_reg.is_xmm(m_gen.instr_set())) { j--; - int offset = j * XMM_SIZE; + int offset = j * SIMD_SIZE; m_gen.add_instr_no_ir(f_rec, - IGen::load128_xmm128_reg_offset(m_gen, saved_reg, RSP, offset), + IGen::load128_simd128_reg_offset(m_gen, saved_reg, RSP, offset), InstructionInfo::Kind::EPILOGUE); } } @@ -390,7 +385,7 @@ void CodeGenerator::do_goal_function_x86(FunctionEnv* env, int f_idx) { if (saved_reg.is_xmm(m_gen.instr_set())) { m_gen.add_instr_no_ir(f_rec, IGen::load128_simd128_gpr64(m_gen, saved_reg, RSP), InstructionInfo::Kind::EPILOGUE); - m_gen.add_instr_no_ir(f_rec, IGen::add_gpr64_imm8s(m_gen, RSP, XMM_SIZE), + m_gen.add_instr_no_ir(f_rec, IGen::add_gpr64_imm8s(m_gen, RSP, SIMD_SIZE), InstructionInfo::Kind::EPILOGUE); } } @@ -400,10 +395,173 @@ void CodeGenerator::do_goal_function_x86(FunctionEnv* env, int f_idx) { } void CodeGenerator::do_goal_function_arm64(FunctionEnv* env, int f_idx) { - throw std::runtime_error("NYI - CodeGenerator::do_goal_function_arm64"); + auto* debug = &m_debug_info->function_by_name(env->name()); + auto f_rec = m_gen.get_existing_function_record(f_idx); + const auto& allocs = env->alloc_result(); + + // keep sp 16-byte aligned and save x30 + int stack_offset = 0; + + std::vector gprs_to_save; + gprs_to_save.push_back(emitter::Register(emitter::ARM64_REG::X30)); + for (auto& saved_reg : allocs.used_saved_regs) { + if (saved_reg.is_gpr(m_gen.instr_set())) { + gprs_to_save.push_back(saved_reg); + } + } + for (size_t i = 0; i < gprs_to_save.size(); i += 2) { + if (i + 1 < gprs_to_save.size()) { + m_gen.add_instr_no_ir(f_rec, + IGen::ARM64::push_pair_gpr64(gprs_to_save[i], gprs_to_save[i + 1]), + InstructionInfo::Kind::PROLOGUE); + } else { + // an odd GPR gets its own 16-byte slot + m_gen.add_instr_no_ir(f_rec, IGen::push_gpr64(m_gen, gprs_to_save[i]), + InstructionInfo::Kind::PROLOGUE); + } + stack_offset += 16; + } + + // save any allocated vector registers + int n_vec_backups = 0; + for (auto& saved_reg : allocs.used_saved_regs) { + if (saved_reg.is_128bit_simd(m_gen.instr_set())) { + n_vec_backups++; + } + } + int vec_backup_size = SIMD_SIZE * n_vec_backups; + if (n_vec_backups > 0) { + m_gen.add_instr_no_ir(f_rec, + IGen::sub_gpr64_imm(m_gen, emitter::ARM64_REG::SP, vec_backup_size), + InstructionInfo::Kind::PROLOGUE); + int i = 0; + for (auto& saved_reg : allocs.used_saved_regs) { + if (saved_reg.is_128bit_simd(m_gen.instr_set())) { + m_gen.add_instr_no_ir(f_rec, + IGen::store128_simd128_reg_offset(m_gen, emitter::ARM64_REG::SP, + saved_reg, i * SIMD_SIZE), + InstructionInfo::Kind::PROLOGUE); + i++; + } + } + stack_offset += vec_backup_size; + } + + // aligned space for spills and stack variables + int manually_added_stack_offset = + GPR_SIZE * (allocs.stack_slots_for_spills + allocs.stack_slots_for_vars); + manually_added_stack_offset = (manually_added_stack_offset + 15) & ~15; + if (manually_added_stack_offset) { + m_gen.add_instr_no_ir( + f_rec, IGen::sub_gpr64_imm(m_gen, emitter::ARM64_REG::SP, manually_added_stack_offset), + InstructionInfo::Kind::PROLOGUE); + stack_offset += manually_added_stack_offset; + } + ASSERT((stack_offset & 15) == 0); + debug->stack_usage = stack_offset; + + for (int ir_idx = 0; ir_idx < int(env->code().size()); ir_idx++) { + auto& ir = env->code().at(ir_idx); + auto i_rec = m_gen.add_ir(f_rec); + + // reload any spilled operands + auto& bonus = allocs.stack_ops.at(ir_idx); + for (auto& op : bonus.ops) { + if (op.load) { + if (op.reg.is_gpr(m_gen.instr_set()) && op.reg_class == RegClass::GPR_64) { + m_gen.add_instr(IGen::load64_gpr64_plus_s32(m_gen, op.reg, + allocs.get_slot_for_spill(op.slot) * GPR_SIZE, + emitter::ARM64_REG::SP), + i_rec); + } else if (op.reg.is_128bit_simd(m_gen.instr_set()) && op.reg_class == RegClass::FLOAT) { + // FLOAT spills use an 8-byte slot but move only 32 bits + m_gen.add_instr( + IGen::load_reg_offset_simd32(m_gen, op.reg, emitter::ARM64_REG::SP, + allocs.get_slot_for_spill(op.slot) * GPR_SIZE), + i_rec); + } else if (op.reg.is_128bit_simd(m_gen.instr_set()) && + (op.reg_class == RegClass::VECTOR_FLOAT || op.reg_class == RegClass::INT_128)) { + m_gen.add_instr( + IGen::load128_simd128_reg_offset(m_gen, op.reg, emitter::ARM64_REG::SP, + allocs.get_slot_for_spill(op.slot) * GPR_SIZE), + i_rec); + } else { + ASSERT(false); + } + } + } + + ir->do_codegen_arm64(&m_gen, allocs, i_rec); + + for (auto& op : bonus.ops) { + if (op.store) { + if (op.reg.is_gpr(m_gen.instr_set()) && op.reg_class == RegClass::GPR_64) { + m_gen.add_instr( + IGen::store64_gpr64_plus_s32(m_gen, emitter::ARM64_REG::SP, + allocs.get_slot_for_spill(op.slot) * GPR_SIZE, op.reg), + i_rec); + } else if (op.reg.is_128bit_simd(m_gen.instr_set()) && op.reg_class == RegClass::FLOAT) { + m_gen.add_instr( + IGen::store_reg_offset_simd32(m_gen, emitter::ARM64_REG::SP, op.reg, + allocs.get_slot_for_spill(op.slot) * GPR_SIZE), + i_rec); + } else if (op.reg.is_128bit_simd(m_gen.instr_set()) && + (op.reg_class == RegClass::VECTOR_FLOAT || op.reg_class == RegClass::INT_128)) { + m_gen.add_instr( + IGen::store128_simd128_reg_offset(m_gen, emitter::ARM64_REG::SP, op.reg, + allocs.get_slot_for_spill(op.slot) * GPR_SIZE), + i_rec); + } else { + ASSERT(false); + } + } + } + } + + if (manually_added_stack_offset) { + m_gen.add_instr_no_ir( + f_rec, IGen::add_gpr64_imm(m_gen, emitter::ARM64_REG::SP, manually_added_stack_offset), + InstructionInfo::Kind::EPILOGUE); + } + + if (n_vec_backups > 0) { + int j = n_vec_backups; + for (int i = int(allocs.used_saved_regs.size()); i-- > 0;) { + auto& saved_reg = allocs.used_saved_regs.at(i); + if (saved_reg.is_128bit_simd(m_gen.instr_set())) { + j--; + m_gen.add_instr_no_ir(f_rec, + IGen::load128_simd128_reg_offset( + m_gen, saved_reg, emitter::ARM64_REG::SP, j * SIMD_SIZE), + InstructionInfo::Kind::EPILOGUE); + } + } + ASSERT(j == 0); + m_gen.add_instr_no_ir(f_rec, + IGen::add_gpr64_imm(m_gen, emitter::ARM64_REG::SP, vec_backup_size), + InstructionInfo::Kind::EPILOGUE); + } + + // restore GPRs in reverse order + for (size_t i = (gprs_to_save.size() + 1) / 2 * 2; i >= 2;) { + i -= 2; + if (i + 1 < gprs_to_save.size()) { + m_gen.add_instr_no_ir(f_rec, + IGen::ARM64::pop_pair_gpr64(gprs_to_save[i], gprs_to_save[i + 1]), + InstructionInfo::Kind::EPILOGUE); + } else { + m_gen.add_instr_no_ir(f_rec, IGen::pop_gpr64(m_gen, gprs_to_save[i]), + InstructionInfo::Kind::EPILOGUE); + } + } + + m_gen.add_instr_no_ir(f_rec, IGen::ret(m_gen), InstructionInfo::Kind::EPILOGUE); } -void CodeGenerator::do_asm_function_x86(FunctionEnv* env, int f_idx, bool allow_saved_regs) { +/*! + * Emit an ASM function without a compiler-generated prologue or epilogue. + */ +void CodeGenerator::do_asm_function(FunctionEnv* env, int f_idx, bool allow_saved_regs) { auto f_rec = m_gen.get_existing_function_record(f_idx); const auto& allocs = env->alloc_result(); @@ -411,7 +569,7 @@ void CodeGenerator::do_asm_function_x86(FunctionEnv* env, int f_idx, bool allow_ std::string err = fmt::format( "ASM Function {}'s coloring using the following callee-saved registers: ", env->name()); for (auto& x : allocs.used_saved_regs) { - err += x.print(); + err += x.print(m_gen.instr_set()); err += " "; } err.pop_back(); @@ -427,7 +585,6 @@ void CodeGenerator::do_asm_function_x86(FunctionEnv* env, int f_idx, bool allow_ throw std::runtime_error("ASM Function has variables on the stack."); } - // emit each IR into x86 instructions. for (int ir_idx = 0; ir_idx < int(env->code().size()); ir_idx++) { auto& ir = env->code().at(ir_idx); // start of IR @@ -439,10 +596,10 @@ void CodeGenerator::do_asm_function_x86(FunctionEnv* env, int f_idx, bool allow_ } // do the actual op - ir->do_codegen_x86(&m_gen, allocs, i_rec); + if (m_gen.instr_set() == InstructionSet::X86) { + ir->do_codegen_x86(&m_gen, allocs, i_rec); + } else { + ir->do_codegen_arm64(&m_gen, allocs, i_rec); + } } } - -void CodeGenerator::do_asm_function_arm64(FunctionEnv* env, int f_idx, bool allow_saved_regs) { - throw std::runtime_error("NYI - CodeGenerator::do_asm_function"); -} diff --git a/goalc/compiler/CodeGenerator.h b/goalc/compiler/CodeGenerator.h index f7e06690af..8fbb29ed7f 100644 --- a/goalc/compiler/CodeGenerator.h +++ b/goalc/compiler/CodeGenerator.h @@ -29,8 +29,7 @@ class CodeGenerator { void do_function(FunctionEnv* env, int f_idx); void do_goal_function_x86(FunctionEnv* env, int f_idx); void do_goal_function_arm64(FunctionEnv* env, int f_idx); - void do_asm_function_x86(FunctionEnv* env, int f_idx, bool allow_saved_regs); - void do_asm_function_arm64(FunctionEnv* env, int f_idx, bool allow_saved_regs); + void do_asm_function(FunctionEnv* env, int f_idx, bool allow_saved_regs); emitter::ObjectGenerator m_gen; FileEnv* m_fe = nullptr; DebugInfo* m_debug_info = nullptr; diff --git a/goalc/compiler/Compiler.cpp b/goalc/compiler/Compiler.cpp index e61c302f7e..a36bb46311 100644 --- a/goalc/compiler/Compiler.cpp +++ b/goalc/compiler/Compiler.cpp @@ -48,6 +48,11 @@ Compiler::Compiler(GameVersion version, Object library_code = m_goos.reader.read_from_file({"goal_src", "goal-lib.gc"}); compile_object_file("goal-lib", library_code, false); + // goal-lib defaults this to x86 for older compilers + m_goos.set_global_variable_by_name( + "INSTRUCTION_SET", + m_goos.intern(m_instr_set == emitter::InstructionSet::ARM64 ? "arm64" : "x86")); + // user profile stuff if (user_profile != "#f" && fs::exists(file_util::get_jak_project_dir() / "goal_src" / "user" / user_profile / "user.gc")) { @@ -180,7 +185,7 @@ std::unique_ptr Compiler::compile_top_level_function(const std::str // only move to return register if we actually got a result if (!dynamic_cast(result)) { fe->emit_ir(code, fe->make_gpr(result->type()), result->to_gpr(code, fe.get()), - emitter::gRegInfo.get_gpr_ret_reg()); + emitter::reg_info(m_instr_set).get_gpr_ret_reg()); } if (!fe->code().empty()) { @@ -257,6 +262,7 @@ void Compiler::color_object_file(FileEnv* env) { for (auto& f : env->functions()) { AllocationInput input; input.is_asm_function = f->is_asm_func; + input.instr_set = m_instr_set; for (auto& i : f->code()) { input.instructions.push_back(i->to_rai()); // input.debug_instruction_names.push_back(i->print()); diff --git a/goalc/compiler/Compiler.h b/goalc/compiler/Compiler.h index 06a8e6475c..727b815291 100644 --- a/goalc/compiler/Compiler.h +++ b/goalc/compiler/Compiler.h @@ -87,6 +87,7 @@ class Compiler { m_ts.add_type_to_allowed_redefinition_list(type_name); } Debugger& get_debugger() { return m_debugger; } + emitter::InstructionSet instruction_set() const { return m_instr_set; } listener::Listener& listener() { return m_listener; } void poke_target() { m_listener.send_poke(); } bool connect_to_target(); @@ -503,6 +504,7 @@ class Compiler { // Asm Val* compile_rlet(const goos::Object& form, const goos::Object& rest, Env* env); Val* compile_asm_ret(const goos::Object& form, const goos::Object& rest, Env* env); + Val* compile_asm_break(const goos::Object& form, const goos::Object& rest, Env* env); Val* compile_asm_push(const goos::Object& form, const goos::Object& rest, Env* env); Val* compile_asm_pop(const goos::Object& form, const goos::Object& rest, Env* env); Val* compile_asm_sub(const goos::Object& form, const goos::Object& rest, Env* env); diff --git a/goalc/compiler/IR.cpp b/goalc/compiler/IR.cpp index ce4f98f91b..2f7068d592 100644 --- a/goalc/compiler/IR.cpp +++ b/goalc/compiler/IR.cpp @@ -6,12 +6,10 @@ #include "goalc/compiler/Env.h" #include "goalc/emitter/IGen.h" +#include "goalc/emitter/IGenARM64.h" #include "fmt/format.h" -// TODO ARM64 - just silencing errors while things are not implemented obviously -#pragma GCC diagnostic ignored "-Wunused-parameter" - using namespace emitter; namespace { Register get_reg(const RegVal* rv, const AllocationResult& allocs, emitter::IR_Record irec) { @@ -69,6 +67,30 @@ Register get_reg_asm(const RegVal* rv, return use_coloring ? get_reg(rv, allocs, irec) : get_no_color_reg(rv); } +/*! + * Load a symbol value through an ARM64 movz/movk relocation. + */ +void arm64_load_symbol_value(emitter::ObjectGenerator* gen, + emitter::IR_Record irec, + Register dst, + const std::string& sym_name, + bool sext) { + const auto& info = emitter::reg_info(gen->instr_set()); + auto instr = gen->add_instr( + IGen::ARM64::mov_gpr32_link_imm32(emitter::X16, LINK_SYM_NO_OFFSET_FLAG), irec); + gen->link_instruction_symbol_mem(instr, sym_name); + // symbol offsets are signed relative to s7 + gen->add_instr(IGen::ARM64::add_gpr64_gpr64_sxtw(emitter::X16, info.get_st_reg(), emitter::X16), + irec); + if (sext) { + gen->add_instr( + IGen::load32s_gpr64_gpr64_plus_gpr64(*gen, dst, info.get_offset_reg(), emitter::X16), irec); + } else { + gen->add_instr( + IGen::load32u_gpr64_gpr64_plus_gpr64(*gen, dst, info.get_offset_reg(), emitter::X16), irec); + } +} + void load_constant(u64 value, emitter::ObjectGenerator* gen, emitter::IR_Record irec, @@ -104,8 +126,8 @@ void regset_common(emitter::ObjectGenerator* gen, auto src_class = src->ireg().reg_class; auto dst_class = dst->ireg().reg_class; - bool src_is_xmm128 = (src_class == RegClass::VECTOR_FLOAT || src_class == RegClass::INT_128); - bool dst_is_xmm128 = (dst_class == RegClass::VECTOR_FLOAT || dst_class == RegClass::INT_128); + bool src_is_simd128 = (src_class == RegClass::VECTOR_FLOAT || src_class == RegClass::INT_128); + bool dst_is_simd128 = (dst_class == RegClass::VECTOR_FLOAT || dst_class == RegClass::INT_128); if (src_class == RegClass::GPR_64 && dst_class == RegClass::GPR_64) { if (src_reg == dst_reg) { @@ -123,7 +145,7 @@ void regset_common(emitter::ObjectGenerator* gen, } else { gen->add_instr(IGen::mov_f32_f32(*gen, dst_reg, src_reg), irec); } - } else if (src_is_xmm128 && dst_is_xmm128) { + } else if (src_is_simd128 && dst_is_simd128) { if (src_reg == dst_reg) { // eliminate move gen->count_eliminated_move(); @@ -132,20 +154,20 @@ void regset_common(emitter::ObjectGenerator* gen, gen->add_instr(IGen::mov_vf_vf(*gen, dst_reg, src_reg), irec); } } else if (src_class == RegClass::FLOAT && dst_class == RegClass::GPR_64) { - // xmm 1x -> gpr + // SIMD scalar -> GPR gen->add_instr(IGen::movd_gpr32_f32(*gen, dst_reg, src_reg), irec); // don't forget to sign extend gen->add_instr(IGen::movsx_r64_r32(*gen, dst_reg, dst_reg), irec); } else if (src_class == RegClass::GPR_64 && dst_class == RegClass::FLOAT) { - // gpr -> xmm 1x + // GPR -> SIMD scalar gen->add_instr(IGen::movd_f32_gpr32(*gen, dst_reg, src_reg), irec); - } else if (src_is_xmm128 && dst_class == RegClass::FLOAT) { + } else if (src_is_simd128 && dst_class == RegClass::FLOAT) { gen->add_instr(IGen::mov_f32_f32(*gen, dst_reg, src_reg), irec); - } else if (src_class == RegClass::FLOAT && dst_is_xmm128) { + } else if (src_class == RegClass::FLOAT && dst_is_simd128) { gen->add_instr(IGen::mov_f32_f32(*gen, dst_reg, src_reg), irec); - } else if (src_class == RegClass::GPR_64 && dst_is_xmm128) { + } else if (src_class == RegClass::GPR_64 && dst_is_simd128) { gen->add_instr(IGen::movq_f64_gpr64(*gen, dst_reg, src_reg), irec); - } else if (src_is_xmm128 && dst_class == RegClass::GPR_64) { + } else if (src_is_simd128 && dst_class == RegClass::GPR_64) { gen->add_instr(IGen::movq_gpr64_f64(*gen, dst_reg, src_reg), irec); } else { ASSERT(false); // unhandled move. @@ -201,7 +223,7 @@ void IR_Return::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_Return::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_Return::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } ///////////////////// @@ -230,7 +252,7 @@ void IR_LoadConstant64::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_LoadConstant64::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_LoadConstant64::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } ///////////////////// @@ -256,21 +278,29 @@ void IR_LoadSymbolPointer::do_codegen_x86(emitter::ObjectGenerator* gen, if (m_name == "#f") { static_assert(false_symbol_offset() == 0, "false symbol location"); if (dest_reg.is_xmm(gen->instr_set())) { - gen->add_instr(IGen::movq_f64_gpr64(*gen, dest_reg, gRegInfo.get_st_reg()), irec); + gen->add_instr( + IGen::movq_f64_gpr64(*gen, dest_reg, emitter::reg_info(gen->instr_set()).get_st_reg()), + irec); } else { - gen->add_instr(IGen::mov_gpr64_gpr64(*gen, dest_reg, gRegInfo.get_st_reg()), irec); + gen->add_instr( + IGen::mov_gpr64_gpr64(*gen, dest_reg, emitter::reg_info(gen->instr_set()).get_st_reg()), + irec); } } else if (m_name == "#t") { - gen->add_instr(IGen::lea_reg_plus_off8(*gen, dest_reg, gRegInfo.get_st_reg(), - true_symbol_offset(gen->version())), - irec); + gen->add_instr( + IGen::lea_reg_plus_off8(*gen, dest_reg, emitter::reg_info(gen->instr_set()).get_st_reg(), + true_symbol_offset(gen->version())), + irec); } else if (m_name == "_empty_") { - gen->add_instr(IGen::lea_reg_plus_off8(*gen, dest_reg, gRegInfo.get_st_reg(), - empty_pair_offset_from_s7(gen->version())), - irec); + gen->add_instr( + IGen::lea_reg_plus_off8(*gen, dest_reg, emitter::reg_info(gen->instr_set()).get_st_reg(), + empty_pair_offset_from_s7(gen->version())), + irec); } else { auto instr = gen->add_instr( - IGen::lea_reg_plus_off32(*gen, dest_reg, gRegInfo.get_st_reg(), 0x0afecafe), irec); + IGen::lea_reg_plus_off32(*gen, dest_reg, emitter::reg_info(gen->instr_set()).get_st_reg(), + 0x0afecafe), + irec); gen->link_instruction_symbol_ptr(instr, m_name); } } @@ -278,7 +308,29 @@ void IR_LoadSymbolPointer::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_LoadSymbolPointer::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_LoadSymbolPointer::do_codegen_arm64"); + auto dest_reg = get_reg(m_dest, allocs, irec); + const auto& st = emitter::reg_info(gen->instr_set()).get_st_reg(); + if (m_name == "#f") { + static_assert(false_symbol_offset() == 0, "#f symbol offset must be zero"); + // use the SIMD register class for an ARM64 #f destination + if (dest_reg.is_128bit_simd(gen->instr_set())) { + gen->add_instr(IGen::movq_f64_gpr64(*gen, dest_reg, st), irec); + } else { + gen->add_instr(IGen::mov_gpr64_gpr64(*gen, dest_reg, st), irec); + } + } else if (m_name == "#t") { + gen->add_instr(IGen::lea_reg_plus_off8(*gen, dest_reg, st, true_symbol_offset(gen->version())), + irec); + } else if (m_name == "_empty_") { + gen->add_instr( + IGen::lea_reg_plus_off8(*gen, dest_reg, st, empty_pair_offset_from_s7(gen->version())), + irec); + } else { + // build the signed link-time offset in a movz/movk pair before adding s7 + auto instr = gen->add_instr(IGen::ARM64::mov_gpr32_link_imm32(emitter::X16, 0x0afecafe), irec); + gen->link_instruction_symbol_ptr(instr, m_name); + gen->add_instr(IGen::ARM64::add_gpr64_gpr64_sxtw(dest_reg, st, emitter::X16), irec); + } } ///////////////////// @@ -304,7 +356,8 @@ void IR_SetSymbolValue::do_codegen_x86(emitter::ObjectGenerator* gen, auto src_reg = get_reg(m_src, allocs, irec); auto instr = gen->add_instr( IGen::store32_gpr64_gpr64_plus_gpr64_plus_s32( - *gen, gRegInfo.get_st_reg(), gRegInfo.get_offset_reg(), src_reg, LINK_SYM_NO_OFFSET_FLAG), + *gen, emitter::reg_info(gen->instr_set()).get_st_reg(), + emitter::reg_info(gen->instr_set()).get_offset_reg(), src_reg, LINK_SYM_NO_OFFSET_FLAG), irec); gen->link_instruction_symbol_mem(instr, m_dest->name()); } @@ -312,7 +365,17 @@ void IR_SetSymbolValue::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_SetSymbolValue::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_SetSymbolValue::do_codegen_arm64"); + auto src_reg = get_reg(m_src, allocs, irec); + const auto& info = emitter::reg_info(gen->instr_set()); + // add the linked symbol offset to s7 in x16 + auto instr = gen->add_instr( + IGen::ARM64::mov_gpr32_link_imm32(emitter::X16, LINK_SYM_NO_OFFSET_FLAG), irec); + gen->link_instruction_symbol_mem(instr, m_dest->name()); + gen->add_instr(IGen::ARM64::add_gpr64_gpr64_sxtw(emitter::X16, info.get_st_reg(), emitter::X16), + irec); + gen->add_instr( + IGen::store32_gpr64_gpr64_plus_gpr64(*gen, info.get_offset_reg(), emitter::X16, src_reg), + irec); } ///////////////////// @@ -337,16 +400,18 @@ void IR_GetSymbolValue::do_codegen_x86(emitter::ObjectGenerator* gen, emitter::IR_Record irec) { auto dst_reg = get_reg(m_dest, allocs, irec); if (m_sext) { - auto instr = gen->add_instr(IGen::load32s_gpr64_gpr64_plus_gpr64_plus_s32( - *gen, dst_reg, gRegInfo.get_st_reg(), gRegInfo.get_offset_reg(), - LINK_SYM_NO_OFFSET_FLAG), - irec); + auto instr = gen->add_instr( + IGen::load32s_gpr64_gpr64_plus_gpr64_plus_s32( + *gen, dst_reg, emitter::reg_info(gen->instr_set()).get_st_reg(), + emitter::reg_info(gen->instr_set()).get_offset_reg(), LINK_SYM_NO_OFFSET_FLAG), + irec); gen->link_instruction_symbol_mem(instr, m_src->name()); } else { - auto instr = gen->add_instr(IGen::load32u_gpr64_gpr64_plus_gpr64_plus_s32( - *gen, dst_reg, gRegInfo.get_st_reg(), gRegInfo.get_offset_reg(), - LINK_SYM_NO_OFFSET_FLAG), - irec); + auto instr = gen->add_instr( + IGen::load32u_gpr64_gpr64_plus_gpr64_plus_s32( + *gen, dst_reg, emitter::reg_info(gen->instr_set()).get_st_reg(), + emitter::reg_info(gen->instr_set()).get_offset_reg(), LINK_SYM_NO_OFFSET_FLAG), + irec); gen->link_instruction_symbol_mem(instr, m_src->name()); } } @@ -354,7 +419,7 @@ void IR_GetSymbolValue::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_GetSymbolValue::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_GetSymbolValue::do_codegen_arm64"); + arm64_load_symbol_value(gen, irec, get_reg(m_dest, allocs, irec), m_src->name(), m_sext); } ///////////////////// @@ -382,7 +447,7 @@ void IR_RegSet::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_RegSet::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_RegSet::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } std::string IR_RegSet::print() { @@ -418,14 +483,13 @@ void IR_GotoLabel::do_codegen_x86(emitter::ObjectGenerator* gen, emitter::IR_Record irec) { (void)allocs; auto instr = gen->add_instr(IGen::jmp_imm(*gen), irec); - // TODO ARM - have to patch this differently, encoding for the immediate is different gen->link_instruction_jump(instr, gen->get_future_ir_record_in_same_func(irec, m_dest->idx)); } void IR_GotoLabel::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_GotoLabel::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } void IR_GotoLabel::resolve(const Label* dest) { @@ -461,6 +525,7 @@ std::string IR_FunctionCall::print() { RegAllocInstr IR_FunctionCall::to_rai() { RegAllocInstr rai; + rai.is_call = true; rai.read.push_back(m_func->ireg()); rai.write.push_back(m_func->ireg()); // todo, can we avoid this? rai.write.push_back(m_ret->ireg()); @@ -468,13 +533,6 @@ RegAllocInstr IR_FunctionCall::to_rai() { rai.read.push_back(arg->ireg()); } - for (int i = 0; i < emitter::RegisterInfo::N_REGS; i++) { - auto& info = emitter::gRegInfo.get_info(i); - if (info.temp()) { - rai.clobber.emplace_back(i); - } - } - return rai; } @@ -500,7 +558,9 @@ void IR_FunctionCall::do_codegen_x86(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { auto freg = get_reg(m_func, allocs, irec); - gen->add_instr(IGen::add_gpr64_gpr64(*gen, freg, emitter::gRegInfo.get_offset_reg()), irec); + gen->add_instr( + IGen::add_gpr64_gpr64(*gen, freg, emitter::reg_info(gen->instr_set()).get_offset_reg()), + irec); gen->add_instr(IGen::call_r64(*gen, freg), irec); // todo, can we do a sub to undo the modification to the register? does that actually work? } @@ -508,7 +568,12 @@ void IR_FunctionCall::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_FunctionCall::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_FunctionCall::do_codegen_arm64"); + // add the executable EE memory base before branching + auto freg = get_reg(m_func, allocs, irec); + gen->add_instr( + IGen::add_gpr64_gpr64(*gen, freg, emitter::reg_info(gen->instr_set()).get_exec_base_reg()), + irec); + gen->add_instr(IGen::call_r64(*gen, freg), irec); } ///////////////////// @@ -536,13 +601,20 @@ void IR_RegValAddr::do_codegen_x86(emitter::ObjectGenerator* gen, // x86 pointer to var gen->add_instr(IGen::lea_reg_plus_off(*gen, dst, RSP, stack_offset), irec); // x86 -> GOAL pointer - gen->add_instr(IGen::sub_gpr64_gpr64(*gen, dst, emitter::gRegInfo.get_offset_reg()), irec); + gen->add_instr( + IGen::sub_gpr64_gpr64(*gen, dst, emitter::reg_info(gen->instr_set()).get_offset_reg()), irec); } void IR_RegValAddr::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_RegValAddr::do_codegen_arm64"); + int stack_offset = get_stack_offset(m_src, allocs); + auto dst = get_reg(m_dest, allocs, irec); + // build the native stack slot address with the SP-compatible ADD encoding + gen->add_instr(IGen::lea_reg_plus_off(*gen, dst, emitter::ARM64_REG::SP, stack_offset), irec); + // convert the native address to a GOAL pointer + gen->add_instr( + IGen::sub_gpr64_gpr64(*gen, dst, emitter::reg_info(gen->instr_set()).get_offset_reg()), irec); } ///////////////////// @@ -568,13 +640,17 @@ void IR_StaticVarAddr::do_codegen_x86(emitter::ObjectGenerator* gen, auto dr = get_reg(m_dest, allocs, irec); auto instr = gen->add_instr(IGen::static_addr(*gen, dr, 0), irec); gen->link_instruction_static(instr, m_src->rec, m_src->get_addr_offset()); - gen->add_instr(IGen::sub_gpr64_gpr64(*gen, dr, emitter::gRegInfo.get_offset_reg()), irec); + gen->add_instr( + IGen::sub_gpr64_gpr64(*gen, dr, emitter::reg_info(gen->instr_set()).get_offset_reg()), irec); } void IR_StaticVarAddr::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_StaticVarAddr::do_codegen_arm64"); + // linker writes a GOAL address directly + auto dr = get_reg(m_dest, allocs, irec); + auto instr = gen->add_instr(IGen::ARM64::mov_gpr32_link_imm32(dr, 0), irec); + gen->link_instruction_static(instr, m_src->rec, m_src->get_addr_offset()); } ///////////////////// @@ -599,13 +675,16 @@ void IR_FunctionAddr::do_codegen_x86(emitter::ObjectGenerator* gen, auto dr = get_reg(m_dest, allocs, irec); auto instr = gen->add_instr(IGen::static_addr(*gen, dr, 0), irec); gen->link_instruction_to_function(instr, gen->get_existing_function_record(m_src->idx_in_file)); - gen->add_instr(IGen::sub_gpr64_gpr64(*gen, dr, emitter::gRegInfo.get_offset_reg()), irec); + gen->add_instr( + IGen::sub_gpr64_gpr64(*gen, dr, emitter::reg_info(gen->instr_set()).get_offset_reg()), irec); } void IR_FunctionAddr::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_FunctionAddr::do_codegen_arm64"); + auto dr = get_reg(m_dest, allocs, irec); + auto instr = gen->add_instr(IGen::ARM64::mov_gpr32_link_imm32(dr, 0), irec); + gen->link_instruction_to_function(instr, gen->get_existing_function_record(m_src->idx_in_file)); } ///////////////////// @@ -613,11 +692,24 @@ void IR_FunctionAddr::do_codegen_arm64(emitter::ObjectGenerator* gen, ///////////////////// IR_IntegerMath::IR_IntegerMath(IntegerMathKind kind, RegVal* dest, RegVal* arg) - : m_kind(kind), m_dest(dest), m_arg(arg) {} + : m_kind(kind), m_dest(dest), m_arg(arg) { + ASSERT_MSG(!is_division(kind), "Division IR requires an instruction set"); +} + +IR_IntegerMath::IR_IntegerMath(IntegerMathKind kind, + RegVal* dest, + RegVal* arg, + emitter::InstructionSet instr_set) + : m_kind(kind), m_dest(dest), m_arg(arg), m_instr_set(instr_set) {} IR_IntegerMath::IR_IntegerMath(IntegerMathKind kind, RegVal* dest, u8 shift_amount) : m_kind(kind), m_dest(dest), m_shift_amount(shift_amount) {} +bool IR_IntegerMath::is_division(IntegerMathKind kind) { + return kind == IntegerMathKind::IDIV_32 || kind == IntegerMathKind::IMOD_32 || + kind == IntegerMathKind::UDIV_32 || kind == IntegerMathKind::UMOD_32; +} + std::string IR_IntegerMath::print() { switch (m_kind) { case IntegerMathKind::ADD_64: @@ -671,8 +763,8 @@ RegAllocInstr IR_IntegerMath::to_rai() { rai.read.push_back(m_arg->ireg()); } - if (m_kind == IntegerMathKind::IDIV_32 || m_kind == IntegerMathKind::IMOD_32 || - m_kind == IntegerMathKind::UDIV_32 || m_kind == IntegerMathKind::UMOD_32) { + // reserve RDX only for x86 division + if (is_division(m_kind) && m_instr_set == emitter::InstructionSet::X86) { rai.exclude.emplace_back(emitter::RDX); } return rai; @@ -712,20 +804,15 @@ void IR_IntegerMath::do_codegen_x86(emitter::ObjectGenerator* gen, ASSERT(!m_arg); break; case IntegerMathKind::SHLV_64: - // TODO ARM - register provided but unused on x86 + // x86 variable shifts always use cl. gen->add_instr(IGen::shl_gpr64_reg(*gen, get_reg(m_dest, allocs, irec), 0), irec); - // TODO ARM - x86 forces you to use CL, which is dumb, but the register allocator - // has that logic baked in somewhere - // ARM has no such constraint, so we should be able to use any register for the shift amount ASSERT(get_reg(m_arg, allocs, irec) == emitter::RCX); break; case IntegerMathKind::SHRV_64: - // TODO ARM - register provided but unused on x86 gen->add_instr(IGen::shr_gpr64_reg(*gen, get_reg(m_dest, allocs, irec), 0), irec); ASSERT(get_reg(m_arg, allocs, irec) == emitter::RCX); break; case IntegerMathKind::SARV_64: - // TODO ARM - register provided but unused on x86 gen->add_instr(IGen::sar_gpr64_reg(*gen, get_reg(m_dest, allocs, irec), 0), irec); ASSERT(get_reg(m_arg, allocs, irec) == emitter::RCX); break; @@ -783,7 +870,64 @@ void IR_IntegerMath::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_IntegerMath::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_IntegerMath::do_codegen_arm64"); + switch (m_kind) { + case IntegerMathKind::ADD_64: + case IntegerMathKind::SUB_64: + case IntegerMathKind::AND_64: + case IntegerMathKind::OR_64: + case IntegerMathKind::XOR_64: + case IntegerMathKind::NOT_64: + case IntegerMathKind::SHL_64: + case IntegerMathKind::SHR_64: + case IntegerMathKind::SAR_64: + case IntegerMathKind::IMUL_32: + case IntegerMathKind::IMUL_64: + do_codegen_x86(gen, allocs, irec); + break; + + // ARM64 variable shifts accept any allocated GPR. + case IntegerMathKind::SHLV_64: + gen->add_instr( + IGen::shl_gpr64_reg(*gen, get_reg(m_dest, allocs, irec), get_reg(m_arg, allocs, irec)), + irec); + break; + case IntegerMathKind::SHRV_64: + gen->add_instr( + IGen::shr_gpr64_reg(*gen, get_reg(m_dest, allocs, irec), get_reg(m_arg, allocs, irec)), + irec); + break; + case IntegerMathKind::SARV_64: + gen->add_instr( + IGen::sar_gpr64_reg(*gen, get_reg(m_dest, allocs, irec), get_reg(m_arg, allocs, irec)), + irec); + break; + + // ARM64 uses SDIV or UDIV for the quotient and MSUB for the remainder. + case IntegerMathKind::IDIV_32: + case IntegerMathKind::UDIV_32: + case IntegerMathKind::IMOD_32: + case IntegerMathKind::UMOD_32: { + auto dest = get_reg(m_dest, allocs, irec); + auto arg = get_reg(m_arg, allocs, irec); + bool is_signed = m_kind == IntegerMathKind::IDIV_32 || m_kind == IntegerMathKind::IMOD_32; + bool wants_remainder = + m_kind == IntegerMathKind::IMOD_32 || m_kind == IntegerMathKind::UMOD_32; + + // keep the dividend in dest while computing the remainder + auto quotient = wants_remainder ? Register(emitter::X16) : dest; + gen->add_instr(is_signed ? IGen::ARM64::sdiv_gpr32(quotient, dest, arg) + : IGen::ARM64::udiv_gpr32(quotient, dest, arg), + irec); + if (wants_remainder) { + // dest = dest - quotient * arg + gen->add_instr(IGen::ARM64::msub_gpr32(dest, quotient, arg, dest), irec); + } + gen->add_instr(IGen::movsx_r64_r32(*gen, dest, dest), irec); + } break; + + default: + ASSERT(false); + } } ///////////////////// @@ -870,7 +1014,7 @@ void IR_FloatMath::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_FloatMath::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_FloatMath::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } ///////////////////// @@ -918,7 +1062,29 @@ void IR_StaticVarLoad::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_StaticVarLoad::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_StaticVarLoad::do_codegen_arm64"); + auto load_info = m_src->get_load_info(); + ASSERT(m_src->get_addr_offset() == 0); + const auto& info = emitter::reg_info(gen->instr_set()); + + // load the static through the writable EE memory base + auto addr = gen->add_instr(IGen::ARM64::mov_gpr32_link_imm32(emitter::X16, 0), irec); + gen->link_instruction_static(addr, m_src->rec, 0); + + if (m_dest->ireg().reg_class == RegClass::FLOAT) { + ASSERT(load_info.load_signed == false); + ASSERT(load_info.load_size == 4); + ASSERT(load_info.requires_load == true); + gen->add_instr(IGen::load32_simd32_gpr64_plus_gpr64(*gen, get_reg(m_dest, allocs, irec), + info.get_offset_reg(), emitter::X16), + irec); + } else if (m_dest->ireg().reg_class == RegClass::VECTOR_FLOAT) { + // load the vector structure without scalar load metadata + gen->add_instr(IGen::loadvf_gpr64_plus_gpr64(*gen, get_reg(m_dest, allocs, irec), + info.get_offset_reg(), emitter::X16), + irec); + } else { + ASSERT(false); + } } ///////////////////// @@ -1028,7 +1194,47 @@ void IR_ConditionalBranch::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_ConditionalBranch::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_ConditionalBranch::do_codegen_arm64"); + ASSERT(m_resolved); + + // FCMP uses signed conditions for COMISS-compatible comparisons. + bool use_signed = condition.is_signed || condition.is_float; + + Instruction jump_instr = InstructionARM64(0); + switch (condition.kind) { + case ConditionKind::EQUAL: + jump_instr = IGen::je_imm(*gen); + break; + case ConditionKind::NOT_EQUAL: + jump_instr = IGen::jne_imm(*gen); + break; + case ConditionKind::LEQ: + jump_instr = use_signed ? IGen::jle_imm(*gen) : IGen::jbe_imm(*gen); + break; + case ConditionKind::GEQ: + jump_instr = use_signed ? IGen::jge_imm(*gen) : IGen::jae_imm(*gen); + break; + case ConditionKind::LT: + jump_instr = use_signed ? IGen::jl_imm(*gen) : IGen::jb_imm(*gen); + break; + case ConditionKind::GT: + jump_instr = use_signed ? IGen::jg_imm(*gen) : IGen::ja_imm(*gen); + break; + default: + ASSERT(false); + } + + if (condition.is_float) { + gen->add_instr(IGen::cmp_f32_f32(*gen, get_reg(condition.a, allocs, irec), + get_reg(condition.b, allocs, irec)), + irec); + } else { + gen->add_instr(IGen::cmp_gpr64_gpr64(*gen, get_reg(condition.a, allocs, irec), + get_reg(condition.b, allocs, irec)), + irec); + } + + auto jump_rec = gen->add_instr(jump_instr, irec); + gen->link_instruction_jump(jump_rec, gen->get_future_ir_record_in_same_func(irec, label.idx)); } ///////////////////// @@ -1060,21 +1266,24 @@ void IR_LoadConstOffset::do_codegen_x86(emitter::ObjectGenerator* gen, auto base_reg = m_use_coloring ? get_reg(m_base, allocs, irec) : get_no_color_reg(m_base); if (m_dest->ireg().reg_class == RegClass::GPR_64) { - gen->add_instr(IGen::load_goal_gpr(*gen, dest_reg, base_reg, emitter::gRegInfo.get_offset_reg(), + gen->add_instr(IGen::load_goal_gpr(*gen, dest_reg, base_reg, + emitter::reg_info(gen->instr_set()).get_offset_reg(), m_offset, m_info.size, m_info.sign_extend), irec); } else if (m_dest->ireg().reg_class == RegClass::FLOAT && m_info.size == 4 && m_info.sign_extend == false && m_info.reg == RegClass::FLOAT) { - gen->add_instr(IGen::load_goal_xmm32(*gen, dest_reg, base_reg, - emitter::gRegInfo.get_offset_reg(), m_offset), - irec); + gen->add_instr( + IGen::load_goal_simd32(*gen, dest_reg, base_reg, + emitter::reg_info(gen->instr_set()).get_offset_reg(), m_offset), + irec); } else if ((m_dest->ireg().reg_class == RegClass::VECTOR_FLOAT || m_dest->ireg().reg_class == RegClass::INT_128) && m_info.size == 16 && m_info.sign_extend == false && m_info.reg == m_dest->ireg().reg_class) { - gen->add_instr(IGen::load_goal_xmm128(*gen, dest_reg, base_reg, - emitter::gRegInfo.get_offset_reg(), m_offset), - irec); + gen->add_instr( + IGen::load_goal_simd128(*gen, dest_reg, base_reg, + emitter::reg_info(gen->instr_set()).get_offset_reg(), m_offset), + irec); } else { throw std::runtime_error("IR_LoadConstOffset::do_codegen_x86 not supported"); } @@ -1083,7 +1292,7 @@ void IR_LoadConstOffset::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_LoadConstOffset::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_LoadConstOffset::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1115,18 +1324,21 @@ void IR_StoreConstOffset::do_codegen_x86(emitter::ObjectGenerator* gen, if (m_value->ireg().reg_class == RegClass::GPR_64) { gen->add_instr(IGen::store_goal_gpr(*gen, base_reg, value_reg, - emitter::gRegInfo.get_offset_reg(), m_offset, m_size), + emitter::reg_info(gen->instr_set()).get_offset_reg(), + m_offset, m_size), irec); } else if (m_value->ireg().reg_class == RegClass::FLOAT && m_size == 4) { - gen->add_instr(IGen::store_goal_xmm32(*gen, base_reg, value_reg, - emitter::gRegInfo.get_offset_reg(), m_offset), - irec); + gen->add_instr( + IGen::store_goal_simd32(*gen, base_reg, value_reg, + emitter::reg_info(gen->instr_set()).get_offset_reg(), m_offset), + irec); } else if ((m_value->ireg().reg_class == RegClass::VECTOR_FLOAT || m_value->ireg().reg_class == RegClass::INT_128) && m_size == 16) { - gen->add_instr(IGen::store_goal_vf(*gen, base_reg, value_reg, - emitter::gRegInfo.get_offset_reg(), m_offset), - irec); + gen->add_instr( + IGen::store_goal_vf(*gen, base_reg, value_reg, + emitter::reg_info(gen->instr_set()).get_offset_reg(), m_offset), + irec); } else { throw std::runtime_error( fmt::format("IR_StoreConstOffset::do_codegen_x86 can't handle this (c {} sz {})", @@ -1137,7 +1349,7 @@ void IR_StoreConstOffset::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_StoreConstOffset::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_StoreConstOffset::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1162,7 +1374,7 @@ void IR_Null::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_Null::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_Null::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1193,7 +1405,7 @@ void IR_ValueReset::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_ValueReset::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_ValueReset::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1226,7 +1438,7 @@ void IR_FloatToInt::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_FloatToInt::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_FloatToInt::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1256,7 +1468,7 @@ void IR_IntToFloat::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_IntToFloat::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_IntToFloat::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1283,19 +1495,30 @@ void IR_GetStackAddr::do_codegen_x86(emitter::ObjectGenerator* gen, if (offset == 0) { gen->add_instr(IGen::mov_gpr64_gpr64(*gen, dest_reg, RSP), irec); - gen->add_instr(IGen::sub_gpr64_gpr64(*gen, dest_reg, gRegInfo.get_offset_reg()), irec); + gen->add_instr( + IGen::sub_gpr64_gpr64(*gen, dest_reg, emitter::reg_info(gen->instr_set()).get_offset_reg()), + irec); } else { // dest = offset + RSP gen->add_instr(IGen::lea_reg_plus_off(*gen, dest_reg, RSP, offset), irec); // dest = offset + RSP - offset - gen->add_instr(IGen::sub_gpr64_gpr64(*gen, dest_reg, gRegInfo.get_offset_reg()), irec); + gen->add_instr( + IGen::sub_gpr64_gpr64(*gen, dest_reg, emitter::reg_info(gen->instr_set()).get_offset_reg()), + irec); } } void IR_GetStackAddr::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_GetStackAddr::do_codegen_arm64"); + auto dest_reg = get_reg(m_dest, allocs, irec); + int offset = GPR_SIZE * allocs.get_slot_for_var(m_slot); + + // use the SP-compatible ADD encoding for zero offsets + gen->add_instr(IGen::lea_reg_plus_off(*gen, dest_reg, emitter::ARM64_REG::SP, offset), irec); + gen->add_instr( + IGen::sub_gpr64_gpr64(*gen, dest_reg, emitter::reg_info(gen->instr_set()).get_offset_reg()), + irec); } /////////////////////// @@ -1321,7 +1544,7 @@ void IR_Nop::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_Nop::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_Nop::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1362,7 +1585,34 @@ void IR_AsmRet::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_AsmRet::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_AsmRet::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); +} + +/////////////////////// +// AsmBreak +/////////////////////// + +IR_AsmBreak::IR_AsmBreak() : IR_Asm(false) {} + +std::string IR_AsmBreak::print() { + return ".break"; +} + +RegAllocInstr IR_AsmBreak::to_rai() { + return {}; +} + +void IR_AsmBreak::do_codegen_x86(emitter::ObjectGenerator* gen, + const AllocationResult& allocs, + emitter::IR_Record irec) { + (void)allocs; + gen->add_instr(IGen::trap(*gen), irec); +} + +void IR_AsmBreak::do_codegen_arm64(emitter::ObjectGenerator* gen, + const AllocationResult& allocs, + emitter::IR_Record irec) { + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1389,7 +1639,7 @@ void IR_AsmFNop::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_AsmFNop::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_AsmFNop::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1416,7 +1666,7 @@ void IR_AsmFWait::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_AsmFWait::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_AsmFWait::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1450,7 +1700,8 @@ void IR_AsmPush::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_AsmPush::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_AsmPush::do_codegen_arm64"); + // ARM64 pushes one GPR in a 16-byte-aligned stack slot. + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1484,7 +1735,8 @@ void IR_AsmPop::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_AsmPop::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_AsmPop::do_codegen_arm64"); + // ARM64 pops one GPR from a 16-byte-aligned stack slot. + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1524,7 +1776,7 @@ void IR_AsmSub::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_AsmSub::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_AsmSub::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1564,7 +1816,7 @@ void IR_AsmAdd::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_AsmAdd::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_AsmAdd::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1594,16 +1846,18 @@ void IR_GetSymbolValueAsm::do_codegen_x86(emitter::ObjectGenerator* gen, emitter::IR_Record irec) { auto dst_reg = m_use_coloring ? get_reg(m_dest, allocs, irec) : get_no_color_reg(m_dest); if (m_sext) { - auto instr = gen->add_instr(IGen::load32s_gpr64_gpr64_plus_gpr64_plus_s32( - *gen, dst_reg, gRegInfo.get_st_reg(), gRegInfo.get_offset_reg(), - LINK_SYM_NO_OFFSET_FLAG), - irec); + auto instr = gen->add_instr( + IGen::load32s_gpr64_gpr64_plus_gpr64_plus_s32( + *gen, dst_reg, emitter::reg_info(gen->instr_set()).get_st_reg(), + emitter::reg_info(gen->instr_set()).get_offset_reg(), LINK_SYM_NO_OFFSET_FLAG), + irec); gen->link_instruction_symbol_mem(instr, m_sym_name); } else { - auto instr = gen->add_instr(IGen::load32u_gpr64_gpr64_plus_gpr64_plus_s32( - *gen, dst_reg, gRegInfo.get_st_reg(), gRegInfo.get_offset_reg(), - LINK_SYM_NO_OFFSET_FLAG), - irec); + auto instr = gen->add_instr( + IGen::load32u_gpr64_gpr64_plus_gpr64_plus_s32( + *gen, dst_reg, emitter::reg_info(gen->instr_set()).get_st_reg(), + emitter::reg_info(gen->instr_set()).get_offset_reg(), LINK_SYM_NO_OFFSET_FLAG), + irec); gen->link_instruction_symbol_mem(instr, m_sym_name); } } @@ -1611,7 +1865,8 @@ void IR_GetSymbolValueAsm::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_GetSymbolValueAsm::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_GetSymbolValueAsm::do_codegen_arm64"); + auto dst_reg = m_use_coloring ? get_reg(m_dest, allocs, irec) : get_no_color_reg(m_dest); + arm64_load_symbol_value(gen, irec, dst_reg, m_sym_name, m_sext); } /////////////////////// @@ -1642,7 +1897,7 @@ void IR_JumpReg::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_JumpReg::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_JumpReg::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1674,7 +1929,7 @@ void IR_RegSetAsm::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_RegSetAsm::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_RegSetAsm::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1766,7 +2021,7 @@ void IR_VFMath3Asm::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_VFMath3Asm::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_VFMath3Asm::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -1944,7 +2199,7 @@ void IR_Int128Math3Asm::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_Int128Math3Asm::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_Int128Math3Asm::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -2001,7 +2256,7 @@ void IR_VFMath2Asm::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_VFMath2Asm::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_VFMath2Asm::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } /////////////////////// @@ -2149,7 +2404,8 @@ void IR_Int128Math2Asm::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_Int128Math2Asm::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_Int128Math2Asm::do_codegen_arm64"); + // IGen dispatches VPSHUFLW and VPSHUFHW to ARM64 encoders. + do_codegen_x86(gen, allocs, irec); } // ---- Blend VF @@ -2188,7 +2444,7 @@ void IR_BlendVF::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_BlendVF::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_BlendVF::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } // ----- Splat VF @@ -2224,7 +2480,7 @@ void IR_SplatVF::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_SplatVF::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_SplatVF::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } // ---- Swizzle VF @@ -2260,7 +2516,7 @@ void IR_SwizzleVF::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_SwizzleVF::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_SwizzleVF::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } // ---- Square Root VF @@ -2293,5 +2549,5 @@ void IR_SqrtVF::do_codegen_x86(emitter::ObjectGenerator* gen, void IR_SqrtVF::do_codegen_arm64(emitter::ObjectGenerator* gen, const AllocationResult& allocs, emitter::IR_Record irec) { - throw std::runtime_error("NYI - IR_SqrtVF::do_codegen_arm64"); + do_codegen_x86(gen, allocs, irec); } diff --git a/goalc/compiler/IR.h b/goalc/compiler/IR.h index 9a098864e9..5d0b027bd0 100644 --- a/goalc/compiler/IR.h +++ b/goalc/compiler/IR.h @@ -249,7 +249,13 @@ enum class IntegerMathKind { class IR_IntegerMath : public IR { public: IR_IntegerMath(IntegerMathKind kind, RegVal* dest, RegVal* arg); + // division register requirements depend on the target instruction set + IR_IntegerMath(IntegerMathKind kind, + RegVal* dest, + RegVal* arg, + emitter::InstructionSet instr_set); IR_IntegerMath(IntegerMathKind kind, RegVal* dest, u8 shift_amount); + static bool is_division(IntegerMathKind kind); std::string print() override; RegAllocInstr to_rai() override; void do_codegen_x86(emitter::ObjectGenerator* gen, @@ -265,6 +271,7 @@ class IR_IntegerMath : public IR { RegVal* m_dest; RegVal* m_arg = nullptr; u8 m_shift_amount = 0; + emitter::InstructionSet m_instr_set = emitter::InstructionSet::X86; }; enum class FloatMathKind { DIV_SS, MUL_SS, ADD_SS, SUB_SS, MIN_SS, MAX_SS, SQRT_SS }; @@ -566,6 +573,19 @@ class IR_AsmAdd : public IR_Asm { const RegVal* m_src = nullptr; }; +class IR_AsmBreak : public IR_Asm { + public: + IR_AsmBreak(); + std::string print() override; + RegAllocInstr to_rai() override; + void do_codegen_x86(emitter::ObjectGenerator* gen, + const AllocationResult& allocs, + emitter::IR_Record irec) override; + void do_codegen_arm64(emitter::ObjectGenerator* gen, + const AllocationResult& allocs, + emitter::IR_Record irec) override; +}; + class IR_AsmFNop : public IR_Asm { public: IR_AsmFNop(); diff --git a/goalc/compiler/StaticObject.cpp b/goalc/compiler/StaticObject.cpp index 76ed059f81..5bdb7e49a7 100644 --- a/goalc/compiler/StaticObject.cpp +++ b/goalc/compiler/StaticObject.cpp @@ -59,7 +59,7 @@ StaticObject::LoadInfo StaticFloat::get_load_info() const { info.requires_load = true; info.load_size = 4; info.load_signed = false; - info.prefer_xmm = true; + info.prefer_simd = true; return info; } @@ -90,7 +90,7 @@ std::string StaticStructure::print() const { StaticObject::LoadInfo StaticStructure::get_load_info() const { LoadInfo info; info.requires_load = false; - info.prefer_xmm = false; + info.prefer_simd = false; return info; } diff --git a/goalc/compiler/StaticObject.h b/goalc/compiler/StaticObject.h index 26cd318b68..03ebd4b07a 100644 --- a/goalc/compiler/StaticObject.h +++ b/goalc/compiler/StaticObject.h @@ -19,7 +19,7 @@ class StaticObject { bool requires_load = false; int load_size = -1; bool load_signed = false; - bool prefer_xmm = false; + bool prefer_simd = false; }; virtual LoadInfo get_load_info() const = 0; diff --git a/goalc/compiler/Val.cpp b/goalc/compiler/Val.cpp index 2ae85f0281..c5599dd0a9 100644 --- a/goalc/compiler/Val.cpp +++ b/goalc/compiler/Val.cpp @@ -34,9 +34,9 @@ RegVal* Val::to_fpr(const goos::Object& form, Env* fe) { } /*! - * Fallback to_xmm128 if a more optimized one is not provided. + * Fallback to_simd128 if a more optimized one is not provided. */ -RegVal* Val::to_xmm128(const goos::Object& form, Env* fe) { +RegVal* Val::to_simd128(const goos::Object& form, Env* fe) { auto rv = to_reg(form, fe); if (rv->ireg().reg_class == RegClass::INT_128 || rv->ireg().reg_class == RegClass::VECTOR_FLOAT) { return rv; @@ -72,7 +72,7 @@ RegVal* RegVal::to_fpr(const goos::Object& form, Env* fe) { } } -RegVal* RegVal::to_xmm128(const goos::Object& form, Env* fe) { +RegVal* RegVal::to_simd128(const goos::Object& form, Env* fe) { if (m_ireg.reg_class == RegClass::INT_128 || m_ireg.reg_class == RegClass::VECTOR_FLOAT) { return this; } else { @@ -98,19 +98,19 @@ RegVal* IntegerConstantVal::to_reg(const goos::Object& form, Env* fe) { } else { auto rv = fe->make_ireg(m_ts, RegClass::INT_128); auto gpr = fe->make_gpr(TypeSpec("object")); - auto xmm_temp = fe->make_ireg(TypeSpec("object"), RegClass::INT_128); + auto simd_temp = fe->make_ireg(TypeSpec("object"), RegClass::INT_128); fe->emit_ir(form, gpr, m_value.value_128_lo()); - fe->emit_ir(form, xmm_temp, gpr); + fe->emit_ir(form, simd_temp, gpr); fe->emit_ir(form, gpr, m_value.value_128_hi()); fe->emit_ir(form, rv, gpr); - fe->emit_ir(form, true, rv, rv, xmm_temp, IR_Int128Math3Asm::Kind::PCPYLD); + fe->emit_ir(form, true, rv, rv, simd_temp, IR_Int128Math3Asm::Kind::PCPYLD); return rv; } } -RegVal* IntegerConstantVal::to_xmm128(const goos::Object& form, Env* fe) { +RegVal* IntegerConstantVal::to_simd128(const goos::Object& form, Env* fe) { if (m_value.is_zero()) { // if we are a constant 0, can use XOR auto rv = fe->make_ireg(m_ts, RegClass::INT_128); @@ -243,8 +243,8 @@ RegVal* AliasVal::to_reg(const goos::Object& form, Env* fe) { return result; } -RegVal* AliasVal::to_xmm128(const goos::Object& form, Env* fe) { - auto as_old_type = base->to_xmm128(form, fe); +RegVal* AliasVal::to_simd128(const goos::Object& form, Env* fe) { + auto as_old_type = base->to_simd128(form, fe); auto result = fe->make_ireg(m_ts, as_old_type->ireg().reg_class); fe->emit(form, std::make_unique(result, as_old_type)); return result; @@ -293,12 +293,12 @@ RegVal* BitFieldVal::to_reg(const goos::Object& form, Env* env) { env->emit(form, std::make_unique(result, gpr)); } else { // we need to get the value as a 128-bit integer - auto xmm = m_parent->to_reg(form, env); - ASSERT(xmm->ireg().reg_class == RegClass::INT_128); - auto xmm_temp = fe->make_ireg(TypeSpec("object"), RegClass::INT_128); - env->emit_ir(form, true, xmm_temp, xmm, xmm, + auto simd = m_parent->to_reg(form, env); + ASSERT(simd->ireg().reg_class == RegClass::INT_128); + auto simd_temp = fe->make_ireg(TypeSpec("object"), RegClass::INT_128); + env->emit_ir(form, true, simd_temp, simd, simd, IR_Int128Math3Asm::Kind::PCPYUD); - env->emit_ir(form, result, xmm_temp); + env->emit_ir(form, result, simd_temp); start_bit = m_offset - 64; } diff --git a/goalc/compiler/Val.h b/goalc/compiler/Val.h index 3d4c902c79..3c0cbe07aa 100644 --- a/goalc/compiler/Val.h +++ b/goalc/compiler/Val.h @@ -42,7 +42,7 @@ class Val { } virtual RegVal* to_gpr(const goos::Object& form, Env* fe); virtual RegVal* to_fpr(const goos::Object& form, Env* fe); - virtual RegVal* to_xmm128(const goos::Object& form, Env* fe); + virtual RegVal* to_simd128(const goos::Object& form, Env* fe); const TypeSpec& type() const { return m_ts; } void set_type(TypeSpec ts) { m_ts = std::move(ts); } @@ -78,7 +78,7 @@ class RegVal : public Val { RegVal* to_reg(const goos::Object& form, Env* fe) override; RegVal* to_gpr(const goos::Object& form, Env* fe) override; RegVal* to_fpr(const goos::Object& form, Env* fe) override; - RegVal* to_xmm128(const goos::Object& form, Env* fe) override; + RegVal* to_simd128(const goos::Object& form, Env* fe) override; void set_rlet_constraint(emitter::Register reg); const std::optional& rlet_constraint() const; void force_on_stack() { m_on_stack = true; } @@ -241,7 +241,7 @@ class AliasVal : public Val { AliasVal(TypeSpec ts, Val* _base) : Val(std::move(ts)), base(_base) {} std::string print() const override { return "alias-of-" + base->print(); } RegVal* to_reg(const goos::Object& form, Env* fe) override; - RegVal* to_xmm128(const goos::Object& form, Env* fe) override; + RegVal* to_simd128(const goos::Object& form, Env* fe) override; Val* base = nullptr; }; @@ -254,7 +254,7 @@ class IntegerConstantVal : public Val { std::string print() const override { return std::string("integer-constant-") + m_value.print(); } RegVal* to_reg(const goos::Object& form, Env* fe) override; - RegVal* to_xmm128(const goos::Object& form, Env* fe) override; + RegVal* to_simd128(const goos::Object& form, Env* fe) override; const ConstantValue& value() const { return m_value; } protected: @@ -311,4 +311,4 @@ struct ValOrConstant { }; using ValOrConstInt = ValOrConstant; -using ValOrConstFloat = ValOrConstant; \ No newline at end of file +using ValOrConstFloat = ValOrConstant; diff --git a/goalc/compiler/compilation/Asm.cpp b/goalc/compiler/compilation/Asm.cpp index afc959481b..cbc89b746e 100644 --- a/goalc/compiler/compilation/Asm.cpp +++ b/goalc/compiler/compilation/Asm.cpp @@ -1,27 +1,55 @@ #include "goalc/compiler/Compiler.h" -namespace { -const char* reg_names[] = { - "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", "r8", "r9", "r10", - "r11", "r12", "r13", "r14", "r15", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", - "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", -}; - -} - emitter::Register Compiler::parse_register(const goos::Object& code) { if (!code.is_symbol()) { throw_compiler_error(code, "Could not parse {} as a register name", code.print()); } - auto nas = code.as_symbol(); - for (int i = 0; i < 32; i++) { - if (std::string_view(nas.name_ptr) == reg_names[i]) { + const auto& info = emitter::reg_info(m_instr_set); + std::string_view name(code.as_symbol().name_ptr); + + // GOAL role names map to this backend's registers + if (name == "pp" || name == "r13") { + return info.get_process_reg(); + } + if (name == "st" || name == "r14") { + return info.get_st_reg(); + } + if (name == "off" || name == "r15") { + return info.get_offset_reg(); + } + if (name == "sp" || name == "rsp") { + return info.get_stack_reg(); + } + // exec-off is the executable mapping base + if (name == "exec-off") { + return info.get_exec_base_reg(); + } + // lr is x30 on ARM64 + if (name == "lr") { + if (m_instr_set != emitter::InstructionSet::ARM64) { + throw_compiler_error(code, "lr is available only on ARM64"); + } + return emitter::Register(emitter::ARM64_REG::X30); + } + // carg0 through carg7 are the C and GOAL argument registers + if (name.size() == 5 && name.substr(0, 4) == "carg") { + int arg_idx = name[4] - '0'; + if (arg_idx >= 0 && arg_idx < emitter::RegisterInfo::N_ARGS) { + return info.get_gpr_arg_reg(arg_idx); + } + } + + // hardware names come from the selected backend + for (int i = 0; i < emitter::RegisterInfo::N_REGS; i++) { + const auto& hw_name = info.get_info(emitter::Register(i)).name; + if (!hw_name.empty() && name == hw_name) { return emitter::Register(i); } } - throw_compiler_error(code, "Could not parse {} as a register name", code.print()); + throw_compiler_error(code, "{} is not a register name for the {} backend", code.print(), + m_instr_set == emitter::InstructionSet::ARM64 ? "arm64" : "x86"); return {}; } @@ -154,6 +182,23 @@ Val* Compiler::compile_asm_ret(const goos::Object& form, const goos::Object& res return get_none(); } +//! Compile the trap instruction used by GOAL (break). +Val* Compiler::compile_asm_break(const goos::Object& form, const goos::Object& rest, Env* env) { + auto args = get_va(form, rest); + va_check(form, args, {}, {}); + + // Keep the old x86 output. + if (m_instr_set == emitter::InstructionSet::X86) { + return compile_div(form, + goos::build_list(std::vector{goos::Object::make_integer(0), + goos::Object::make_integer(0)}), + env); + } + + env->emit_ir(form); + return get_none(); +} + Val* Compiler::compile_asm_pop(const goos::Object& form, const goos::Object& rest, Env* env) { auto args = get_va(form, rest); va_check(form, args, {{}}, {{"color", {false, goos::ObjectType::SYMBOL}}}); @@ -403,7 +448,7 @@ Val* Compiler::compile_asm_mov_vf(const goos::Object& form, const goos::Object& } auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); + auto src = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); check_vector_float_regs(form, env, {{"destination", dest}, {"source", src}}); u8 mask = 0b1111; @@ -430,8 +475,8 @@ Val* Compiler::compile_asm_blend_vf(const goos::Object& form, const goos::Object } auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); - auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_xmm128(form, env); + auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); + auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_simd128(form, env); check_vector_float_regs(form, env, {{"destination", dest}, {"first source", src1}, {"second source", src2}}); @@ -463,8 +508,8 @@ Val* Compiler::compile_asm_vf_math3(const goos::Object& form, } auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); - auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_xmm128(form, env); + auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); + auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_simd128(form, env); check_vector_float_regs(form, env, {{"destination", dest}, {"first source", src1}, {"second source", src2}}); @@ -524,8 +569,8 @@ Val* Compiler::compile_asm_int128_math3(const goos::Object& form, } auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); - auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_xmm128(form, env); + auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); + auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_simd128(form, env); if (!dest->settable()) { throw_compiler_error(form, "Cannot set destination"); @@ -548,7 +593,7 @@ Val* Compiler::compile_asm_vf_math2(const goos::Object& form, } auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); + auto src = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); check_vector_float_regs(form, env, {{"destination", dest}, {"source", src}}); u8 mask = 0b1111; @@ -588,7 +633,7 @@ Val* Compiler::compile_asm_int128_math2_imm_u8(const goos::Object& form, } auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); + auto src = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); s64 imm = get_constant_integer_or_error(args.unnamed.at(2), env); if (imm < 0 || imm > 255) { @@ -648,8 +693,8 @@ Val* Compiler::compile_asm_pnor(const goos::Object& form, const goos::Object& re va_check(form, args, {{}, {}, {}}, {}); auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); // rs - auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_xmm128(form, env); // rt + auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); // rs + auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_simd128(form, env); // rt auto temp = env->make_ireg(TypeSpec("uint128"), RegClass::INT_128); if (!ireg_is_128_ok(dest->ireg())) { @@ -756,8 +801,8 @@ Val* Compiler::compile_asm_ppach(const goos::Object& form, const goos::Object& r va_check(form, args, {{}, {}, {}}, {}); auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); // rs - auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_xmm128(form, env); // rt + auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); // rs + auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_simd128(form, env); // rt auto temp = env->make_ireg(TypeSpec("uint128"), RegClass::INT_128); if (!dest->settable()) { @@ -781,8 +826,8 @@ Val* Compiler::compile_asm_ppacb(const goos::Object& form, const goos::Object& r va_check(form, args, {{}, {}, {}}, {}); auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); // rs - auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_xmm128(form, env); // rt + auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); // rs + auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_simd128(form, env); // rt auto temp = env->make_ireg(TypeSpec("uint128"), RegClass::INT_128); if (!dest->settable()) { @@ -809,8 +854,8 @@ Val* Compiler::compile_asm_xorp(const goos::Object& form, const goos::Object& re va_check(form, args, {{}, {}, {}}, {}); auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); // rs - auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_xmm128(form, env); // rt + auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); // rs + auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_simd128(form, env); // rt if (!dest->settable()) { throw_compiler_error(form, "Cannot set destination"); @@ -974,15 +1019,15 @@ Val* Compiler::compile_asm_vf_math4_two_operation(const goos::Object& form, } auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); - auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_xmm128(form, env); + auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); + auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_simd128(form, env); // This third register is intended for the ACC/Q/ETC, and is used to temporarily store the value // that eventually goes into the destination // // For example VMADDA: // > ACC += src1 * src2 // > DEST = ACC - auto src3 = compile_error_guard(args.unnamed.at(3), env)->to_xmm128(form, env); + auto src3 = compile_error_guard(args.unnamed.at(3), env)->to_simd128(form, env); check_vector_float_regs(form, env, {{"destination", dest}, {"first source", src1}, @@ -1131,7 +1176,7 @@ Val* Compiler::compile_asm_abs_vf(const goos::Object& form, const goos::Object& } auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); + auto src = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); check_vector_float_regs(form, env, {{"destination", dest}, {"source", src}}); u8 mask = 0b1111; @@ -1208,8 +1253,8 @@ Val* Compiler::compile_asm_div_vf(const goos::Object& form, const goos::Object& } auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); - auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_xmm128(form, env); + auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); + auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_simd128(form, env); check_vector_float_regs(form, env, {{"destination", dest}, {"first source", src1}, {"second source", src2}}); @@ -1257,7 +1302,7 @@ Val* Compiler::compile_asm_sqrt_vf(const goos::Object& form, const goos::Object& } auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); + auto src = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); check_vector_float_regs(form, env, {{"destination", dest}, {"source", src}}); u8 ftf = args.named.at("ftf").as_int(); @@ -1298,8 +1343,8 @@ Val* Compiler::compile_asm_inv_sqrt_vf(const goos::Object& form, } auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); - auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_xmm128(form, env); + auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); + auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_simd128(form, env); check_vector_float_regs(form, env, {{"destination", dest}, {"first source", src1}, {"second source", src2}}); @@ -1339,8 +1384,8 @@ Val* Compiler::compile_asm_outer_product_vf(const goos::Object& form, } auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); - auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_xmm128(form, env); + auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); + auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_simd128(form, env); check_vector_float_regs(form, env, {{"destination", dest}, {"first source", src1}, {"second source", src2}}); @@ -1402,8 +1447,8 @@ Val* Compiler::compile_asm_outer_product_a_vf(const goos::Object& form, } auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); - auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_xmm128(form, env); + auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); + auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_simd128(form, env); check_vector_float_regs(form, env, {{"destination", dest}, {"first source", src1}, {"second source", src2}}); @@ -1444,9 +1489,9 @@ Val* Compiler::compile_asm_outer_product_b_vf(const goos::Object& form, } auto dest = compile_error_guard(args.unnamed.at(0), env)->to_reg(form, env); - auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_xmm128(form, env); - auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_xmm128(form, env); - auto acc = compile_error_guard(args.unnamed.at(3), env)->to_xmm128(form, env); + auto src1 = compile_error_guard(args.unnamed.at(1), env)->to_simd128(form, env); + auto src2 = compile_error_guard(args.unnamed.at(2), env)->to_simd128(form, env); + auto acc = compile_error_guard(args.unnamed.at(3), env)->to_simd128(form, env); check_vector_float_regs(form, env, {{"destination", dest}, {"first source", src1}, diff --git a/goalc/compiler/compilation/Atoms.cpp b/goalc/compiler/compilation/Atoms.cpp index 25a584f508..b8115177fb 100644 --- a/goalc/compiler/compilation/Atoms.cpp +++ b/goalc/compiler/compilation/Atoms.cpp @@ -15,6 +15,8 @@ const std::unordered_map g_goal_forms = { // INLINE ASM {".nop", {.docstring = "A simple no-op, does nothing", .form_function = &Compiler::compile_nop}}, + {".break", + {.docstring = "A trap for the (break) macro", .form_function = &Compiler::compile_asm_break}}, {".ret", {.form_function = &Compiler::compile_asm_ret}}, {".push", {.form_function = &Compiler::compile_asm_push}}, {".pop", {.form_function = &Compiler::compile_asm_pop}}, diff --git a/goalc/compiler/compilation/Block.cpp b/goalc/compiler/compilation/Block.cpp index d9b7d3d539..7ee7bec2e5 100644 --- a/goalc/compiler/compilation/Block.cpp +++ b/goalc/compiler/compilation/Block.cpp @@ -54,7 +54,7 @@ Val* Compiler::compile_block(const goos::Object& form, const goos::Object& _rest // we need to create a return value register, as a "return-from" statement inside the block may // set it. for now it has a type of none, but we will set it after compiling the block. - // TODO - determine if GOAL blocks _always_ return gprs, or if it's possible to return xmms. + // TODO - determine if GOAL blocks _always_ return GPRs, or if they can return SIMD registers. block_env->return_value = env->make_gpr(m_ts.make_typespec("none")); // create label to the end of the block (we don't yet know where it is...) @@ -71,7 +71,7 @@ Val* Compiler::compile_block(const goos::Object& form, const goos::Object& _rest // if no return-from's were used, we can ignore the return_value register, and basically turn this // into a begin. this allows a block which returns a floating point value to return the value in - // an xmm register, which is likely to eliminate a gpr->xmm move. + // a SIMD register, which is likely to eliminate a GPR-to-SIMD move. // TODO - does this happen in GOAL? if (block_env->return_types.empty()) { return result; @@ -90,8 +90,8 @@ Val* Compiler::compile_block(const goos::Object& form, const goos::Object& _rest auto ir_move_rv = std::make_unique(block_env->return_value, result->to_gpr(form, fe)); - // note - one drawback of doing this single pass is that a block always evaluates to a gpr. - // so we may have an unneeded xmm -> gpr move that could have been an xmm -> xmm that could have + // note - one drawback of doing this single pass is that a block always evaluates to a GPR. + // so we may have an unneeded SIMD -> GPR move that could have been SIMD -> SIMD that could have // been eliminated. env->emit(form, std::move(ir_move_rv)); } @@ -193,4 +193,4 @@ Val* Compiler::compile_nop(const goos::Object& form, const goos::Object& rest, E va_check(form, args, {}, {}); env->emit_ir(form); return get_none(); -} \ No newline at end of file +} diff --git a/goalc/compiler/compilation/Define.cpp b/goalc/compiler/compilation/Define.cpp index a2351fb27c..e18cb08fd0 100644 --- a/goalc/compiler/compilation/Define.cpp +++ b/goalc/compiler/compilation/Define.cpp @@ -201,41 +201,41 @@ void Compiler::set_bitfield_128(const goos::Object& form, BitFieldVal* dst, RegV // first, get the value we want to modify: ASSERT(m_ts.lookup_type(dst->parent()->type())->get_preferred_reg_class() == RegClass::INT_128); - RegVal* original_original = dst->parent()->to_xmm128(form, env); + RegVal* original_original = dst->parent()->to_simd128(form, env); - // next, get the 64-bit part we want to modify in the lower 64 bits of an XMM - RegVal* xmm_temp = fe->make_ireg(original_original->type(), RegClass::INT_128); + // next, get the 64-bit part we want to modify in the lower 64 bits of a SIMD register + RegVal* simd_temp = fe->make_ireg(original_original->type(), RegClass::INT_128); if (get_top) { - env->emit_ir(form, true, xmm_temp, original_original, original_original, + env->emit_ir(form, true, simd_temp, original_original, original_original, IR_Int128Math3Asm::Kind::PCPYUD); } else { - env->emit_ir(form, xmm_temp, original_original); + env->emit_ir(form, simd_temp, original_original); } - // convert that xmm to a GPR. + // convert that SIMD value to a GPR. RegVal* gpr_64_section = fe->make_gpr(original_original->type()); - env->emit_ir(form, gpr_64_section, xmm_temp); + env->emit_ir(form, gpr_64_section, simd_temp); // set the bits in the GPR int corrected_offset = get_top ? dst->offset() - 64 : dst->offset(); set_bits_in_bitfield(form, dst->size(), corrected_offset, gpr_64_section, src, fe, env); - // back to xmm - env->emit_ir(form, xmm_temp, gpr_64_section); + // back to SIMD + env->emit_ir(form, simd_temp, gpr_64_section); - // rebuild the xmm + // rebuild the SIMD value if (get_top) { - env->emit_ir(form, true, xmm_temp, xmm_temp, original_original, + env->emit_ir(form, true, simd_temp, simd_temp, original_original, IR_Int128Math3Asm::Kind::PCPYLD); } else { - env->emit_ir(form, true, xmm_temp, xmm_temp, xmm_temp, + env->emit_ir(form, true, simd_temp, simd_temp, simd_temp, IR_Int128Math3Asm::Kind::PCPYLD); - env->emit_ir(form, true, xmm_temp, xmm_temp, original_original, + env->emit_ir(form, true, simd_temp, simd_temp, original_original, IR_Int128Math3Asm::Kind::PCPYUD); } // set - do_set(form, dst->parent(), xmm_temp, xmm_temp, env); + do_set(form, dst->parent(), simd_temp, simd_temp, env); } /*! diff --git a/goalc/compiler/compilation/Function.cpp b/goalc/compiler/compilation/Function.cpp index b3c703c2ab..845d6e6ff9 100644 --- a/goalc/compiler/compilation/Function.cpp +++ b/goalc/compiler/compilation/Function.cpp @@ -186,7 +186,7 @@ Val* Compiler::compile_lambda(const goos::Object& form, const goos::Object& rest for (auto& parm : lambda.params) { arg_types.push_back(parm.type); } - auto arg_regs = get_arg_registers(m_ts, arg_types); + auto arg_regs = get_arg_registers(m_ts, arg_types, m_instr_set); for (u32 i = 0; i < lambda.params.size(); i++) { IRegConstraint constr; @@ -207,7 +207,7 @@ Val* Compiler::compile_lambda(const goos::Object& form, const goos::Object& rest self_var->mark_as_settable(); IRegConstraint constr; constr.contrain_everywhere = true; - constr.desired_register = emitter::gRegInfo.get_process_reg(); + constr.desired_register = emitter::reg_info(m_instr_set).get_process_reg(); constr.ireg = self_var->ireg(); self_var->set_rlet_constraint(constr.desired_register); new_func_env->constrain(constr); @@ -266,10 +266,10 @@ Val* Compiler::compile_lambda(const goos::Object& form, const goos::Object& rest // got a result, so to_gpr it and return it. RegVal* final_result; - emitter::Register ret_hw_reg = emitter::gRegInfo.get_gpr_ret_reg(); + emitter::Register ret_hw_reg = emitter::reg_info(m_instr_set).get_gpr_ret_reg(); if (m_ts.lookup_type(result->type())->get_load_size() == 16) { - ret_hw_reg = emitter::gRegInfo.get_xmm_ret_reg(); - final_result = result->to_xmm128(form, new_func_env.get()); + ret_hw_reg = emitter::reg_info(m_instr_set).get_simd_ret_reg(); + final_result = result->to_simd128(form, new_func_env.get()); return_reg->change_class(RegClass::INT_128); } else { final_result = result->to_gpr(form, new_func_env.get()); @@ -608,7 +608,7 @@ Val* Compiler::compile_real_function_call(const goos::Object& form, return_ts = function->type().last_arg(); } - auto cc = get_function_calling_convention(function->type(), m_ts); + auto cc = get_function_calling_convention(function->type(), m_ts, m_instr_set); RegClass ret_reg_class = RegClass::GPR_64; if (cc.return_reg && cc.return_reg->is_128bit_simd(m_instr_set)) { ret_reg_class = RegClass::INT_128; diff --git a/goalc/compiler/compilation/Math.cpp b/goalc/compiler/compilation/Math.cpp index 5a8b803363..9a36ebeb55 100644 --- a/goalc/compiler/compilation/Math.cpp +++ b/goalc/compiler/compilation/Math.cpp @@ -459,18 +459,23 @@ Val* Compiler::compile_div(const goos::Object& form, const goos::Object& rest, E env->emit_ir(form, IntegerMathKind::SHR_64, result, power_of_two); } } else { - IRegConstraint result_rax_constraint; - result_rax_constraint.instr_idx = fe->code().size(); - result_rax_constraint.ireg = result->ireg(); - result_rax_constraint.desired_register = emitter::RAX; - fe->constrain(result_rax_constraint); + // x86 IDIV needs the quotient in RAX + if (m_instr_set == emitter::InstructionSet::X86) { + IRegConstraint result_rax_constraint; + result_rax_constraint.instr_idx = fe->code().size(); + result_rax_constraint.ireg = result->ireg(); + result_rax_constraint.desired_register = emitter::RAX; + fe->constrain(result_rax_constraint); + } if (is_singed_integer_or_binteger(first_type)) { env->emit_ir(form, IntegerMathKind::IDIV_32, result, - to_math_type(form, val, math_type, env)->to_gpr(form, env)); + to_math_type(form, val, math_type, env)->to_gpr(form, env), + m_instr_set); } else { env->emit_ir(form, IntegerMathKind::UDIV_32, result, - to_math_type(form, val, math_type, env)->to_gpr(form, env)); + to_math_type(form, val, math_type, env)->to_gpr(form, env), + m_instr_set); } auto result_moved = env->make_gpr(first_type); @@ -522,6 +527,8 @@ Val* Compiler::compile_variable_shift(const goos::Object& form, env->emit_ir(form, sa_in, sa); auto fenv = env->function_env(); + // x86 variable shifts need RCX + bool needs_shift_amount_in_rcx = m_instr_set == emitter::InstructionSet::X86; IRegConstraint sa_con; sa_con.ireg = sa_in->ireg(); sa_con.instr_idx = fenv->code().size(); @@ -532,7 +539,9 @@ Val* Compiler::compile_variable_shift(const goos::Object& form, throw_compiler_error(form, "Cannot shift a {} by a {}", in->type().print(), sa->type().print()); } - fenv->constrain(sa_con); + if (needs_shift_amount_in_rcx) { + fenv->constrain(sa_con); + } env->emit_ir(form, kind, result, sa_in); return result; } @@ -626,17 +635,19 @@ Val* Compiler::compile_mod(const goos::Object& form, const goos::Object& rest, E auto result = env->make_gpr(first->type()); env->emit_ir(form, result, first); - IRegConstraint con; - con.ireg = result->ireg(); - con.instr_idx = fenv->code().size(); - con.desired_register = emitter::RAX; - - fenv->constrain(con); + // x86 IDIV needs the dividend in RAX + if (m_instr_set == emitter::InstructionSet::X86) { + IRegConstraint con; + con.ireg = result->ireg(); + con.instr_idx = fenv->code().size(); + con.desired_register = emitter::RAX; + fenv->constrain(con); + } env->emit_ir(form, is_singed_integer_or_binteger(first->type()) ? IntegerMathKind::IMOD_32 : IntegerMathKind::UMOD_32, - result, second); + result, second, m_instr_set); auto result_moved = env->make_gpr(first->type()); env->emit_ir(form, result_moved, result); diff --git a/goalc/compiler/compilation/Static.cpp b/goalc/compiler/compilation/Static.cpp index c06b9a2829..b54bd53e1b 100644 --- a/goalc/compiler/compilation/Static.cpp +++ b/goalc/compiler/compilation/Static.cpp @@ -522,7 +522,7 @@ Val* Compiler::compile_bitfield_definition(const goos::Object& form, auto integer_hi = compile_integer(constant_integer_part.hi, env)->to_gpr(form, env); auto fe = env->function_env(); auto rv = fe->make_ireg(type, RegClass::INT_128); - auto xmm_temp = fe->make_ireg(TypeSpec("object"), RegClass::INT_128); + auto simd_temp = fe->make_ireg(TypeSpec("object"), RegClass::INT_128); for (auto& def : dynamic_defs) { auto field_val_in = def.value; @@ -560,9 +560,10 @@ Val* Compiler::compile_bitfield_definition(const goos::Object& form, start_lo ? integer_lo : integer_hi, field_val); } - fe->emit_ir(form, xmm_temp, integer_lo); + fe->emit_ir(form, simd_temp, integer_lo); fe->emit_ir(form, rv, integer_hi); - fe->emit_ir(form, true, rv, rv, xmm_temp, IR_Int128Math3Asm::Kind::PCPYLD); + fe->emit_ir(form, true, rv, rv, simd_temp, + IR_Int128Math3Asm::Kind::PCPYLD); return rv; } else { RegVal* integer_reg = integer->to_gpr(form, env); diff --git a/goalc/compiler/compilation/Type.cpp b/goalc/compiler/compilation/Type.cpp index 4d4e7d6510..c91e97f39d 100644 --- a/goalc/compiler/compilation/Type.cpp +++ b/goalc/compiler/compilation/Type.cpp @@ -270,7 +270,8 @@ Val* Compiler::generate_inspector_for_structure_type(const goos::Object& form, IRegConstraint constraint; constraint.instr_idx = 0; // constraint at the start of the function constraint.ireg = input_arg->ireg(); // constrain this register - constraint.desired_register = emitter::gRegInfo.get_gpr_arg_reg(0); // to the first argument + constraint.desired_register = + emitter::reg_info(m_instr_set).get_gpr_arg_reg(0); // to the first argument method_env->constrain(constraint); // Inform the compiler that `input`'s value will be written to `rdi` (first arg register) method_env->emit_ir(form, std::vector{input_arg}); @@ -314,7 +315,7 @@ Val* Compiler::generate_inspector_for_structure_type(const goos::Object& form, } method_env->emit_ir(form, method_env->make_gpr(input->type()), input, - emitter::gRegInfo.get_gpr_ret_reg()); + emitter::reg_info(m_instr_set).get_gpr_ret_reg()); // add this function to the object file auto fe = env->function_env(); @@ -351,9 +352,11 @@ Val* Compiler::generate_inspector_for_bitfield_type(const goos::Object& form, constraint.instr_idx = 0; // constraint at the start of the function constraint.ireg = input_arg->ireg(); // constrain this register if (bitfield_128) { - constraint.desired_register = emitter::gRegInfo.get_xmm_arg_reg(0); // to the first argument + constraint.desired_register = + emitter::reg_info(m_instr_set).get_simd_arg_reg(0); // to the first argument } else { - constraint.desired_register = emitter::gRegInfo.get_gpr_arg_reg(0); // to the first argument + constraint.desired_register = + emitter::reg_info(m_instr_set).get_gpr_arg_reg(0); // to the first argument } method_env->constrain(constraint); @@ -381,10 +384,10 @@ Val* Compiler::generate_inspector_for_bitfield_type(const goos::Object& form, if (bitfield_128) { method_env->emit_ir(form, method_env->make_ireg(input->type(), RegClass::INT_128), - input, emitter::gRegInfo.get_gpr_ret_reg()); + input, emitter::reg_info(m_instr_set).get_gpr_ret_reg()); } else { method_env->emit_ir(form, method_env->make_gpr(input->type()), input, - emitter::gRegInfo.get_gpr_ret_reg()); + emitter::reg_info(m_instr_set).get_gpr_ret_reg()); } // add this function to the object file @@ -566,7 +569,7 @@ Val* Compiler::compile_defmethod(const goos::Object& form, const goos::Object& _ for (auto& parm : lambda.params) { arg_types.push_back(parm.type); } - auto arg_regs = get_arg_registers(m_ts, arg_types); + auto arg_regs = get_arg_registers(m_ts, arg_types, m_instr_set); for (u32 i = 0; i < lambda.params.size(); i++) { IRegConstraint constr; @@ -587,7 +590,7 @@ Val* Compiler::compile_defmethod(const goos::Object& form, const goos::Object& _ auto self_var = new_func_env->make_gpr(m_ts.make_typespec(*behavior)); IRegConstraint constr; constr.contrain_everywhere = true; - constr.desired_register = emitter::gRegInfo.get_process_reg(); + constr.desired_register = emitter::reg_info(m_instr_set).get_process_reg(); constr.ireg = self_var->ireg(); self_var->set_rlet_constraint(constr.desired_register); new_func_env->constrain(constr); @@ -643,10 +646,10 @@ Val* Compiler::compile_defmethod(const goos::Object& form, const goos::Object& _ lambda_ts.add_arg(new_func_env->asm_func_return_type); } else if (result && !dynamic_cast(result) && result->type() != TypeSpec("none")) { RegVal* final_result; - emitter::Register ret_hw_reg = emitter::gRegInfo.get_gpr_ret_reg(); + emitter::Register ret_hw_reg = emitter::reg_info(m_instr_set).get_gpr_ret_reg(); if (m_ts.lookup_type(result->type())->get_load_size() == 16) { - ret_hw_reg = emitter::gRegInfo.get_xmm_ret_reg(); - final_result = result->to_xmm128(form, new_func_env.get()); + ret_hw_reg = emitter::reg_info(m_instr_set).get_simd_ret_reg(); + final_result = result->to_simd128(form, new_func_env.get()); return_reg->change_class(RegClass::INT_128); } else { final_result = result->to_gpr(form, new_func_env.get()); diff --git a/goalc/debugger/Debugger.cpp b/goalc/debugger/Debugger.cpp index b9a483be48..231dbfa39f 100644 --- a/goalc/debugger/Debugger.cpp +++ b/goalc/debugger/Debugger.cpp @@ -461,6 +461,7 @@ void Debugger::reload_break_state() { m_memory_map = m_listener->build_memory_map(); // lg::print("{}", m_memory_map.print()); read_symbol_table(); + // xdbg::Regs uses the x86 register layout and the macOS backend is not implemented. m_regs_valid = xdbg::get_regs_now(m_debug_context.tid, &m_regs_at_break); if (regs_valid()) { diff --git a/goalc/emitter/CallingConvention.cpp b/goalc/emitter/CallingConvention.cpp index 0e64a9b839..2159886a98 100644 --- a/goalc/emitter/CallingConvention.cpp +++ b/goalc/emitter/CallingConvention.cpp @@ -3,37 +3,38 @@ #include "common/util/Assert.h" CallingConvention get_function_calling_convention(const TypeSpec& function_type, - const TypeSystem& type_system) { + const TypeSystem& type_system, + emitter::InstructionSet instr_set) { ASSERT(function_type.base_type() == "function"); ASSERT(function_type.arg_count() > 0); ASSERT(function_type.arg_count() <= 9); int gpr_idx = 0; - int xmm_idx = 0; CallingConvention cc; if (function_type.arg_count() == 2 && function_type.get_arg(0).print() == "_varargs_") { for (int i = 0; i < 8; i++) { - cc.arg_regs.push_back(emitter::gRegInfo.get_gpr_arg_reg(gpr_idx++)); + cc.arg_regs.push_back(emitter::reg_info(instr_set).get_gpr_arg_reg(gpr_idx++)); } } else { + int simd_idx = 0; for (int i = 0; i < (int)function_type.arg_count() - 1; i++) { auto info = type_system.lookup_type_allow_partial_def(function_type.get_arg(i)); auto load_size = type_system.get_load_size_allow_partial_def(function_type.get_arg(i)); if (dynamic_cast(info) && load_size == 16) { - cc.arg_regs.push_back(emitter::gRegInfo.get_xmm_arg_reg(xmm_idx++)); + cc.arg_regs.push_back(emitter::reg_info(instr_set).get_simd_arg_reg(simd_idx++)); } else { - cc.arg_regs.push_back(emitter::gRegInfo.get_gpr_arg_reg(gpr_idx++)); + cc.arg_regs.push_back(emitter::reg_info(instr_set).get_gpr_arg_reg(gpr_idx++)); } } } if (function_type.last_arg() != TypeSpec("none")) { if (type_system.get_load_size_allow_partial_def(function_type.last_arg()) == 16) { - cc.return_reg = emitter::gRegInfo.get_xmm_ret_reg(); + cc.return_reg = emitter::reg_info(instr_set).get_simd_ret_reg(); } else { - cc.return_reg = emitter::gRegInfo.get_gpr_ret_reg(); + cc.return_reg = emitter::reg_info(instr_set).get_gpr_ret_reg(); } } @@ -41,16 +42,17 @@ CallingConvention get_function_calling_convention(const TypeSpec& function_type, } std::vector get_arg_registers(const TypeSystem& type_system, - const std::vector& arg_types) { + const std::vector& arg_types, + emitter::InstructionSet instr_set) { std::vector result; int gpr_idx = 0; - int xmm_idx = 0; + int simd_idx = 0; for (auto& type : arg_types) { auto load_size = type_system.get_load_size_allow_partial_def(type); if (load_size == 16) { - result.push_back(emitter::gRegInfo.get_xmm_arg_reg(xmm_idx++)); + result.push_back(emitter::reg_info(instr_set).get_simd_arg_reg(simd_idx++)); } else { - result.push_back(emitter::gRegInfo.get_gpr_arg_reg(gpr_idx++)); + result.push_back(emitter::reg_info(instr_set).get_gpr_arg_reg(gpr_idx++)); } } return result; diff --git a/goalc/emitter/CallingConvention.h b/goalc/emitter/CallingConvention.h index 5ac144c561..df26dcd588 100644 --- a/goalc/emitter/CallingConvention.h +++ b/goalc/emitter/CallingConvention.h @@ -5,6 +5,7 @@ #include "common/type_system/TypeSystem.h" +#include "goalc/emitter/InstructionSet.h" #include "goalc/emitter/Register.h" struct CallingConvention { @@ -13,6 +14,8 @@ struct CallingConvention { }; std::vector get_arg_registers(const TypeSystem& type_system, - const std::vector& arg_types); + const std::vector& arg_types, + emitter::InstructionSet instr_set); CallingConvention get_function_calling_convention(const TypeSpec& function_type, - const TypeSystem& type_system); \ No newline at end of file + const TypeSystem& type_system, + emitter::InstructionSet instr_set); \ No newline at end of file diff --git a/goalc/emitter/CodeTester.cpp b/goalc/emitter/CodeTester.cpp index c8ec65cc01..83d106f904 100644 --- a/goalc/emitter/CodeTester.cpp +++ b/goalc/emitter/CodeTester.cpp @@ -18,6 +18,9 @@ #elif _WIN32 #include "third-party/mman/mman.h" #endif +#if defined(__APPLE__) && defined(__aarch64__) +#include // pthread_jit_write_protect_np +#endif #include @@ -31,7 +34,9 @@ namespace emitter { CodeTester::CodeTester() : m_info(RegisterInfo::make_register_info()), m_gen(GameVersion::Jak1) {} CodeTester::CodeTester(InstructionSet instruction_set) - : m_info(RegisterInfo::make_register_info()), m_gen(GameVersion::Jak1, instruction_set) {} + : m_info(instruction_set == InstructionSet::ARM64 ? RegisterInfo::make_register_info_arm64() + : RegisterInfo::make_register_info()), + m_gen(GameVersion::Jak1, instruction_set) {} /*! * Convert to a string for comparison against an assembler or tests. @@ -291,7 +296,7 @@ void CodeTester::emit_push_all_gprs(bool exclude_return_register) { } /*! - * Push all xmm registers (all 128-bits) to the stack. + * Push all SIMD registers (all 128 bits) to the stack. */ void CodeTester::emit_push_all_simd() { if (m_gen.instr_set() == InstructionSet::X86) { @@ -301,8 +306,7 @@ void CodeTester::emit_push_all_simd() { emit(IGen::store128_gpr64_simd128(m_gen, RSP, XMM0 + i)); } } else if (m_gen.instr_set() == InstructionSet::ARM64) { - // TODO - 16 or 32 simd regs available? - for (int i = 0; i < 16; i++) { + for (int i = 0; i < 32; i++) { emit(IGen::sub_gpr64_imm8s(m_gen, SP, 16)); emit(IGen::store128_gpr64_simd128(m_gen, SP, V0 + i)); } @@ -312,7 +316,7 @@ void CodeTester::emit_push_all_simd() { } /*! - * Pop all xmm registers (all 128-bits) from the stack + * Pop all SIMD registers (all 128 bits) from the stack. */ void CodeTester::emit_pop_all_simd() { if (m_gen.instr_set() == InstructionSet::X86) { @@ -322,7 +326,7 @@ void CodeTester::emit_pop_all_simd() { } emit(IGen::add_gpr64_imm8s(m_gen, RSP, 8)); } else if (m_gen.instr_set() == InstructionSet::ARM64) { - for (int i = 15; i >= 0; i--) { + for (int i = 31; i >= 0; i--) { emit(IGen::load128_simd128_gpr64(m_gen, V0 + i, SP)); emit(IGen::add_gpr64_imm8s(m_gen, SP, 16)); } @@ -349,12 +353,10 @@ u64 CodeTester::execute() { #endif // clang-format off #if defined(__APPLE__) && defined(__aarch64__) - // TODO - we may need to switch to using pthread_jit_write_protect_np - // there may also be issues if multiple threasd are involved - // but this seems to work so keep it simple until something proves otherwise. - mprotect(code_buffer, code_buffer_capacity, PROT_EXEC | PROT_READ); + // block writes while this thread runs the MAP_JIT buffer + pthread_jit_write_protect_np(1); auto ret = ((u64(*)())code_buffer)(); - mprotect(code_buffer, code_buffer_capacity, PROT_WRITE | PROT_READ); + pthread_jit_write_protect_np(0); return ret; #else return ((u64(*)())code_buffer)(); @@ -367,11 +369,14 @@ u64 CodeTester::execute() { * arguments will appear in (will handle windows/linux differences) */ u64 CodeTester::execute(u64 in0, u64 in1, u64 in2, u64 in3) { +#if defined(__aarch64__) + __builtin___clear_cache((char*)code_buffer, (char*)code_buffer + code_buffer_size); +#endif // clang-format off #if defined(__APPLE__) && defined(__aarch64__) - mprotect(code_buffer, code_buffer_capacity, PROT_EXEC | PROT_READ); + pthread_jit_write_protect_np(1); auto ret = ((u64(*)(u64, u64, u64, u64))code_buffer)(in0, in1, in2, in3); - mprotect(code_buffer, code_buffer_capacity, PROT_WRITE | PROT_READ); + pthread_jit_write_protect_np(0); return ret; #else return ((u64(*)(u64, u64, u64, u64))code_buffer)(in0, in1, in2, in3); @@ -383,23 +388,21 @@ u64 CodeTester::execute(u64 in0, u64 in1, u64 in2, u64 in3) { * Allocate a code buffer of the given size. */ void CodeTester::init_code_buffer(int capacity) { -// TODO Apple Silicon - You cannot make a page be RWX, -// or more specifically it can't be both writable and executable at the same time -// -// https://github.com/zherczeg/sljit/issues/99 -// -// The solution to this is to flip-flop between permissions, or perhaps have two threads -// one that has writing permission, and another with executable permission + // MAP_JIT write protection is per thread #if defined(__APPLE__) && defined(__aarch64__) - code_buffer = (u8*)mmap(nullptr, capacity, PROT_WRITE | PROT_READ, - MAP_ANONYMOUS | MAP_PRIVATE | MAP_JIT, 0, 0); + code_buffer = (u8*)mmap(nullptr, capacity, PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_JIT, -1, 0); #else code_buffer = (u8*)mmap(nullptr, capacity, PROT_EXEC | PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); #endif if (code_buffer == (u8*)(-1)) { ASSERT_MSG(false, "[CodeTester] Failed to map memory!"); } +#if defined(__APPLE__) && defined(__aarch64__) + // allow writes before the first instruction + pthread_jit_write_protect_np(0); +#endif code_buffer_capacity = capacity; code_buffer_size = 0; diff --git a/goalc/emitter/CodeTester.h b/goalc/emitter/CodeTester.h index 7bfd6f19bc..4a4ed94689 100644 --- a/goalc/emitter/CodeTester.h +++ b/goalc/emitter/CodeTester.h @@ -24,6 +24,9 @@ #elif _WIN32 #include "third-party/mman/mman.h" #endif +#if defined(__APPLE__) && defined(__aarch64__) +#include // pthread_jit_write_protect_np +#endif namespace emitter { class CodeTester { @@ -70,12 +73,10 @@ class CodeTester { #endif // clang-format off #if defined(__APPLE__) && defined(__aarch64__) - // TODO - we may need to switch to using pthread_jit_write_protect_np - // there may also be issues if multiple threasd are involved - // but this seems to work so keep it simple until something proves otherwise. - mprotect(code_buffer, code_buffer_capacity, PROT_EXEC | PROT_READ); + // block writes while generated code runs + pthread_jit_write_protect_np(1); u64 result_u64 = ((u64(*)(u64, u64, u64, u64))code_buffer)(in0, in1, in2, in3); - mprotect(code_buffer, code_buffer_capacity, PROT_WRITE | PROT_READ); + pthread_jit_write_protect_np(0); T result_T; memcpy(&result_T, &result_u64, sizeof(T)); return result_T; @@ -110,7 +111,7 @@ class CodeTester { int get_simd_reg_count() { if (m_gen.instr_set() == InstructionSet::ARM64) { - return 16; // TODO - check if platform has 16 or 32 + return 32; } else { return -1; // TODO } @@ -158,7 +159,7 @@ class CodeTester { throw std::runtime_error("Invalid ARM64 arg register index"); } } - // TODO ARM64 - x86 specific + // x86 ABI registers differ by platform. #ifdef _WIN32 switch (i) { case 0: diff --git a/goalc/emitter/IGen.cpp b/goalc/emitter/IGen.cpp index 61e0d92100..b53420da24 100644 --- a/goalc/emitter/IGen.cpp +++ b/goalc/emitter/IGen.cpp @@ -321,12 +321,12 @@ Instruction store_goal_gpr(const ObjectGenerator& gen, IGEN_DISPATCH(store_goal_gpr, addr, value, off, offset, size); } -Instruction load_goal_xmm128(const ObjectGenerator& gen, - Register dst, - Register addr, - Register off, - int offset) { - IGEN_DISPATCH(load_goal_xmm128, dst, addr, off, offset); +Instruction load_goal_simd128(const ObjectGenerator& gen, + Register dst, + Register addr, + Register off, + int offset) { + IGEN_DISPATCH(load_goal_simd128, dst, addr, off, offset); } Instruction load_goal_gpr(const ObjectGenerator& gen, @@ -339,42 +339,42 @@ Instruction load_goal_gpr(const ObjectGenerator& gen, IGEN_DISPATCH(load_goal_gpr, dst, addr, off, offset, size, sign_extend); } -Instruction store32_xmm32_gpr64_plus_gpr64(const ObjectGenerator& gen, +Instruction store32_simd32_gpr64_plus_gpr64(const ObjectGenerator& gen, + Register addr1, + Register addr2, + Register simd_value) { + IGEN_DISPATCH(store32_simd32_gpr64_plus_gpr64, addr1, addr2, simd_value); +} + +Instruction load32_simd32_gpr64_plus_gpr64(const ObjectGenerator& gen, + Register simd_dest, Register addr1, - Register addr2, - Register xmm_value) { - IGEN_DISPATCH(store32_xmm32_gpr64_plus_gpr64, addr1, addr2, xmm_value); + Register addr2) { + IGEN_DISPATCH(load32_simd32_gpr64_plus_gpr64, simd_dest, addr1, addr2); } -Instruction load32_xmm32_gpr64_plus_gpr64(const ObjectGenerator& gen, - Register simd_dest, - Register addr1, - Register addr2) { - IGEN_DISPATCH(load32_xmm32_gpr64_plus_gpr64, simd_dest, addr1, addr2); -} - -Instruction store32_xmm32_gpr64_plus_gpr64_plus_s8(const ObjectGenerator& gen, - Register addr1, - Register addr2, - Register xmm_value, - s64 offset) { - IGEN_DISPATCH(store32_xmm32_gpr64_plus_gpr64_plus_s8, addr1, addr2, xmm_value, offset); -} - -Instruction load32_xmm32_gpr64_plus_gpr64_plus_s8(const ObjectGenerator& gen, - Register simd_dest, - Register addr1, - Register addr2, - s64 offset) { - IGEN_DISPATCH(load32_xmm32_gpr64_plus_gpr64_plus_s8, simd_dest, addr1, addr2, offset); -} - -Instruction store32_xmm32_gpr64_plus_gpr64_plus_s32(const ObjectGenerator& gen, +Instruction store32_simd32_gpr64_plus_gpr64_plus_s8(const ObjectGenerator& gen, Register addr1, Register addr2, - Register xmm_value, + Register simd_value, s64 offset) { - IGEN_DISPATCH(store32_xmm32_gpr64_plus_gpr64_plus_s32, addr1, addr2, xmm_value, offset); + IGEN_DISPATCH(store32_simd32_gpr64_plus_gpr64_plus_s8, addr1, addr2, simd_value, offset); +} + +Instruction load32_simd32_gpr64_plus_gpr64_plus_s8(const ObjectGenerator& gen, + Register simd_dest, + Register addr1, + Register addr2, + s64 offset) { + IGEN_DISPATCH(load32_simd32_gpr64_plus_gpr64_plus_s8, simd_dest, addr1, addr2, offset); +} + +Instruction store32_simd32_gpr64_plus_gpr64_plus_s32(const ObjectGenerator& gen, + Register addr1, + Register addr2, + Register simd_value, + s64 offset) { + IGEN_DISPATCH(store32_simd32_gpr64_plus_gpr64_plus_s32, addr1, addr2, simd_value, offset); } Instruction lea_reg_plus_off32(const ObjectGenerator& gen, @@ -395,90 +395,90 @@ Instruction lea_reg_plus_off(const ObjectGenerator& gen, Register dest, Register IGEN_DISPATCH(lea_reg_plus_off, dest, base, offset); } -Instruction store32_xmm32_gpr64_plus_s32(const ObjectGenerator& gen, +Instruction store32_simd32_gpr64_plus_s32(const ObjectGenerator& gen, + Register base, + Register simd_value, + s64 offset) { + IGEN_DISPATCH(store32_simd32_gpr64_plus_s32, base, simd_value, offset); +} + +Instruction store32_simd32_gpr64_plus_s8(const ObjectGenerator& gen, Register base, - Register xmm_value, + Register simd_value, s64 offset) { - IGEN_DISPATCH(store32_xmm32_gpr64_plus_s32, base, xmm_value, offset); + IGEN_DISPATCH(store32_simd32_gpr64_plus_s8, base, simd_value, offset); } -Instruction store32_xmm32_gpr64_plus_s8(const ObjectGenerator& gen, - Register base, - Register xmm_value, - s64 offset) { - IGEN_DISPATCH(store32_xmm32_gpr64_plus_s8, base, xmm_value, offset); +Instruction load32_simd32_gpr64_plus_gpr64_plus_s32(const ObjectGenerator& gen, + Register simd_dest, + Register addr1, + Register addr2, + s64 offset) { + IGEN_DISPATCH(load32_simd32_gpr64_plus_gpr64_plus_s32, simd_dest, addr1, addr2, offset); } -Instruction load32_xmm32_gpr64_plus_gpr64_plus_s32(const ObjectGenerator& gen, - Register simd_dest, - Register addr1, - Register addr2, - s64 offset) { - IGEN_DISPATCH(load32_xmm32_gpr64_plus_gpr64_plus_s32, simd_dest, addr1, addr2, offset); +Instruction load32_simd32_gpr64_plus_s32(const ObjectGenerator& gen, + Register simd_dest, + Register base, + s64 offset) { + IGEN_DISPATCH(load32_simd32_gpr64_plus_s32, simd_dest, base, offset); } -Instruction load32_xmm32_gpr64_plus_s32(const ObjectGenerator& gen, +Instruction load32_simd32_gpr64_plus_s8(const ObjectGenerator& gen, Register simd_dest, Register base, s64 offset) { - IGEN_DISPATCH(load32_xmm32_gpr64_plus_s32, simd_dest, base, offset); + IGEN_DISPATCH(load32_simd32_gpr64_plus_s8, simd_dest, base, offset); } -Instruction load32_xmm32_gpr64_plus_s8(const ObjectGenerator& gen, - Register simd_dest, - Register base, - s64 offset) { - IGEN_DISPATCH(load32_xmm32_gpr64_plus_s8, simd_dest, base, offset); -} - -Instruction load_goal_xmm32(const ObjectGenerator& gen, - Register simd_dest, - Register addr, - Register off, - s64 offset) { - IGEN_DISPATCH(load_goal_xmm32, simd_dest, addr, off, offset); -} - -Instruction store_goal_xmm32(const ObjectGenerator& gen, +Instruction load_goal_simd32(const ObjectGenerator& gen, + Register simd_dest, Register addr, - Register xmm_value, Register off, s64 offset) { - IGEN_DISPATCH(store_goal_xmm32, addr, xmm_value, off, offset); + IGEN_DISPATCH(load_goal_simd32, simd_dest, addr, off, offset); } -Instruction store_reg_offset_xmm32(const ObjectGenerator& gen, +Instruction store_goal_simd32(const ObjectGenerator& gen, + Register addr, + Register simd_value, + Register off, + s64 offset) { + IGEN_DISPATCH(store_goal_simd32, addr, simd_value, off, offset); +} + +Instruction store_reg_offset_simd32(const ObjectGenerator& gen, + Register base, + Register simd_value, + s64 offset) { + IGEN_DISPATCH(store_reg_offset_simd32, base, simd_value, offset); +} + +Instruction load_reg_offset_simd32(const ObjectGenerator& gen, + Register simd_dest, Register base, - Register xmm_value, s64 offset) { - IGEN_DISPATCH(store_reg_offset_xmm32, base, xmm_value, offset); -} - -Instruction load_reg_offset_xmm32(const ObjectGenerator& gen, - Register simd_dest, - Register base, - s64 offset) { - IGEN_DISPATCH(load_reg_offset_xmm32, simd_dest, base, offset); + IGEN_DISPATCH(load_reg_offset_simd32, simd_dest, base, offset); } Instruction store128_gpr64_simd128(const ObjectGenerator& gen, Register gpr_addr, - Register xmm_value) { - IGEN_DISPATCH(store128_gpr64_simd128, gpr_addr, xmm_value); + Register simd_value) { + IGEN_DISPATCH(store128_gpr64_simd128, gpr_addr, simd_value); } Instruction store128_gpr64_simd128_s32(const ObjectGenerator& gen, Register gpr_addr, - Register xmm_value, + Register simd_value, s64 offset) { - IGEN_DISPATCH(store128_gpr64_simd128_s32, gpr_addr, xmm_value, offset); + IGEN_DISPATCH(store128_gpr64_simd128_s32, gpr_addr, simd_value, offset); } Instruction store128_gpr64_simd128_s8(const ObjectGenerator& gen, Register gpr_addr, - Register xmm_value, + Register simd_value, s64 offset) { - IGEN_DISPATCH(store128_gpr64_simd128_s8, gpr_addr, xmm_value, offset); + IGEN_DISPATCH(store128_gpr64_simd128_s8, gpr_addr, simd_value, offset); } Instruction load128_simd128_gpr64(const ObjectGenerator& gen, @@ -501,18 +501,18 @@ Instruction load128_simd128_gpr64_s8(const ObjectGenerator& gen, IGEN_DISPATCH(load128_simd128_gpr64_s8, simd_dest, gpr_addr, offset); } -Instruction load128_xmm128_reg_offset(const ObjectGenerator& gen, - Register simd_dest, - Register base, - s64 offset) { - IGEN_DISPATCH(load128_xmm128_reg_offset, simd_dest, base, offset); +Instruction load128_simd128_reg_offset(const ObjectGenerator& gen, + Register simd_dest, + Register base, + s64 offset) { + IGEN_DISPATCH(load128_simd128_reg_offset, simd_dest, base, offset); } -Instruction store128_xmm128_reg_offset(const ObjectGenerator& gen, - Register base, - Register xmm_val, - s64 offset) { - IGEN_DISPATCH(store128_xmm128_reg_offset, base, xmm_val, offset); +Instruction store128_simd128_reg_offset(const ObjectGenerator& gen, + Register base, + Register simd_val, + s64 offset) { + IGEN_DISPATCH(store128_simd128_reg_offset, base, simd_val, offset); } Instruction load64_rip_s32(const ObjectGenerator& gen, Register dest, s64 offset) { @@ -579,8 +579,8 @@ Instruction static_load_f32(const ObjectGenerator& gen, Register simd_dest, s64 IGEN_DISPATCH(static_load_f32, simd_dest, offset); } -Instruction static_store_f32(const ObjectGenerator& gen, Register xmm_value, s64 offset) { - IGEN_DISPATCH(static_store_f32, xmm_value, offset); +Instruction static_store_f32(const ObjectGenerator& gen, Register simd_value, s64 offset) { + IGEN_DISPATCH(static_store_f32, simd_value, offset); } Instruction load64_gpr64_plus_s32(const ObjectGenerator& gen, @@ -805,6 +805,10 @@ Instruction nop(const ObjectGenerator& gen) { IGEN_DISPATCH(nop); } +Instruction trap(const ObjectGenerator& gen) { + IGEN_DISPATCH(trap); +} + Instruction null(const ObjectGenerator& gen) { IGEN_DISPATCH(null); } diff --git a/goalc/emitter/IGen.h b/goalc/emitter/IGen.h index 27cbcc8c16..a077bd41e5 100644 --- a/goalc/emitter/IGen.h +++ b/goalc/emitter/IGen.h @@ -33,32 +33,32 @@ Instruction mov_gpr64_u32(const ObjectGenerator& gen, Register dst, uint64_t val Instruction mov_gpr64_s32(const ObjectGenerator& gen, Register dst, int64_t val); /*! - * Move 32-bits of xmm to 32 bits of gpr (no sign extension). + * Move 32-bits of SIMD register to 32 bits of gpr (no sign extension). */ Instruction movd_gpr32_f32(const ObjectGenerator& gen, Register dst, Register src); /*! - * Move 32-bits of gpr to 32-bits of xmm (no sign extension) + * Move 32-bits of gpr to 32-bits of SIMD register (no sign extension) */ Instruction movd_f32_gpr32(const ObjectGenerator& gen, Register dst, Register src); /*! - * Move 64-bits of xmm to 64 bits of gpr (no sign extension). + * Move 64-bits of SIMD register to 64 bits of gpr (no sign extension). */ Instruction movq_gpr64_f64(const ObjectGenerator& gen, Register dst, Register src); /*! - * Move 64-bits of gpr to 64-bits of xmm (no sign extension) + * Move 64-bits of gpr to 64-bits of SIMD register (no sign extension) */ Instruction movq_f64_gpr64(const ObjectGenerator& gen, Register dst, Register src); /*! - * Move 32-bits between xmm's + * Move 32-bits between SIMD registers */ Instruction mov_f32_f32(const ObjectGenerator& gen, Register dst, Register src); -// todo - GPR64 -> XMM64 (zext) -// todo - XMM -> GPR64 +// todo - GPR64 -> SIMD64 (zext) +// todo - SIMD -> GPR64 //;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; // GOAL Loads and Stores @@ -299,11 +299,11 @@ Instruction store_goal_gpr(const ObjectGenerator& gen, int offset, int size); -Instruction load_goal_xmm128(const ObjectGenerator& gen, - Register dst, - Register addr, - Register off, - int offset); +Instruction load_goal_simd128(const ObjectGenerator& gen, + Register dst, + Register addr, + Register off, + int offset); /*! * Load memory at addr + offset, where addr is a GOAL pointer and off is the offset register. @@ -318,36 +318,36 @@ Instruction load_goal_gpr(const ObjectGenerator& gen, bool sign_extend); //;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -// LOADS n' STORES - XMM32 +// LOADS n' STORES - SIMD32 //;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -Instruction store32_xmm32_gpr64_plus_gpr64(const ObjectGenerator& gen, +Instruction store32_simd32_gpr64_plus_gpr64(const ObjectGenerator& gen, + Register addr1, + Register addr2, + Register simd_value); + +Instruction load32_simd32_gpr64_plus_gpr64(const ObjectGenerator& gen, + Register simd_dest, Register addr1, - Register addr2, - Register xmm_value); + Register addr2); -Instruction load32_xmm32_gpr64_plus_gpr64(const ObjectGenerator& gen, - Register simd_dest, - Register addr1, - Register addr2); - -Instruction store32_xmm32_gpr64_plus_gpr64_plus_s8(const ObjectGenerator& gen, - Register addr1, - Register addr2, - Register xmm_value, - s64 offset); - -Instruction load32_xmm32_gpr64_plus_gpr64_plus_s8(const ObjectGenerator& gen, - Register simd_dest, - Register addr1, - Register addr2, - s64 offset); - -Instruction store32_xmm32_gpr64_plus_gpr64_plus_s32(const ObjectGenerator& gen, +Instruction store32_simd32_gpr64_plus_gpr64_plus_s8(const ObjectGenerator& gen, Register addr1, Register addr2, - Register xmm_value, + Register simd_value, s64 offset); +Instruction load32_simd32_gpr64_plus_gpr64_plus_s8(const ObjectGenerator& gen, + Register simd_dest, + Register addr1, + Register addr2, + s64 offset); + +Instruction store32_simd32_gpr64_plus_gpr64_plus_s32(const ObjectGenerator& gen, + Register addr1, + Register addr2, + Register simd_value, + s64 offset); + Instruction lea_reg_plus_off32(const ObjectGenerator& gen, Register dest, Register base, @@ -357,73 +357,73 @@ Instruction lea_reg_plus_off8(const ObjectGenerator& gen, Register dest, Registe Instruction lea_reg_plus_off(const ObjectGenerator& gen, Register dest, Register base, s64 offset); -Instruction store32_xmm32_gpr64_plus_s32(const ObjectGenerator& gen, +Instruction store32_simd32_gpr64_plus_s32(const ObjectGenerator& gen, + Register base, + Register simd_value, + s64 offset); + +Instruction store32_simd32_gpr64_plus_s8(const ObjectGenerator& gen, Register base, - Register xmm_value, + Register simd_value, s64 offset); -Instruction store32_xmm32_gpr64_plus_s8(const ObjectGenerator& gen, - Register base, - Register xmm_value, - s64 offset); +Instruction load32_simd32_gpr64_plus_gpr64_plus_s32(const ObjectGenerator& gen, + Register simd_dest, + Register addr1, + Register addr2, + s64 offset); -Instruction load32_xmm32_gpr64_plus_gpr64_plus_s32(const ObjectGenerator& gen, - Register simd_dest, - Register addr1, - Register addr2, - s64 offset); +Instruction load32_simd32_gpr64_plus_s32(const ObjectGenerator& gen, + Register simd_dest, + Register base, + s64 offset); -Instruction load32_xmm32_gpr64_plus_s32(const ObjectGenerator& gen, +Instruction load32_simd32_gpr64_plus_s8(const ObjectGenerator& gen, Register simd_dest, Register base, s64 offset); -Instruction load32_xmm32_gpr64_plus_s8(const ObjectGenerator& gen, - Register simd_dest, - Register base, - s64 offset); - -Instruction load_goal_xmm32(const ObjectGenerator& gen, - Register simd_dest, - Register addr, - Register off, - s64 offset); - -Instruction store_goal_xmm32(const ObjectGenerator& gen, +Instruction load_goal_simd32(const ObjectGenerator& gen, + Register simd_dest, Register addr, - Register xmm_value, Register off, s64 offset); -Instruction store_reg_offset_xmm32(const ObjectGenerator& gen, +Instruction store_goal_simd32(const ObjectGenerator& gen, + Register addr, + Register simd_value, + Register off, + s64 offset); + +Instruction store_reg_offset_simd32(const ObjectGenerator& gen, + Register base, + Register simd_value, + s64 offset); + +Instruction load_reg_offset_simd32(const ObjectGenerator& gen, + Register simd_dest, Register base, - Register xmm_value, s64 offset); -Instruction load_reg_offset_xmm32(const ObjectGenerator& gen, - Register simd_dest, - Register base, - s64 offset); - //;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -// LOADS n' STORES - XMM128 +// LOADS n' STORES - SIMD128 //;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; /*! - * Store a 128-bit xmm into an address stored in a register, no offset + * Store a 128-bit SIMD register into an address stored in a register, no offset */ Instruction store128_gpr64_simd128(const ObjectGenerator& gen, Register gpr_addr, - Register xmm_value); + Register simd_value); Instruction store128_gpr64_simd128_s32(const ObjectGenerator& gen, Register gpr_addr, - Register xmm_value, + Register simd_value, s64 offset); Instruction store128_gpr64_simd128_s8(const ObjectGenerator& gen, Register gpr_addr, - Register xmm_value, + Register simd_value, s64 offset); Instruction load128_simd128_gpr64(const ObjectGenerator& gen, @@ -440,16 +440,16 @@ Instruction load128_simd128_gpr64_s8(const ObjectGenerator& gen, Register gpr_addr, s64 offset); -Instruction load128_xmm128_reg_offset(const ObjectGenerator& gen, - Register simd_dest, - Register base, - s64 offset); - -Instruction store128_xmm128_reg_offset(const ObjectGenerator& gen, +Instruction load128_simd128_reg_offset(const ObjectGenerator& gen, + Register simd_dest, Register base, - Register xmm_val, s64 offset); +Instruction store128_simd128_reg_offset(const ObjectGenerator& gen, + Register base, + Register simd_val, + s64 offset); + //;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; // RIP loads and stores //;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -488,7 +488,7 @@ Instruction static_addr(const ObjectGenerator& gen, Register dst, s64 offset); Instruction static_load_f32(const ObjectGenerator& gen, Register simd_dest, s64 offset); -Instruction static_store_f32(const ObjectGenerator& gen, Register xmm_value, s64 offset); +Instruction static_store_f32(const ObjectGenerator& gen, Register simd_value, s64 offset); // TODO, special load/stores of 128 bit values. @@ -767,6 +767,9 @@ Instruction nop(const ObjectGenerator& gen); // UTILITIES //;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +//! Emit the trap used by GOAL (break) without colliding with debugger breakpoints. +Instruction trap(const ObjectGenerator& gen); + /*! * A "null" instruction. This instruction does not generate any bytes * but can be referred to by a label. Useful to insert in place of a real instruction @@ -843,10 +846,10 @@ shuffle_vf(const ObjectGenerator& gen, Register dst, Register src, u8 dx, u8 dy, - 10b - Copy the third element (from the right) (Z) - 11b - Copy the most significant element (W) Examples - ; xmm1 = (1.5, 2.5, 3.5, 4.5) (W,Z,Y,X in x86 land) - SHUFPS xmm1, xmm1, 0xff ; Copy the most significant element to all positions + ; simd1 = (1.5, 2.5, 3.5, 4.5) (W,Z,Y,X in x86 land) + SHUFPS simd1, simd1, 0xff ; Copy the most significant element to all positions > (1.5, 1.5, 1.5, 1.5) - SHUFPS xmm1, xmm1, 0x39 ; Rotate right + SHUFPS simd1, simd1, 0x39 ; Rotate right > (4.5, 1.5, 2.5, 3.5) */ Instruction swizzle_vf(const ObjectGenerator& gen, Register dst, Register src, u8 controlBytes); @@ -854,10 +857,10 @@ Instruction swizzle_vf(const ObjectGenerator& gen, Register dst, Register src, u /* Splats a single element in 'src' to all elements in 'dst' For example (pseudocode): - xmm1 = (1.5, 2.5, 3.5, 4.5) - xmm2 = (1, 2, 3, 4) - splat_vf(xmm1, xmm2, XMM_ELEMENT::X); - xmm1 = (4, 4, 4, 4) + simd1 = (1.5, 2.5, 3.5, 4.5) + simd2 = (1, 2, 3, 4) + splat_vf(simd1, simd2, VF_ELEMENT::X); + simd1 = (4, 4, 4, 4) */ Instruction splat_vf(const ObjectGenerator& gen, Register dst, diff --git a/goalc/emitter/IGenARM64.cpp b/goalc/emitter/IGenARM64.cpp index 16807a716e..b0d38978df 100644 --- a/goalc/emitter/IGenARM64.cpp +++ b/goalc/emitter/IGenARM64.cpp @@ -3,6 +3,7 @@ #include +#include "common/arm64/encoding.h" #include "common/util/Assert.h" #include "goalc/emitter/IGen.h" @@ -14,9 +15,6 @@ // https://armconverter.com/?code=ret // https://developer.arm.com/documentation/ddi0487/latest -// TODO ARM64 - just silencing errors while things are not implemented obviously -#pragma GCC diagnostic ignored "-Wunused-parameter" - namespace emitter { namespace IGen { namespace ARM64 { @@ -110,29 +108,34 @@ std::vector construct_multiple_imm12_subs(int64_t imm, u32 reg //;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; InstructionARM64 mov_gpr64_gpr64(Register dst, Register src) { - // https://www.scs.stanford.edu/~zyedidia/arm64/mov_orr_log_shift.html - // MOV , ASSERT(dst.is_gpr(instr_set)); ASSERT(src.is_gpr(instr_set)); - return InstructionARM64(Base(0b10101010000, 11), Rm(src.id()), Rn(0b11111), Rd(dst.id()), - Imm6(0)); + if (dst == SP || src == SP) { + // use ADD with a zero immediate because ORR treats register 31 as XZR instead of SP + // https://www.scs.stanford.edu/~zyedidia/arm64/mov_add_addsub_imm.html + return InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rn(src.hw_id(instr_set)), + Rd(dst.hw_id(instr_set))); + } + // https://www.scs.stanford.edu/~zyedidia/arm64/mov_orr_log_shift.html + // MOV , + return InstructionARM64(Base(0b10101010000, 11), Rm(src.hw_id(instr_set)), Rn(0b11111), + Rd(dst.hw_id(instr_set)), Imm6(0)); } std::vector mov_gpr64_u64_instrs(Register dst, uint64_t val) { // Cannot be done in a single instruction, must combine multiple MOVZ/MOVKs std::vector instrs; + const auto reg = u32(dst.hw_id(instr_set)); auto imm_chunks = decompose_into_imm16_chunks(val); for (const auto& [imm_chunk, shift] : imm_chunks) { if (shift == 0) { // https://www.scs.stanford.edu/~zyedidia/arm64/movz.html // MOVZ , #{, LSL #/16} - instrs.emplace_back( - InstructionARM64(Base(0b110100101, 9), Hw(shift), Imm16(imm_chunk), Rd(dst.id()))); + instrs.emplace_back(arm64::encode_movz_64(reg, imm_chunk, shift)); } else { // https://www.scs.stanford.edu/~zyedidia/arm64/movk.html // MOVK , #{, LSL #/16} - instrs.emplace_back( - InstructionARM64(Base(0b111100101, 9), Hw(shift), Imm16(imm_chunk), Rd(dst.id()))); + instrs.emplace_back(arm64::encode_movk_64(reg, imm_chunk, shift)); } } return instrs; @@ -152,12 +155,38 @@ InstructionARM64 mov_gpr64_s32(Register dst, int64_t val) { return mov_gpr64_u64(dst, raw_val); } +InstructionARM64 mov_gpr32_link_imm32(Register dst, u32 initial) { + ASSERT(dst.is_gpr(instr_set)); + // always emit both words so the linker patch size stays fixed + // https://www.scs.stanford.edu/~zyedidia/arm64/movz.html + // MOVZ , # + auto movz = InstructionARM64(Base(0b010100101, 9), Hw(0), Imm16(initial & 0xffff), + Rd(dst.hw_id(instr_set))); + // https://www.scs.stanford.edu/~zyedidia/arm64/movk.html + // MOVK , #, LSL #16 + auto movk = + InstructionARM64(Base(0b011100101, 9), Hw(1), Imm16(initial >> 16), Rd(dst.hw_id(instr_set))); + return InstructionARM64(movz, movk).with_reloc(emitter::ARM64::RelocKind::MOV32); +} + +InstructionARM64 add_gpr64_gpr64_sxtw(Register dst, Register base, Register idx) { + ASSERT(dst.is_gpr(instr_set)); + ASSERT(base.is_gpr(instr_set)); + ASSERT(idx.is_gpr(instr_set)); + // SXTW is option 0b110 with no shift + // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html + // ADD , , , SXTW + return InstructionARM64(Base(0b10001011001000001100000000000000, 32), Rm(idx.hw_id(instr_set)), + Rn(base.hw_id(instr_set)), Rd(dst.hw_id(instr_set))); +} + InstructionARM64 movd_gpr32_f32(Register dst, Register src) { // https://www.scs.stanford.edu/~zyedidia/arm64/fmov_float_gen.html // Single-precision to 32-bit (sf == 0 && ftype == 00 && rmode == 00 && opcode == 110) // FMOV , ASSERT(dst.is_gpr(instr_set)); - return InstructionARM64(Base(0b0001111000100110000000, 22), Rn(src.id()), Rd(dst.id())); + return InstructionARM64(Base(0b0001111000100110000000, 22), Rn(src.hw_id(instr_set)), + Rd(dst.hw_id(instr_set))); } InstructionARM64 movd_f32_gpr32(Register dst, Register src) { @@ -165,7 +194,8 @@ InstructionARM64 movd_f32_gpr32(Register dst, Register src) { // 32-bit to single-precision (sf == 0 && ftype == 00 && rmode == 00 && opcode == 111) // FMOV , ASSERT(src.is_gpr(instr_set)); - return InstructionARM64(Base(0b0001111000100111000000, 22), Rn(src.id()), Rd(dst.id())); + return InstructionARM64(Base(0b0001111000100111000000, 22), Rn(src.hw_id(instr_set)), + Rd(dst.hw_id(instr_set))); } InstructionARM64 movq_gpr64_f64(Register dst, Register src) { @@ -173,7 +203,8 @@ InstructionARM64 movq_gpr64_f64(Register dst, Register src) { // Double-precision to 64-bit (sf == 1 && ftype == 01 && rmode == 00 && opcode == 110) // FMOV , ASSERT(dst.is_gpr(instr_set)); - return InstructionARM64(Base(0b1001111001100110000000, 22), Rn(src.id()), Rd(dst.id())); + return InstructionARM64(Base(0b1001111001100110000000, 22), Rn(src.hw_id(instr_set)), + Rd(dst.hw_id(instr_set))); } InstructionARM64 movq_f64_gpr64(Register dst, Register src) { @@ -181,14 +212,16 @@ InstructionARM64 movq_f64_gpr64(Register dst, Register src) { // 64-bit to double-precision (sf == 1 && ftype == 01 && rmode == 00 && opcode == 111) // FMOV , ASSERT(src.is_gpr(instr_set)); - return InstructionARM64(Base(0b1001111001100111000000, 22), Rn(src.id()), Rd(dst.id())); + return InstructionARM64(Base(0b1001111001100111000000, 22), Rn(src.hw_id(instr_set)), + Rd(dst.hw_id(instr_set))); } InstructionARM64 mov_f32_f32(Register dst, Register src) { // https://www.scs.stanford.edu/~zyedidia/arm64/fmov_float.html // Single-precision (ftype == 00) // FMOV , - return InstructionARM64(Base(0b0001111000100000010000, 22), Rn(src.id()), Rd(dst.id())); + return InstructionARM64(Base(0b0001111000100000010000, 22), Rn(src.hw_id(instr_set)), + Rd(dst.hw_id(instr_set))); } //;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -205,8 +238,8 @@ InstructionARM64 load8s_gpr64_gpr64_plus_gpr64(Register dst, Register addr1, Reg ASSERT(addr1 != addr2); ASSERT(addr1 != SP); ASSERT(addr2 != SP); - return InstructionARM64(Base(0b0011100010100000111010, 22), Rt(dst.id()), Rn(addr1.id()), - Rm(addr2.id())); + return InstructionARM64(Base(0b0011100010100000111010, 22), Rt(dst.hw_id(instr_set)), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))); } InstructionARM64 store8_gpr64_gpr64_plus_gpr64(Register addr1, Register addr2, Register value) { @@ -219,13 +252,11 @@ InstructionARM64 store8_gpr64_gpr64_plus_gpr64(Register addr1, Register addr2, R ASSERT(addr1 != addr2); ASSERT(addr1 != SP); ASSERT(addr2 != SP); - return InstructionARM64(Base(0b0011100000100000111010, 22), Rt(value.id()), Rn(addr1.id()), - Rm(addr2.id())); + return InstructionARM64(Base(0b0011100000100000111010, 22), Rt(value.hw_id(instr_set)), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))); } -// TODO ARM64 - x16 needs to be reserved, started leveraging it here -// yes it would be possible to only reserve it _sometimes_, but keep things simple -// we have SO many more registers already over x86, 1 less isn't going to be that big of a deal +// x16 is reserved as an emitter scratch register. InstructionARM64 load8s_gpr64_gpr64_plus_gpr64_plus_s8(Register dst, Register addr1, @@ -238,14 +269,14 @@ InstructionARM64 load8s_gpr64_gpr64_plus_gpr64_plus_s8(Register dst, ASSERT(addr1 != SP); ASSERT(addr2 != SP); ASSERT(offset >= INT8_MIN && offset <= INT8_MAX); - return InstructionARM64( - {// https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html - // ADD , , {, {#}} - InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), Rn(addr1.id()), - Rm(addr2.id())), - // https://www.scs.stanford.edu/~zyedidia/arm64/ldursb.html - // LDURSB , [{, #}] - InstructionARM64(Base(0b0011100010000000000000, 22), Imm9s(offset), Rt(dst.id()), Rn(X16))}); + // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html + // ADD , , {, {#}} + return InstructionARM64({InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))), + // https://www.scs.stanford.edu/~zyedidia/arm64/ldursb.html + // LDURSB , [{, #}] + InstructionARM64(Base(0b0011100010000000000000, 22), Imm9s(offset), + Rt(dst.hw_id(instr_set)), Rn(X16))}); } InstructionARM64 store8_gpr64_gpr64_plus_gpr64_plus_s8(Register addr1, @@ -262,11 +293,11 @@ InstructionARM64 store8_gpr64_gpr64_plus_gpr64_plus_s8(Register addr1, return InstructionARM64({// https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html // ADD , , {, {#}} InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), - Rn(addr1.id()), Rm(addr2.id())), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))), // https://www.scs.stanford.edu/~zyedidia/arm64/sturb.html // STURB , [{, #}] InstructionARM64(Base(0b0011100000000000000000, 22), Imm9s(offset), - Rt(value.id()), Rn(X16))}); + Rt(value.hw_id(instr_set)), Rn(X16))}); } InstructionARM64 load8s_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, @@ -284,7 +315,8 @@ InstructionARM64 load8s_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -298,7 +330,8 @@ InstructionARM64 load8s_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, // finally do the load // https://www.scs.stanford.edu/~zyedidia/arm64/ldrsb_imm.html // LDRSB , [], # - instrs.emplace_back(InstructionARM64(Base(0b0011100110, 10), Imm12(0), Rt(dst.id()), Rn(X16))); + instrs.emplace_back( + InstructionARM64(Base(0b0011100110, 10), Imm12(0), Rt(dst.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } @@ -317,7 +350,8 @@ InstructionARM64 store8_gpr64_gpr64_plus_gpr64_plus_s32(Register addr1, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -331,7 +365,8 @@ InstructionARM64 store8_gpr64_gpr64_plus_gpr64_plus_s32(Register addr1, // https://www.scs.stanford.edu/~zyedidia/arm64/strb_imm.html // unsigned offset // STRB , [{, #}] - instrs.emplace_back(InstructionARM64(Base(0b0011100100, 10), Imm12(0), Rt(value.id()), Rn(X16))); + instrs.emplace_back( + InstructionARM64(Base(0b0011100100, 10), Imm12(0), Rt(value.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } @@ -345,8 +380,8 @@ InstructionARM64 load8u_gpr64_gpr64_plus_gpr64(Register dst, Register addr1, Reg ASSERT(addr1 != addr2); ASSERT(addr1 != SP); ASSERT(addr2 != SP); - return InstructionARM64(Base(0b0011100001100000111010, 22), Rt(dst.id()), Rn(addr1.id()), - Rm(addr2.id())); + return InstructionARM64(Base(0b0011100001100000111010, 22), Rt(dst.hw_id(instr_set)), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))); } InstructionARM64 load8u_gpr64_gpr64_plus_gpr64_plus_s8(Register dst, @@ -362,23 +397,24 @@ InstructionARM64 load8u_gpr64_gpr64_plus_gpr64_plus_s8(Register dst, ASSERT(offset >= INT8_MIN && offset <= INT8_MAX); std::vector instrs; if (offset > 0) { - instrs = {// https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html - // ADD , , {, {#}} - InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), Rn(addr1.id()), - Rm(addr2.id())), - // https://www.scs.stanford.edu/~zyedidia/arm64/ldrb_imm.html - // Unsigned offset mode - // LDRB , [], # - InstructionARM64(Base(0b0011100101, 10), Imm12(offset), Rt(dst.id()), Rn(X16))}; - } else { instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html // ADD , , {, {#}} - InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), Rn(addr1.id()), - Rm(addr2.id())), - // https://www.scs.stanford.edu/~zyedidia/arm64/ldurb.html - // LDURB , [{, #}] - InstructionARM64(Base(0b0011100001000000000000, 22), Imm9s(offset), Rt(dst.id()), Rn(X16))}; + InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), + // https://www.scs.stanford.edu/~zyedidia/arm64/ldrb_imm.html + // Unsigned offset mode + // LDRB , [], # + InstructionARM64(Base(0b0011100101, 10), Imm12(offset), Rt(dst.hw_id(instr_set)), Rn(X16))}; + } else { + // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html + // ADD , , {, {#}} + instrs = {InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))), + // https://www.scs.stanford.edu/~zyedidia/arm64/ldurb.html + // LDURB , [{, #}] + InstructionARM64(Base(0b0011100001000000000000, 22), Imm9s(offset), + Rt(dst.hw_id(instr_set)), Rn(X16))}; } return InstructionARM64(instrs); } @@ -398,7 +434,8 @@ InstructionARM64 load8u_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; // TODO - movk instead eventually if (offset < 0) { @@ -413,7 +450,8 @@ InstructionARM64 load8u_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, // finally do the load // https://www.scs.stanford.edu/~zyedidia/arm64/ldrb_imm.html // LDRB , [], # - instrs.emplace_back(InstructionARM64(Base(0b0011100101, 10), Imm12(0), Rt(dst.id()), Rn(X16))); + instrs.emplace_back( + InstructionARM64(Base(0b0011100101, 10), Imm12(0), Rt(dst.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } @@ -426,8 +464,8 @@ InstructionARM64 load16s_gpr64_gpr64_plus_gpr64(Register dst, Register addr1, Re ASSERT(addr1 != addr2); ASSERT(addr1 != SP); ASSERT(addr2 != SP); - return InstructionARM64(Base(0b0111100010100000111010, 22), Rt(dst.id()), Rn(addr1.id()), - Rm(addr2.id())); + return InstructionARM64(Base(0b0111100010100000111010, 22), Rt(dst.hw_id(instr_set)), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))); } InstructionARM64 store16_gpr64_gpr64_plus_gpr64(Register addr1, Register addr2, Register value) { @@ -439,8 +477,8 @@ InstructionARM64 store16_gpr64_gpr64_plus_gpr64(Register addr1, Register addr2, ASSERT(addr1 != addr2); ASSERT(addr1 != SP); ASSERT(addr2 != SP); - return InstructionARM64(Base(0b0111100000100000111010, 22), Rt(value.id()), Rn(addr1.id()), - Rm(addr2.id())); + return InstructionARM64(Base(0b0111100000100000111010, 22), Rt(value.hw_id(instr_set)), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))); } InstructionARM64 store16_gpr64_gpr64_plus_gpr64_plus_s8(Register addr1, @@ -457,11 +495,11 @@ InstructionARM64 store16_gpr64_gpr64_plus_gpr64_plus_s8(Register addr1, return InstructionARM64({// https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html // ADD , , {, {#}} InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), - Rn(addr1.id()), Rm(addr2.id())), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))), // https://www.scs.stanford.edu/~zyedidia/arm64/sturh.html // STURH , [{, #}] InstructionARM64(Base(0b0111100000000000000000, 22), Imm9s(offset), - Rt(value.id()), Rn(X16))}); + Rt(value.hw_id(instr_set)), Rn(X16))}); } InstructionARM64 store16_gpr64_gpr64_plus_gpr64_plus_s32(Register addr1, @@ -479,7 +517,8 @@ InstructionARM64 store16_gpr64_gpr64_plus_gpr64_plus_s32(Register addr1, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -493,7 +532,8 @@ InstructionARM64 store16_gpr64_gpr64_plus_gpr64_plus_s32(Register addr1, // finally do the load // https://www.scs.stanford.edu/~zyedidia/arm64/strh_imm.html // STRH , [{, #}] - instrs.emplace_back(InstructionARM64(Base(0b0111100100, 10), Imm12(0), Rt(value.id()), Rn(X16))); + instrs.emplace_back( + InstructionARM64(Base(0b0111100100, 10), Imm12(0), Rt(value.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } @@ -508,14 +548,14 @@ InstructionARM64 load16s_gpr64_gpr64_plus_gpr64_plus_s8(Register dst, ASSERT(addr1 != SP); ASSERT(addr2 != SP); ASSERT(offset >= INT8_MIN && offset <= INT8_MAX); - return InstructionARM64( - {// https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html - // ADD , , {, {#}} - InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), Rn(addr1.id()), - Rm(addr2.id())), - // https://www.scs.stanford.edu/~zyedidia/arm64/ldursh.html - // LDURSH , [{, #}] - InstructionARM64(Base(0b0111100010000000000000, 22), Imm9s(offset), Rt(dst.id()), Rn(X16))}); + // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html + // ADD , , {, {#}} + return InstructionARM64({InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))), + // https://www.scs.stanford.edu/~zyedidia/arm64/ldursh.html + // LDURSH , [{, #}] + InstructionARM64(Base(0b0111100010000000000000, 22), Imm9s(offset), + Rt(dst.hw_id(instr_set)), Rn(X16))}); } InstructionARM64 load16s_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, @@ -533,7 +573,8 @@ InstructionARM64 load16s_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -547,7 +588,8 @@ InstructionARM64 load16s_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, // finally do the load // https://www.scs.stanford.edu/~zyedidia/arm64/ldrsh_imm.html // LDRSH , [], # - instrs.emplace_back(InstructionARM64(Base(0b0111100110, 10), Imm12(0), Rt(dst.id()), Rn(X16))); + instrs.emplace_back( + InstructionARM64(Base(0b0111100110, 10), Imm12(0), Rt(dst.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } @@ -561,8 +603,8 @@ InstructionARM64 load16u_gpr64_gpr64_plus_gpr64(Register dst, Register addr1, Re ASSERT(addr1 != addr2); ASSERT(addr1 != SP); ASSERT(addr2 != SP); - return InstructionARM64(Base(0b0111100001100000111010, 22), Rt(dst.id()), Rn(addr1.id()), - Rm(addr2.id())); + return InstructionARM64(Base(0b0111100001100000111010, 22), Rt(dst.hw_id(instr_set)), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))); } InstructionARM64 load16u_gpr64_gpr64_plus_gpr64_plus_s8(Register dst, @@ -576,27 +618,15 @@ InstructionARM64 load16u_gpr64_gpr64_plus_gpr64_plus_s8(Register dst, ASSERT(addr1 != SP); ASSERT(addr2 != SP); ASSERT(offset >= INT8_MIN && offset <= INT8_MAX); - std::vector instrs; - if (offset > 0) { - instrs = {// https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html - // ADD , , {, {#}} - InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), Rn(addr1.id()), - Rm(addr2.id())), - // https://www.scs.stanford.edu/~zyedidia/arm64/ldrh_imm.html - // Unsigned offset mode - // LDRH , [{, #}] - InstructionARM64(Base(0b0111100101, 10), Imm12(offset), Rt(dst.id()), Rn(X16))}; - } else { - instrs = { - // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html - // ADD , , {, {#}} - InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), Rn(addr1.id()), - Rm(addr2.id())), - // https://www.scs.stanford.edu/~zyedidia/arm64/ldurh.html - // LDURH , [{, #}] - InstructionARM64(Base(0b0111100001000000000000, 22), Imm9s(offset), Rt(dst.id()), Rn(X16))}; - } - return InstructionARM64(instrs); + // LDURH covers the full signed byte offset range + // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html + // ADD , , {, {#}} + return InstructionARM64({InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))), + // https://www.scs.stanford.edu/~zyedidia/arm64/ldurh.html + // LDURH , [{, #}] + InstructionARM64(Base(0b0111100001000000000000, 22), Imm9s(offset), + Rt(dst.hw_id(instr_set)), Rn(X16))}); } InstructionARM64 load16u_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, @@ -614,7 +644,8 @@ InstructionARM64 load16u_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -628,7 +659,8 @@ InstructionARM64 load16u_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, // finally do the load // https://www.scs.stanford.edu/~zyedidia/arm64/ldrh_imm.html // LDRH , [{, #}] - instrs.emplace_back(InstructionARM64(Base(0b0111100101, 10), Imm12(0), Rt(dst.id()), Rn(X16))); + instrs.emplace_back( + InstructionARM64(Base(0b0111100101, 10), Imm12(0), Rt(dst.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } @@ -641,8 +673,8 @@ InstructionARM64 load32s_gpr64_gpr64_plus_gpr64(Register dst, Register addr1, Re ASSERT(addr1 != addr2); ASSERT(addr1 != SP); ASSERT(addr2 != SP); - return InstructionARM64(Base(0b1011100010100000111010, 22), Rt(dst.id()), Rn(addr1.id()), - Rm(addr2.id())); + return InstructionARM64(Base(0b1011100010100000111010, 22), Rt(dst.hw_id(instr_set)), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))); } InstructionARM64 store32_gpr64_gpr64_plus_gpr64(Register addr1, Register addr2, Register value) { @@ -654,8 +686,8 @@ InstructionARM64 store32_gpr64_gpr64_plus_gpr64(Register addr1, Register addr2, ASSERT(addr1 != addr2); ASSERT(addr1 != SP); ASSERT(addr2 != SP); - return InstructionARM64(Base(0b1011100000100000111010, 22), Rt(value.id()), Rn(addr1.id()), - Rm(addr2.id())); + return InstructionARM64(Base(0b1011100000100000111010, 22), Rt(value.hw_id(instr_set)), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))); } InstructionARM64 load32s_gpr64_gpr64_plus_gpr64_plus_s8(Register dst, @@ -669,14 +701,14 @@ InstructionARM64 load32s_gpr64_gpr64_plus_gpr64_plus_s8(Register dst, ASSERT(addr1 != SP); ASSERT(addr2 != SP); ASSERT(offset >= INT8_MIN && offset <= INT8_MAX); - return InstructionARM64( - {// https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html - // ADD , , {, {#}} - InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), Rn(addr1.id()), - Rm(addr2.id())), - // https://www.scs.stanford.edu/~zyedidia/arm64/ldursw.html - // LDURSW , [{, #}] - InstructionARM64(Base(0b1011100010000000000000, 22), Imm9s(offset), Rt(dst.id()), Rn(X16))}); + // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html + // ADD , , {, {#}} + return InstructionARM64({InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))), + // https://www.scs.stanford.edu/~zyedidia/arm64/ldursw.html + // LDURSW , [{, #}] + InstructionARM64(Base(0b1011100010000000000000, 22), Imm9s(offset), + Rt(dst.hw_id(instr_set)), Rn(X16))}); } InstructionARM64 store32_gpr64_gpr64_plus_gpr64_plus_s8(Register addr1, @@ -694,7 +726,8 @@ InstructionARM64 store32_gpr64_gpr64_plus_gpr64_plus_s8(Register addr1, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -707,7 +740,8 @@ InstructionARM64 store32_gpr64_gpr64_plus_gpr64_plus_s8(Register addr1, } // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_gen.html // STR , [{, #}] - instrs.emplace_back(InstructionARM64(Base(0b1011100100, 10), Imm12(0), Rt(value.id()), Rn(X16))); + instrs.emplace_back( + InstructionARM64(Base(0b1011100100, 10), Imm12(0), Rt(value.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } @@ -726,7 +760,8 @@ InstructionARM64 load32s_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -740,7 +775,8 @@ InstructionARM64 load32s_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, // finally do the load // https://www.scs.stanford.edu/~zyedidia/arm64/ldrsw_imm.html // LDRSW , [], # - instrs.emplace_back(InstructionARM64(Base(0b1011100110, 10), Imm12(0), Rt(dst.id()), Rn(X16))); + instrs.emplace_back( + InstructionARM64(Base(0b1011100110, 10), Imm12(0), Rt(dst.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } @@ -759,7 +795,8 @@ InstructionARM64 store32_gpr64_gpr64_plus_gpr64_plus_s32(Register addr1, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -772,7 +809,8 @@ InstructionARM64 store32_gpr64_gpr64_plus_gpr64_plus_s32(Register addr1, } // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_gen.html // STR , [{, #}] - instrs.emplace_back(InstructionARM64(Base(0b1011100100, 10), Imm12(0), Rt(value.id()), Rn(X16))); + instrs.emplace_back( + InstructionARM64(Base(0b1011100100, 10), Imm12(0), Rt(value.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } @@ -786,8 +824,8 @@ InstructionARM64 load32u_gpr64_gpr64_plus_gpr64(Register dst, Register addr1, Re ASSERT(addr1 != addr2); ASSERT(addr1 != SP); ASSERT(addr2 != SP); - return InstructionARM64(Base(0b1011100001100000111010, 22), Rt(dst.id()), Rn(addr1.id()), - Rm(addr2.id())); + return InstructionARM64(Base(0b1011100001100000111010, 22), Rt(dst.hw_id(instr_set)), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))); } InstructionARM64 load32u_gpr64_gpr64_plus_gpr64_plus_s8(Register dst, @@ -801,15 +839,15 @@ InstructionARM64 load32u_gpr64_gpr64_plus_gpr64_plus_s8(Register dst, ASSERT(addr1 != SP); ASSERT(addr2 != SP); ASSERT(offset >= INT8_MIN && offset <= INT8_MAX); - return InstructionARM64( - {// https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html - // ADD , , {, {#}} - InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), Rn(addr1.id()), - Rm(addr2.id())), - // https://www.scs.stanford.edu/~zyedidia/arm64/ldur_gen.html - // 32 bit - // LDUR , [{, #}] - InstructionARM64(Base(0b1011100001000000000000, 22), Imm9s(offset), Rt(dst.id()), Rn(X16))}); + // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html + // ADD , , {, {#}} + return InstructionARM64({InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))), + // https://www.scs.stanford.edu/~zyedidia/arm64/ldur_gen.html + // 32 bit + // LDUR , [{, #}] + InstructionARM64(Base(0b1011100001000000000000, 22), Imm9s(offset), + Rt(dst.hw_id(instr_set)), Rn(X16))}); } InstructionARM64 load32u_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, @@ -827,7 +865,8 @@ InstructionARM64 load32u_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -839,11 +878,11 @@ InstructionARM64 load32u_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end()); } // finally do the load - // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_gen.html + // https://www.scs.stanford.edu/~zyedidia/arm64/ldur_gen.html // 32-bit variant - // LDR , [], # - instrs.emplace_back( - InstructionARM64(Base(0b1011100001000000000001, 22), Imm9s(0), Rt(dst.id()), Rn(X16))); + // LDUR , [{, #}] + instrs.emplace_back(InstructionARM64(Base(0b1011100001000000000000, 22), Imm9s(0), + Rt(dst.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } @@ -857,8 +896,8 @@ InstructionARM64 load64_gpr64_gpr64_plus_gpr64(Register dst, Register addr1, Reg ASSERT(addr1 != addr2); ASSERT(addr1 != SP); ASSERT(addr2 != SP); - return InstructionARM64(Base(0b1111100001100000111010, 22), Rt(dst.id()), Rn(addr1.id()), - Rm(addr2.id())); + return InstructionARM64(Base(0b1111100001100000111010, 22), Rt(dst.hw_id(instr_set)), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))); } InstructionARM64 store64_gpr64_gpr64_plus_gpr64(Register addr1, Register addr2, Register value) { @@ -870,8 +909,8 @@ InstructionARM64 store64_gpr64_gpr64_plus_gpr64(Register addr1, Register addr2, ASSERT(addr1 != addr2); ASSERT(addr1 != SP); ASSERT(addr2 != SP); - return InstructionARM64(Base(0b1111100000100000111010, 22), Rt(value.id()), Rn(addr1.id()), - Rm(addr2.id())); + return InstructionARM64(Base(0b1111100000100000111010, 22), Rt(value.hw_id(instr_set)), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))); } InstructionARM64 load64_gpr64_gpr64_plus_gpr64_plus_s8(Register dst, @@ -885,15 +924,15 @@ InstructionARM64 load64_gpr64_gpr64_plus_gpr64_plus_s8(Register dst, ASSERT(addr1 != SP); ASSERT(addr2 != SP); ASSERT(offset >= INT8_MIN && offset <= INT8_MAX); - return InstructionARM64( - {// https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html - // ADD , , {, {#}} - InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), Rn(addr1.id()), - Rm(addr2.id())), - // https://www.scs.stanford.edu/~zyedidia/arm64/ldur_gen.html - // 64 bit - // LDUR , [{, #}] - InstructionARM64(Base(0b1111100001000000000000, 22), Imm9s(offset), Rt(dst.id()), Rn(X16))}); + // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_ext.html + // ADD , , {, {#}} + return InstructionARM64({InstructionARM64(Base(0b1000101100100000111000, 22), Rd(X16), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))), + // https://www.scs.stanford.edu/~zyedidia/arm64/ldur_gen.html + // 64 bit + // LDUR , [{, #}] + InstructionARM64(Base(0b1111100001000000000000, 22), Imm9s(offset), + Rt(dst.hw_id(instr_set)), Rn(X16))}); } InstructionARM64 store64_gpr64_gpr64_plus_gpr64_plus_s8(Register addr1, @@ -911,7 +950,8 @@ InstructionARM64 store64_gpr64_gpr64_plus_gpr64_plus_s8(Register addr1, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -924,7 +964,8 @@ InstructionARM64 store64_gpr64_gpr64_plus_gpr64_plus_s8(Register addr1, } // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_gen.html // STR , [{, #}] - instrs.emplace_back(InstructionARM64(Base(0b1111100100, 10), Imm12(0), Rt(value.id()), Rn(X16))); + instrs.emplace_back( + InstructionARM64(Base(0b1111100100, 10), Imm12(0), Rt(value.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } @@ -943,7 +984,8 @@ InstructionARM64 load64_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -955,11 +997,11 @@ InstructionARM64 load64_gpr64_gpr64_plus_gpr64_plus_s32(Register dst, instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end()); } // finally do the load - // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_gen.html + // https://www.scs.stanford.edu/~zyedidia/arm64/ldur_gen.html // 64-bit variant - // LDR , [], # - instrs.emplace_back( - InstructionARM64(Base(0b1111100001000000000001, 22), Imm9s(0), Rt(dst.id()), Rn(X16))); + // LDUR , [{, #}] + instrs.emplace_back(InstructionARM64(Base(0b1111100001000000000000, 22), Imm9s(0), + Rt(dst.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } @@ -978,7 +1020,8 @@ InstructionARM64 store64_gpr64_gpr64_plus_gpr64_plus_s32(Register addr1, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -991,19 +1034,21 @@ InstructionARM64 store64_gpr64_gpr64_plus_gpr64_plus_s32(Register addr1, } // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_gen.html // STR , [{, #}] - instrs.emplace_back(InstructionARM64(Base(0b1111100100, 10), Imm12(0), Rt(value.id()), Rn(X16))); + instrs.emplace_back( + InstructionARM64(Base(0b1111100100, 10), Imm12(0), Rt(value.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } InstructionARM64 load64_gpr64_plus_s32(Register dst_reg, int32_t offset, Register src_reg) { ASSERT(dst_reg.is_gpr(instr_set)); ASSERT(src_reg.is_gpr(instr_set)); - ASSERT(src_reg != SP); + // allow SP as the base for spill-slot addressing with immediate ADD ASSERT(offset >= INT32_MIN && offset <= INT32_MAX); std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html // ADD , , #{, } - InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(src_reg.id())), + InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), + Rn(src_reg.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -1015,23 +1060,23 @@ InstructionARM64 load64_gpr64_plus_s32(Register dst_reg, int32_t offset, Registe instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end()); } // finally do the load - // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_gen.html + // https://www.scs.stanford.edu/~zyedidia/arm64/ldur_gen.html // 64-bit variant - // LDR , [], # - instrs.emplace_back( - InstructionARM64(Base(0b1111100001000000000001, 22), Imm9s(0), Rt(dst_reg.id()), Rn(X16))); + // LDUR , [{, #}] + instrs.emplace_back(InstructionARM64(Base(0b1111100001000000000000, 22), Imm9s(0), + Rt(dst_reg.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } InstructionARM64 store64_gpr64_plus_s32(Register addr, int32_t offset, Register value) { ASSERT(value.is_gpr(instr_set)); ASSERT(addr.is_gpr(instr_set)); - ASSERT(addr != SP); + // allow SP as the base for immediate ADD ASSERT(offset >= INT32_MIN && offset <= INT32_MAX); std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html // ADD , , #{, } - InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(addr.id())), + InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(addr.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -1044,7 +1089,8 @@ InstructionARM64 store64_gpr64_plus_s32(Register addr, int32_t offset, Register } // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_gen.html // STR , [{, #}] - instrs.emplace_back(InstructionARM64(Base(0b1111100100, 10), Imm12(0), Rt(value.id()), Rn(X16))); + instrs.emplace_back( + InstructionARM64(Base(0b1111100100, 10), Imm12(0), Rt(value.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } @@ -1108,7 +1154,7 @@ InstructionARM64 store_goal_gpr(Register addr, Register value, Register off, int } } -InstructionARM64 load_goal_xmm128(Register dst, Register addr, Register off, int offset) { +InstructionARM64 load_goal_simd128(Register dst, Register addr, Register off, int offset) { if (offset == 0) { return loadvf_gpr64_plus_gpr64(dst, addr, off); } else if (offset >= INT8_MIN && offset <= INT8_MAX) { @@ -1221,15 +1267,16 @@ InstructionARM64 lea_reg_plus_off32(Register dest, Register base, s64 offset) { std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html // ADD , , #{, } - InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(dest.id()), Rn(base.id())), + InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(dest.hw_id(instr_set)), + Rn(base.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead offset = std::abs(offset); - const auto sub_instrs = construct_multiple_imm12_subs(offset, dest.id()); + const auto sub_instrs = construct_multiple_imm12_subs(offset, dest.hw_id(instr_set)); instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end()); } else { - const auto add_instrs = construct_multiple_imm12_adds(offset, dest.id()); + const auto add_instrs = construct_multiple_imm12_adds(offset, dest.hw_id(instr_set)); instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end()); } return InstructionARM64(instrs); @@ -1243,15 +1290,16 @@ InstructionARM64 lea_reg_plus_off8(Register dest, Register base, s64 offset) { std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html // ADD , , #{, } - InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(dest.id()), Rn(base.id())), + InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(dest.hw_id(instr_set)), + Rn(base.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead offset = std::abs(offset); - const auto sub_instrs = construct_multiple_imm12_subs(offset, dest.id()); + const auto sub_instrs = construct_multiple_imm12_subs(offset, dest.hw_id(instr_set)); instrs.insert(instrs.end(), sub_instrs.begin(), sub_instrs.end()); } else { - const auto add_instrs = construct_multiple_imm12_adds(offset, dest.id()); + const auto add_instrs = construct_multiple_imm12_adds(offset, dest.hw_id(instr_set)); instrs.insert(instrs.end(), add_instrs.begin(), add_instrs.end()); } return InstructionARM64(instrs); @@ -1269,40 +1317,42 @@ InstructionARM64 lea_reg_plus_off(Register dest, Register base, s64 offset) { } //;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -// LOADS n' STORES - XMM32 +// LOADS n' STORES - SIMD32 //;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -// TODO - rename these to f32 instead of xmm +// TODO - rename these to f32 -InstructionARM64 store32_xmm32_gpr64_plus_gpr64(Register addr1, - Register addr2, - Register xmm_value) { - ASSERT(xmm_value.is_128bit_simd(instr_set)); +InstructionARM64 store32_simd32_gpr64_plus_gpr64(Register addr1, + Register addr2, + Register simd_value) { + ASSERT(simd_value.is_128bit_simd(instr_set)); ASSERT(addr1.is_gpr(instr_set)); ASSERT(addr2.is_gpr(instr_set)); // https://www.scs.stanford.edu/~zyedidia/arm64/str_reg_fpsimd.html // 32-bit variant // STR , [, (|){, {}}] - return InstructionARM64(Base(0b1011110000100000110010, 22), Rt(xmm_value.id()), Rm(addr2.id()), - Rn(addr1.id())); + return InstructionARM64(Base(0b1011110000100000111010, 22), Rt(simd_value.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set)), Rn(addr1.hw_id(instr_set))); } -InstructionARM64 load32_xmm32_gpr64_plus_gpr64(Register simd_dest, Register addr1, Register addr2) { +InstructionARM64 load32_simd32_gpr64_plus_gpr64(Register simd_dest, + Register addr1, + Register addr2) { ASSERT(simd_dest.is_128bit_simd(instr_set)); ASSERT(addr1.is_gpr(instr_set)); ASSERT(addr2.is_gpr(instr_set)); // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_reg_fpsimd.html // 32-bit variant // LDR , [, (|){, {}}] - return InstructionARM64(Base(0b1011110001100000111010, 22), Rt(simd_dest.id()), Rm(addr1.id()), - Rn(addr2.id())); + return InstructionARM64(Base(0b1011110001100000111010, 22), Rt(simd_dest.hw_id(instr_set)), + Rn(addr1.hw_id(instr_set)), Rm(addr2.hw_id(instr_set))); } -InstructionARM64 store32_xmm32_gpr64_plus_gpr64_plus_s8(Register addr1, - Register addr2, - Register xmm_value, - s64 offset) { - ASSERT(xmm_value.is_128bit_simd(instr_set)); +InstructionARM64 store32_simd32_gpr64_plus_gpr64_plus_s8(Register addr1, + Register addr2, + Register simd_value, + s64 offset) { + ASSERT(simd_value.is_128bit_simd(instr_set)); ASSERT(addr1.is_gpr(instr_set)); ASSERT(addr2.is_gpr(instr_set)); ASSERT(offset >= INT8_MIN && offset <= INT8_MAX); @@ -1310,7 +1360,8 @@ InstructionARM64 store32_xmm32_gpr64_plus_gpr64_plus_s8(Register addr1, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; // TODO - optimization, if its less than imm12 we can just do an add/sub auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset)); @@ -1325,15 +1376,15 @@ InstructionARM64 store32_xmm32_gpr64_plus_gpr64_plus_s8(Register addr1, // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html // 32-bit variant // STR , [], # - instrs.emplace_back( - InstructionARM64(Base(0b1011110100000000000000, 22), Imm12(0), Rt(xmm_value.id()), Rn(X16))); + instrs.emplace_back(InstructionARM64(Base(0b1011110100000000000000, 22), Imm12(0), + Rt(simd_value.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } -InstructionARM64 load32_xmm32_gpr64_plus_gpr64_plus_s8(Register simd_dest, - Register addr1, - Register addr2, - s64 offset) { +InstructionARM64 load32_simd32_gpr64_plus_gpr64_plus_s8(Register simd_dest, + Register addr1, + Register addr2, + s64 offset) { ASSERT(simd_dest.is_128bit_simd(instr_set)); ASSERT(addr1.is_gpr(instr_set)); ASSERT(addr2.is_gpr(instr_set)); @@ -1342,7 +1393,8 @@ InstructionARM64 load32_xmm32_gpr64_plus_gpr64_plus_s8(Register simd_dest, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; // TODO - optimization, if its less than imm12 we can just do an add/sub auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset)); @@ -1358,15 +1410,15 @@ InstructionARM64 load32_xmm32_gpr64_plus_gpr64_plus_s8(Register simd_dest, // 32-bit variant // LDR , [{, #}] instrs.emplace_back( - InstructionARM64(Base(0b1011110101, 10), Imm12(0), Rt(simd_dest.id()), Rn(X16))); + InstructionARM64(Base(0b1011110101, 10), Imm12(0), Rt(simd_dest.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } -InstructionARM64 store32_xmm32_gpr64_plus_gpr64_plus_s32(Register addr1, - Register addr2, - Register xmm_value, - s64 offset) { - ASSERT(xmm_value.is_128bit_simd(instr_set)); +InstructionARM64 store32_simd32_gpr64_plus_gpr64_plus_s32(Register addr1, + Register addr2, + Register simd_value, + s64 offset) { + ASSERT(simd_value.is_128bit_simd(instr_set)); ASSERT(addr1.is_gpr(instr_set)); ASSERT(addr2.is_gpr(instr_set)); ASSERT(offset >= INT32_MIN && offset <= INT32_MAX); @@ -1374,7 +1426,8 @@ InstructionARM64 store32_xmm32_gpr64_plus_gpr64_plus_s32(Register addr1, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; // TODO - optimization, if its less than imm12 we can just do an add/sub auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset)); @@ -1389,20 +1442,20 @@ InstructionARM64 store32_xmm32_gpr64_plus_gpr64_plus_s32(Register addr1, // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html // 32-bit variant // STR , [], # - instrs.emplace_back( - InstructionARM64(Base(0b1011110100000000000000, 22), Imm12(0), Rt(xmm_value.id()), Rn(X16))); + instrs.emplace_back(InstructionARM64(Base(0b1011110100000000000000, 22), Imm12(0), + Rt(simd_value.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } -InstructionARM64 store32_xmm32_gpr64_plus_s32(Register base, Register xmm_value, s64 offset) { - ASSERT(xmm_value.is_128bit_simd(instr_set)); +InstructionARM64 store32_simd32_gpr64_plus_s32(Register base, Register simd_value, s64 offset) { + ASSERT(simd_value.is_128bit_simd(instr_set)); ASSERT(base.is_gpr(instr_set)); ASSERT(offset >= INT32_MIN && offset <= INT32_MAX); // first establish the base+index value in x16 std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html // ADD , , #{, } - InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(base.id())), + InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(base.hw_id(instr_set))), }; // TODO - optimization, if its less than imm12 we can just do an add/sub auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset)); @@ -1417,20 +1470,20 @@ InstructionARM64 store32_xmm32_gpr64_plus_s32(Register base, Register xmm_value, // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html // 32-bit variant // STR , [], # - instrs.emplace_back( - InstructionARM64(Base(0b1011110100000000000000, 22), Imm12(0), Rt(xmm_value.id()), Rn(X16))); + instrs.emplace_back(InstructionARM64(Base(0b1011110100000000000000, 22), Imm12(0), + Rt(simd_value.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } -InstructionARM64 store32_xmm32_gpr64_plus_s8(Register base, Register xmm_value, s64 offset) { - ASSERT(xmm_value.is_128bit_simd(instr_set)); +InstructionARM64 store32_simd32_gpr64_plus_s8(Register base, Register simd_value, s64 offset) { + ASSERT(simd_value.is_128bit_simd(instr_set)); ASSERT(base.is_gpr(instr_set)); ASSERT(offset >= INT8_MIN && offset <= INT8_MAX); // first establish the base+index value in x16 std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html // ADD , , #{, } - InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(base.id())), + InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(base.hw_id(instr_set))), }; // TODO - optimization, if its less than imm12 we can just do an add/sub auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset)); @@ -1445,15 +1498,15 @@ InstructionARM64 store32_xmm32_gpr64_plus_s8(Register base, Register xmm_value, // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html // 32-bit variant, unsigned // STR , [], # - instrs.emplace_back( - InstructionARM64(Base(0b1011110100000000000000, 22), Imm12(0), Rt(xmm_value.id()), Rn(X16))); + instrs.emplace_back(InstructionARM64(Base(0b1011110100000000000000, 22), Imm12(0), + Rt(simd_value.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } -InstructionARM64 load32_xmm32_gpr64_plus_gpr64_plus_s32(Register simd_dest, - Register addr1, - Register addr2, - s64 offset) { +InstructionARM64 load32_simd32_gpr64_plus_gpr64_plus_s32(Register simd_dest, + Register addr1, + Register addr2, + s64 offset) { ASSERT(simd_dest.is_128bit_simd(instr_set)); ASSERT(addr1.is_gpr(instr_set)); ASSERT(addr2.is_gpr(instr_set)); @@ -1462,7 +1515,8 @@ InstructionARM64 load32_xmm32_gpr64_plus_gpr64_plus_s32(Register simd_dest, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_shift.html // ADD , , {, #} - InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.id()), Rm(addr2.id())), + InstructionARM64(Base(0b10001011000, 11), Rd(X16), Imm6(0), Rn(addr1.hw_id(instr_set)), + Rm(addr2.hw_id(instr_set))), }; // TODO - optimization, if its less than imm12 we can just do an add/sub auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset)); @@ -1478,11 +1532,11 @@ InstructionARM64 load32_xmm32_gpr64_plus_gpr64_plus_s32(Register simd_dest, // 32-bit variant // LDR , [{, #}] instrs.emplace_back( - InstructionARM64(Base(0b1011110101, 10), Imm12(0), Rt(simd_dest.id()), Rn(X16))); + InstructionARM64(Base(0b1011110101, 10), Imm12(0), Rt(simd_dest.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } -InstructionARM64 load32_xmm32_gpr64_plus_s32(Register simd_dest, Register base, s64 offset) { +InstructionARM64 load32_simd32_gpr64_plus_s32(Register simd_dest, Register base, s64 offset) { ASSERT(simd_dest.is_128bit_simd(instr_set)); ASSERT(base.is_gpr(instr_set)); ASSERT(offset >= INT32_MIN && offset <= INT32_MAX); @@ -1490,7 +1544,7 @@ InstructionARM64 load32_xmm32_gpr64_plus_s32(Register simd_dest, Register base, std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html // ADD , , #{, } - InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(base.id())), + InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(base.hw_id(instr_set))), }; // TODO - optimization, if its less than imm12 we can just do an add/sub auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset)); @@ -1506,11 +1560,11 @@ InstructionARM64 load32_xmm32_gpr64_plus_s32(Register simd_dest, Register base, // 32-bit variant // LDR , [{, #}] instrs.emplace_back( - InstructionARM64(Base(0b1011110101, 10), Imm12(0), Rt(simd_dest.id()), Rn(X16))); + InstructionARM64(Base(0b1011110101, 10), Imm12(0), Rt(simd_dest.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } -InstructionARM64 load32_xmm32_gpr64_plus_s8(Register simd_dest, Register base, s64 offset) { +InstructionARM64 load32_simd32_gpr64_plus_s8(Register simd_dest, Register base, s64 offset) { ASSERT(simd_dest.is_128bit_simd(instr_set)); ASSERT(base.is_gpr(instr_set)); ASSERT(offset >= INT8_MIN && offset <= INT8_MAX); @@ -1518,7 +1572,7 @@ InstructionARM64 load32_xmm32_gpr64_plus_s8(Register simd_dest, Register base, s std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html // ADD , , #{, } - InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(base.id())), + InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(base.hw_id(instr_set))), }; // TODO - optimization, if its less than imm12 we can just do an add/sub auto mov_instrs = mov_gpr64_u64_instrs(X17, std::abs(offset)); @@ -1534,52 +1588,52 @@ InstructionARM64 load32_xmm32_gpr64_plus_s8(Register simd_dest, Register base, s // 32-bit variant // LDR , [{, #}] instrs.emplace_back( - InstructionARM64(Base(0b1011110101, 10), Imm12(0), Rt(simd_dest.id()), Rn(X16))); + InstructionARM64(Base(0b1011110101, 10), Imm12(0), Rt(simd_dest.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } -InstructionARM64 load_goal_xmm32(Register simd_dest, Register addr, Register off, s64 offset) { +InstructionARM64 load_goal_simd32(Register simd_dest, Register addr, Register off, s64 offset) { if (offset == 0) { - return load32_xmm32_gpr64_plus_gpr64(simd_dest, addr, off); + return load32_simd32_gpr64_plus_gpr64(simd_dest, addr, off); } else if (offset >= INT8_MIN && offset <= INT8_MAX) { - return load32_xmm32_gpr64_plus_gpr64_plus_s8(simd_dest, addr, off, offset); + return load32_simd32_gpr64_plus_gpr64_plus_s8(simd_dest, addr, off, offset); } else if (offset >= INT32_MIN && offset <= INT32_MAX) { - return load32_xmm32_gpr64_plus_gpr64_plus_s32(simd_dest, addr, off, offset); + return load32_simd32_gpr64_plus_gpr64_plus_s32(simd_dest, addr, off, offset); } else { ASSERT(false); return {0}; } } -InstructionARM64 store_goal_xmm32(Register addr, Register xmm_value, Register off, s64 offset) { +InstructionARM64 store_goal_simd32(Register addr, Register simd_value, Register off, s64 offset) { if (offset == 0) { - return store32_xmm32_gpr64_plus_gpr64(addr, off, xmm_value); + return store32_simd32_gpr64_plus_gpr64(addr, off, simd_value); } else if (offset >= INT8_MIN && offset <= INT8_MAX) { - return store32_xmm32_gpr64_plus_gpr64_plus_s8(addr, off, xmm_value, offset); + return store32_simd32_gpr64_plus_gpr64_plus_s8(addr, off, simd_value, offset); } else if (offset >= INT32_MIN && offset <= INT32_MAX) { - return store32_xmm32_gpr64_plus_gpr64_plus_s32(addr, off, xmm_value, offset); + return store32_simd32_gpr64_plus_gpr64_plus_s32(addr, off, simd_value, offset); } else { ASSERT(false); return {0}; } } -InstructionARM64 store_reg_offset_xmm32(Register base, Register xmm_value, s64 offset) { +InstructionARM64 store_reg_offset_simd32(Register base, Register simd_value, s64 offset) { if (offset >= INT8_MIN && offset <= INT8_MAX) { - return store32_xmm32_gpr64_plus_s8(base, xmm_value, offset); + return store32_simd32_gpr64_plus_s8(base, simd_value, offset); } else if (offset >= INT32_MIN && offset <= INT32_MAX) { - return store32_xmm32_gpr64_plus_s32(base, xmm_value, offset); + return store32_simd32_gpr64_plus_s32(base, simd_value, offset); } else { ASSERT(false); return {0}; } } -InstructionARM64 load_reg_offset_xmm32(Register simd_dest, Register base, s64 offset) { +InstructionARM64 load_reg_offset_simd32(Register simd_dest, Register base, s64 offset) { if (offset >= INT8_MIN && offset <= INT8_MAX) { - return load32_xmm32_gpr64_plus_s8(simd_dest, base, offset); + return load32_simd32_gpr64_plus_s8(simd_dest, base, offset); } else if (offset >= INT32_MIN && offset <= INT32_MAX) { - return load32_xmm32_gpr64_plus_s32(simd_dest, base, offset); + return load32_simd32_gpr64_plus_s32(simd_dest, base, offset); } else { ASSERT(false); return {0}; @@ -1594,23 +1648,21 @@ InstructionARM64 store128_gpr64_simd128(Register gpr_addr, Register simd_reg) { // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html // - STR Qn, [Xn] (unsigned offset) ASSERT(gpr_addr.is_gpr(instr_set)); - ASSERT( - simd_reg.is_128bit_simd(instr_set)); // TODO ARM64 - this assertion isn't as useful for ARM - // since Q registers are not unique in terms of their id - return InstructionARM64(Base(0b0011110110, 10), Rn(gpr_addr.id()), Rt(simd_reg.id()), Imm12(0)); + ASSERT(simd_reg.is_128bit_simd(instr_set)); + return InstructionARM64(Base(0b0011110110, 10), Rn(gpr_addr.hw_id(instr_set)), + Rt(simd_reg.hw_id(instr_set)), Imm12(0)); } -InstructionARM64 store128_gpr64_simd128_s32(Register gpr_addr, Register xmm_value, s64 offset) { +InstructionARM64 store128_gpr64_simd128_s32(Register gpr_addr, Register simd_value, s64 offset) { ASSERT(gpr_addr.is_gpr(instr_set)); - ASSERT(xmm_value.is_128bit_simd( - instr_set)); // TODO ARM64 - this assertion isn't as useful for ARM - // since Q registers are not unique in terms of their id + ASSERT(simd_value.is_128bit_simd(instr_set)); ASSERT(offset >= INT32_MIN && offset <= INT32_MAX); // first establish the base+index value in x16 std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html // ADD , , #{, } - InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(gpr_addr.id())), + InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), + Rn(gpr_addr.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -1624,22 +1676,22 @@ InstructionARM64 store128_gpr64_simd128_s32(Register gpr_addr, Register xmm_valu // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html // 128-bit variant, unsigned offset // STR , [{, #}] + // width 10 keeps the literal in its opcode field instrs.emplace_back( - InstructionARM64(Base(0b0011110110, 22), Imm12(0), Rt(xmm_value.id()), Rn(X16))); + InstructionARM64(Base(0b0011110110, 10), Imm12(0), Rt(simd_value.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } -InstructionARM64 store128_gpr64_simd128_s8(Register gpr_addr, Register xmm_value, s64 offset) { +InstructionARM64 store128_gpr64_simd128_s8(Register gpr_addr, Register simd_value, s64 offset) { ASSERT(gpr_addr.is_gpr(instr_set)); - ASSERT(xmm_value.is_128bit_simd( - instr_set)); // TODO ARM64 - this assertion isn't as useful for ARM - // since Q registers are not unique in terms of their id + ASSERT(simd_value.is_128bit_simd(instr_set)); ASSERT(offset >= INT8_MIN && offset <= INT8_MAX); // first establish the base+index value in x16 std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html // ADD , , #{, } - InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(gpr_addr.id())), + InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), + Rn(gpr_addr.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -1653,8 +1705,9 @@ InstructionARM64 store128_gpr64_simd128_s8(Register gpr_addr, Register xmm_value // https://www.scs.stanford.edu/~zyedidia/arm64/str_imm_fpsimd.html // 128-bit variant, unsigned offset // STR , [{, #}] + // width 10 keeps the literal in its opcode field instrs.emplace_back( - InstructionARM64(Base(0b0011110110, 22), Imm12(0), Rt(xmm_value.id()), Rn(X16))); + InstructionARM64(Base(0b0011110110, 10), Imm12(0), Rt(simd_value.hw_id(instr_set)), Rn(X16))); return InstructionARM64(instrs); } @@ -1662,23 +1715,21 @@ InstructionARM64 load128_simd128_gpr64(Register simd_dest, Register gpr_addr) { // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_fpsimd.html // - LDR , [{, #}] ASSERT(gpr_addr.is_gpr(instr_set)); - ASSERT(simd_dest.is_128bit_simd( - instr_set)); // TODO ARM64 - this assertion isn't as useful for ARM - // since Q registers are not unique in terms of their id - return InstructionARM64(Base(0b0011110111, 10), Rn(gpr_addr.id()), Rt(simd_dest.id()), Imm12(0)); + ASSERT(simd_dest.is_128bit_simd(instr_set)); + return InstructionARM64(Base(0b0011110111, 10), Rn(gpr_addr.hw_id(instr_set)), + Rt(simd_dest.hw_id(instr_set)), Imm12(0)); } InstructionARM64 load128_simd128_gpr64_s32(Register simd_dest, Register gpr_addr, s64 offset) { ASSERT(gpr_addr.is_gpr(instr_set)); - ASSERT(simd_dest.is_128bit_simd( - instr_set)); // TODO ARM64 - this assertion isn't as useful for ARM - // since Q registers are not unique in terms of their id + ASSERT(simd_dest.is_128bit_simd(instr_set)); ASSERT(offset >= INT32_MIN && offset <= INT32_MAX); // first establish the base+index value in x16 std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html // ADD , , #{, } - InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(gpr_addr.id())), + InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), + Rn(gpr_addr.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -1692,21 +1743,20 @@ InstructionARM64 load128_simd128_gpr64_s32(Register simd_dest, Register gpr_addr // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_fpsimd.html // - LDR , [{, #}] instrs.emplace_back( - InstructionARM64(Base(0b0011110111, 10), Rn(X16), Rt(simd_dest.id()), Imm12(0))); + InstructionARM64(Base(0b0011110111, 10), Rn(X16), Rt(simd_dest.hw_id(instr_set)), Imm12(0))); return InstructionARM64(instrs); } InstructionARM64 load128_simd128_gpr64_s8(Register simd_dest, Register gpr_addr, s64 offset) { ASSERT(gpr_addr.is_gpr(instr_set)); - ASSERT(simd_dest.is_128bit_simd( - instr_set)); // TODO ARM64 - this assertion isn't as useful for ARM - // since Q registers are not unique in terms of their id + ASSERT(simd_dest.is_128bit_simd(instr_set)); ASSERT(offset >= INT8_MIN && offset <= INT8_MAX); // first establish the base+index value in x16 std::vector instrs = { // https://www.scs.stanford.edu/~zyedidia/arm64/add_addsub_imm.html // ADD , , #{, } - InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), Rn(gpr_addr.id())), + InstructionARM64(Base(0b100100010, 9), Sh(0), Imm12(0), Rd(X16), + Rn(gpr_addr.hw_id(instr_set))), }; if (offset < 0) { // we'll subtract instead @@ -1720,11 +1770,11 @@ InstructionARM64 load128_simd128_gpr64_s8(Register simd_dest, Register gpr_addr, // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_imm_fpsimd.html // - LDR , [{, #}] instrs.emplace_back( - InstructionARM64(Base(0b0011110111, 10), Rn(X16), Rt(simd_dest.id()), Imm12(0))); + InstructionARM64(Base(0b0011110111, 10), Rn(X16), Rt(simd_dest.hw_id(instr_set)), Imm12(0))); return InstructionARM64(instrs); } -InstructionARM64 load128_xmm128_reg_offset(Register simd_dest, Register base, s64 offset) { +InstructionARM64 load128_simd128_reg_offset(Register simd_dest, Register base, s64 offset) { if (offset == 0) { return load128_simd128_gpr64(simd_dest, base); } else if (offset >= INT8_MIN && offset <= INT8_MAX) { @@ -1737,13 +1787,13 @@ InstructionARM64 load128_xmm128_reg_offset(Register simd_dest, Register base, s6 } } -InstructionARM64 store128_xmm128_reg_offset(Register base, Register xmm_val, s64 offset) { +InstructionARM64 store128_simd128_reg_offset(Register base, Register simd_val, s64 offset) { if (offset == 0) { - return store128_gpr64_simd128(base, xmm_val); + return store128_gpr64_simd128(base, simd_val); } else if (offset >= INT8_MIN && offset <= INT8_MAX) { - return store128_gpr64_simd128_s8(base, xmm_val, offset); + return store128_gpr64_simd128_s8(base, simd_val, offset); } else if (offset >= INT32_MIN && offset <= INT32_MAX) { - return store128_gpr64_simd128_s32(base, xmm_val, offset); + return store128_gpr64_simd128_s32(base, simd_val, offset); } else { ASSERT(false); return {0}; @@ -1767,6 +1817,7 @@ InstructionARM64 store128_xmm128_reg_offset(Register base, Register xmm_val, s64 // const int ARM64_LDR_MIN = -(1 << 18) * 4; // const int ARM64_LDR_MAX = ((1 << 18) - 1) * 4; +//! PC-relative static access does not work when EE mappings have different bases. InstructionARM64 load64_pcRel_s32(Register dest, s64 offset) { ASSERT(dest.is_gpr(instr_set)); ASSERT_MSG(offset != 0, @@ -1775,7 +1826,7 @@ InstructionARM64 load64_pcRel_s32(Register dest, s64 offset) { // "PC Relative offset is too large for ARM64, fix it."); // https://www.scs.stanford.edu/~zyedidia/arm64/ldr_lit_gen.html // LDR ,