From 8adac544cfc43abd59cdfa224cd8fc6398782ae5 Mon Sep 17 00:00:00 2001 From: ManDude <7569514+ManDude@users.noreply.github.com> Date: Fri, 25 Feb 2022 03:43:00 +0000 Subject: [PATCH] Allow symbol table expansion. (#1191) * Allow symbol table expansion. * fix debugger * fix bits_for_sym * use a `static_assert` over `throw` --- CMakeLists.txt | 1 + common/goal_constants.h | 30 +++++++++++++++++-- decompiler/IR2/AtomicOpTypeAnalysis.cpp | 3 +- decompiler/IR2/FormExpressionAnalysis.cpp | 3 +- decompiler/config/all-types.gc | 2 +- .../jak1_ntsc_black_label/type_casts.jsonc | 10 +++++-- .../jak1_ntsc_black_label/var_names.jsonc | 6 ++++ decompiler/util/goal_constants.h | 8 +++++ game/kernel/kscheme.cpp | 12 ++++---- game/kernel/kscheme.h | 1 + goal_src/engine/ambient/ambient.gc | 2 +- goal_src/engine/game/effect-control.gc | 9 ++---- goal_src/engine/game/generic-obs.gc | 20 ++++++------- goal_src/engine/gfx/merc/merc-death.gc | 2 +- goalc/debugger/Debugger.cpp | 11 +++---- .../reference/engine/ambient/ambient_REF.gc | 2 +- .../engine/game/effect-control_REF.gc | 9 ++---- .../reference/engine/game/generic-obs_REF.gc | 20 ++++++------- .../engine/gfx/merc/merc-death_REF.gc | 2 +- .../reference/engine/level/level_REF.gc | 2 +- 20 files changed, 97 insertions(+), 58 deletions(-) create mode 100644 decompiler/util/goal_constants.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 58c6ea5701..ce7fd8bd03 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ if(MSVC AND (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")) "-Xclang -fcxx-exceptions \ -Xclang -fexceptions \ -Xclang -std=c++17 \ + -Xclang -D_CRT_SECURE_NO_WARNINGS \ -mavx \ -Wno-c++11-narrowing -W3") diff --git a/common/goal_constants.h b/common/goal_constants.h index 10bdd7fdaa..a25888ca2d 100644 --- a/common/goal_constants.h +++ b/common/goal_constants.h @@ -9,8 +9,32 @@ constexpr int BASIC_OFFSET = 4; constexpr int STRUCTURE_ALIGNMENT = 16; constexpr int ARRAY_DATA_OFFSET = 12; // not including type tag -constexpr s32 GOAL_MAX_SYMBOLS = 0x2000; -constexpr s32 SYM_INFO_OFFSET = 0xff34; +/*! + * Here you can change the size of the symbol table! + * Make sure to also edit the constant in gcommon.gc + */ +constexpr s32 GOAL_MAX_SYMBOLS = 8192; // this MUST be a multiple of 2!! +constexpr s32 SYM_INFO_OFFSET = 8167 * 8 - 4; +// constexpr s32 GOAL_MAX_SYMBOLS = 16384; +// constexpr s32 SYM_INFO_OFFSET = GOAL_MAX_SYMBOLS * 8 - 4; + +constexpr s32 SYM_TABLE_MEM_SIZE = GOAL_MAX_SYMBOLS * 8 * 2; + +constexpr int bits_for_sym() { + int b = -1; + for (int i = 0; i < 32; ++i) { + if ((GOAL_MAX_SYMBOLS & (1 << i)) != 0) { + if (b != -1) { + // already got a set bit... not a multiple of 2! + // throw 0; + return -1; + } + b = i; + } + } + return b + 1; +} +static_assert(bits_for_sym() != 1, "symbol table invalid length"); enum class RegClass { GPR_64, FLOAT, INT_128, VECTOR_FLOAT, INVALID }; @@ -38,4 +62,4 @@ constexpr double DEGREES_PER_ROT = 65536.0; constexpr double DEGREES_LENGTH = DEGREES_PER_ROT / 360.0; constexpr u64 TICKS_PER_SECOND = 300.0; -constexpr float DEFAULT_RES_TIME = -1000000000.0; \ No newline at end of file +constexpr float DEFAULT_RES_TIME = -1000000000.0; diff --git a/decompiler/IR2/AtomicOpTypeAnalysis.cpp b/decompiler/IR2/AtomicOpTypeAnalysis.cpp index d08dcd6f01..6c294b10cc 100644 --- a/decompiler/IR2/AtomicOpTypeAnalysis.cpp +++ b/decompiler/IR2/AtomicOpTypeAnalysis.cpp @@ -7,6 +7,7 @@ #include "decompiler/IR2/bitfields.h" #include "common/type_system/state.h" #include "common/util/BitUtils.h" +#include "decompiler/util/goal_constants.h" namespace decompiler { @@ -640,7 +641,7 @@ TP_Type SimpleExpression::get_type_int2(const TypeState& input, if (tc(dts, TypeSpec("structure"), arg1_type) && !m_args[0].is_int() && is_int_or_uint(dts, arg0_type)) { if (arg1_type.typespec() == TypeSpec("symbol") && - arg0_type.is_integer_constant(SYM_INFO_OFFSET + POINTER_SIZE)) { + arg0_type.is_integer_constant(DECOMP_SYM_INFO_OFFSET + POINTER_SIZE)) { // symbol -> GOAL String // NOTE - the offset doesn't fit in a s16, so it's loaded into a register first. // so we expect the arg to be a variable, and the type propagation will figure out the diff --git a/decompiler/IR2/FormExpressionAnalysis.cpp b/decompiler/IR2/FormExpressionAnalysis.cpp index c1c83df59c..bd6414be69 100644 --- a/decompiler/IR2/FormExpressionAnalysis.cpp +++ b/decompiler/IR2/FormExpressionAnalysis.cpp @@ -10,6 +10,7 @@ #include "common/type_system/state.h" #include "common/util/print_float.h" #include "decompiler/IR2/ExpressionHelpers.h" +#include "decompiler/util/goal_constants.h" /* * TODO @@ -956,7 +957,7 @@ void SimpleExpressionElement::update_from_stack_add_i(const Env& env, // try to find symbol to string stuff auto arg0_int = get_goal_integer_constant(args.at(0), env); - if (arg0_int && (*arg0_int == SYM_INFO_OFFSET + 4) && + if (arg0_int && (*arg0_int == DECOMP_SYM_INFO_OFFSET + 4) && arg1_type.typespec() == TypeSpec("symbol")) { result->push_back(pool.alloc_element(args.at(1))); return; diff --git a/decompiler/config/all-types.gc b/decompiler/config/all-types.gc index a982351c28..aa27033101 100644 --- a/decompiler/config/all-types.gc +++ b/decompiler/config/all-types.gc @@ -18823,7 +18823,7 @@ (timer uint8 :offset-assert 6) (overlap uint8 :offset-assert 7) (effect uint32 :offset-assert 8) - (sound basic :offset-assert 12) + (sound symbol :offset-assert 12) ) :method-count-assert 9 :size-assert #x10 diff --git a/decompiler/config/jak1_ntsc_black_label/type_casts.jsonc b/decompiler/config/jak1_ntsc_black_label/type_casts.jsonc index aca78f6120..be7d916c6f 100644 --- a/decompiler/config/jak1_ntsc_black_label/type_casts.jsonc +++ b/decompiler/config/jak1_ntsc_black_label/type_casts.jsonc @@ -806,7 +806,8 @@ "update-sound-banks": [[[21, 52], "t0", "symbol"]], "(method 16 level-group)": [ [[122, 146], "s1", "continue-point"], - [[115, 154], "s3", "continue-point"] + [[115, 154], "s3", "continue-point"], + [444, "v1", "symbol"] ], "(method 20 level)": [[[43, 45], "s3", "ramdisk-rpc-fill"]], @@ -2269,7 +2270,10 @@ "(method 28 entity-ambient)": [[79, "v1", "int"]], - "(method 27 entity-ambient)": [[[15, 250], "s5", "symbol"]], + "(method 27 entity-ambient)": [ + [[15, 250], "s5", "symbol"], + [32, "v0", "symbol"] + ], "birth-func-vector-orient": [[[7, 24], "s3", "sprite-vec-data-2d"]], @@ -4763,7 +4767,7 @@ "(method 11 med-res-level)": [ ["_stack_", 16, "res-tag"], - [21, "v0", "(pointer (pointer sparticle-launch-group))"], + [21, "v0", "(pointer symbol)"], [57, "s4", "(pointer sparticle-launch-group)"] ], diff --git a/decompiler/config/jak1_ntsc_black_label/var_names.jsonc b/decompiler/config/jak1_ntsc_black_label/var_names.jsonc index 3ddc85c3b7..ceb0db4f9c 100644 --- a/decompiler/config/jak1_ntsc_black_label/var_names.jsonc +++ b/decompiler/config/jak1_ntsc_black_label/var_names.jsonc @@ -4022,5 +4022,11 @@ } }, + "(method 11 med-res-level)": { + "vars": { + "s4-0": ["s4-0", "(pointer sparticle-launch-group)"] + } + }, + "aaaaaaaaaaaaaaaaaaaaaaa": {} } diff --git a/decompiler/util/goal_constants.h b/decompiler/util/goal_constants.h new file mode 100644 index 0000000000..6d0a7cf484 --- /dev/null +++ b/decompiler/util/goal_constants.h @@ -0,0 +1,8 @@ +#pragma once + +#include "common/common_types.h" + +// Separate constants for the decompiler, so we can make changes to the game without breaking +// decompilation. + +constexpr s32 DECOMP_SYM_INFO_OFFSET = 8167 * 8 - 4; diff --git a/game/kernel/kscheme.cpp b/game/kernel/kscheme.cpp index a8b0b83198..5280e60ffd 100644 --- a/game/kernel/kscheme.cpp +++ b/game/kernel/kscheme.cpp @@ -255,7 +255,7 @@ void delete_structure(u32 s) { /*! * Allocate a basic of fixed size. */ -u64 new_basic(u32 heap, u32 type, u32 /*unused*/, u32 pp) { +u64 new_basic(u32 heap, u32 type, u32 /*size*/, u32 pp) { return alloc_heap_object(heap, type, Ptr(type)->allocated_size, pp); } @@ -708,8 +708,9 @@ Ptr find_symbol_from_c(const char* name) { } } - s32 sh1 = hash << 0x13; - s32 sh2 = sh1 >> 0x10; + auto bits = bits_for_sym() - 1; + s32 sh1 = hash << (0x20 - bits); + s32 sh2 = sh1 >> (0x20 - bits - 3); // will be signed, bottom 3 bits 0 (for alignment, symbol are every 8 bytes) // upper 16 bits are the same, so we will reach +/- 8 kb around 0. @@ -1716,14 +1717,15 @@ s32 InitHeapAndSymbol() { // reset all mips2c functions Mips2C::gLinkedFunctionTable = {}; // allocate memory for the symbol table - auto symbol_table = kmalloc(kglobalheap, 0x20000, KMALLOC_MEMSET, "symbol-table").cast(); + auto symbol_table = + kmalloc(kglobalheap, SYM_TABLE_MEM_SIZE, KMALLOC_MEMSET, "symbol-table").cast(); // pointer to the middle symbol is stored in the s7 register. s7 = symbol_table + (GOAL_MAX_SYMBOLS / 2) * 8 + BASIC_OFFSET; // pointer to the first symbol (SymbolTable2 is the "lower" symbol table) SymbolTable2 = symbol_table + BASIC_OFFSET; // the last symbol we will ever access. - LastSymbol = symbol_table + 0xff00; + LastSymbol = symbol_table + SYM_TABLE_END * 8; NumSymbols = 0; // inform compiler the symbol table is reset, and where it is. reset_output(); diff --git a/game/kernel/kscheme.h b/game/kernel/kscheme.h index c73ab8d8cc..921c4eaf15 100644 --- a/game/kernel/kscheme.h +++ b/game/kernel/kscheme.h @@ -20,6 +20,7 @@ extern Ptr LastSymbol; constexpr u32 EMPTY_HASH = 0x8454B6E6; constexpr u32 OFFSET_MASK = 7; constexpr u32 CRC_POLY = 0x04c11db7; +constexpr u32 SYM_TABLE_END = GOAL_MAX_SYMBOLS - 32; constexpr u32 DEFAULT_METHOD_COUNT = 12; constexpr u32 FALLBACK_UNKNOWN_METHOD_COUNT = 44; diff --git a/goal_src/engine/ambient/ambient.gc b/goal_src/engine/ambient/ambient.gc index 7fbba76bf1..1ebb0e8fe3 100644 --- a/goal_src/engine/ambient/ambient.gc +++ b/goal_src/engine/ambient/ambient.gc @@ -1120,7 +1120,7 @@ (add-debug-text-3d #t (bucket-id debug-draw1) - (the-as string (-> (the-as (pointer uint32) (+ #xff38 (res-lump-struct obj 'effect-name int :time 0.0))))) + (symbol->string (res-lump-struct obj 'effect-name symbol :time 0.0)) gp-0 (font-color white) (new 'static 'vector2h :y 24) diff --git a/goal_src/engine/game/effect-control.gc b/goal_src/engine/game/effect-control.gc index d6084151b0..535905d644 100644 --- a/goal_src/engine/game/effect-control.gc +++ b/goal_src/engine/game/effect-control.gc @@ -527,13 +527,8 @@ (s0-3 (-> (the-as death-info s3-0) sound)) ) (set! sv-288 (-> obj res)) - (let ((t1-11 - (string->sound-name - (the-as string (-> (the-as (pointer uint32) (+ #xff38 (the-as int (-> (the-as death-info s3-0) sound)))))) - ) - ) - ) - (s1-3 s2-5 (the-as symbol s0-3) arg1 s5-0 sv-288 t1-11) + (let ((t1-11 (string->sound-name (symbol->string (-> (the-as death-info s3-0) sound))))) + (s1-3 s2-5 s0-3 arg1 s5-0 sv-288 t1-11) ) ) ) diff --git a/goal_src/engine/game/generic-obs.gc b/goal_src/engine/game/generic-obs.gc index 5d5edb2d40..deb01ce6f0 100644 --- a/goal_src/engine/game/generic-obs.gc +++ b/goal_src/engine/game/generic-obs.gc @@ -1330,32 +1330,32 @@ "#f" (let ((s4-0 (the-as (pointer sparticle-launch-group) #f))) (set! sv-16 (new 'static 'res-tag)) - (let* ((s3-0 (res-lump-data arg0 'art-name (pointer (pointer sparticle-launch-group)) :tag-ptr (& sv-16))) + (let* ((s3-0 (res-lump-data arg0 'art-name (pointer symbol) :tag-ptr (& sv-16))) (s2-0 (-> s3-0 0)) ) (cond ((not s3-0) ) - ((part-group-pointer? s2-0) - (set! s4-0 (-> s3-0 0)) + ((part-group-pointer? (the-as pointer s2-0)) + (set! s4-0 (the-as (pointer sparticle-launch-group) (-> s3-0 0))) ) - ((= (-> s2-0 -1) string) + ((= (-> s2-0 type) string) (set! s4-0 (lookup-part-group-pointer-by-name (the-as string s2-0))) (if s4-0 - (set! (-> s3-0 0) s4-0) + (set! (-> s3-0 0) (the-as symbol s4-0)) ) ) - ((= (-> s2-0 -1) symbol) - (let ((a0-7 (-> (&+ s2-0 #xff38) 0))) - (set! s4-0 (lookup-part-group-pointer-by-name (the-as string a0-7))) + ((= (-> s2-0 type) symbol) + (let ((a0-7 (symbol->string s2-0))) + (set! s4-0 (lookup-part-group-pointer-by-name a0-7)) ) (if s4-0 - (set! (-> s3-0 0) s4-0) + (set! (-> s3-0 0) (the-as symbol s4-0)) ) ) ) ) - (when s4-0 + (when (the-as object s4-0) (let ((a0-8 (-> s4-0 0))) (if (and (nonzero? a0-8) (= (-> a0-8 type) sparticle-launch-group)) (set! (-> obj part) (create-launch-control a0-8 obj)) diff --git a/goal_src/engine/gfx/merc/merc-death.gc b/goal_src/engine/gfx/merc/merc-death.gc index 6c49689d21..696a4a95d5 100644 --- a/goal_src/engine/gfx/merc/merc-death.gc +++ b/goal_src/engine/gfx/merc/merc-death.gc @@ -14,7 +14,7 @@ (timer uint8 :offset-assert 6) (overlap uint8 :offset-assert 7) (effect uint32 :offset-assert 8) - (sound basic :offset-assert 12) + (sound symbol :offset-assert 12) ) :method-count-assert 9 :size-assert #x10 diff --git a/goalc/debugger/Debugger.cpp b/goalc/debugger/Debugger.cpp index 84fbf4e17c..b4194c4b6a 100644 --- a/goalc/debugger/Debugger.cpp +++ b/goalc/debugger/Debugger.cpp @@ -521,14 +521,15 @@ void Debugger::read_symbol_table() { u32 empty_pair_offset = (m_debug_context.s7 + FIX_SYM_EMPTY_PAIR - PAIR_OFFSET) - st_base; std::vector mem; - mem.resize(0x20000); + mem.resize(SYM_TABLE_MEM_SIZE); - if (!xdbg::read_goal_memory(mem.data(), 0x20000, st_base, m_debug_context, m_memory_handle)) { + if (!xdbg::read_goal_memory(mem.data(), SYM_TABLE_MEM_SIZE, st_base, m_debug_context, + m_memory_handle)) { fmt::print("Read failed during read_symbol_table\n"); return; } reads++; - bytes_read += 0x20000; + bytes_read += SYM_TABLE_MEM_SIZE; struct SymLower { u32 type; @@ -581,8 +582,8 @@ void Debugger::read_symbol_table() { // GOAL sym - s7 auto sym_offset = s32(offset + st_base + BASIC_OFFSET) - s32(m_debug_context.s7); - ASSERT(sym_offset >= INT16_MIN); - ASSERT(sym_offset <= INT16_MAX); + ASSERT(sym_offset >= -SYM_TABLE_MEM_SIZE / 4); + ASSERT(sym_offset < SYM_TABLE_MEM_SIZE / 4); std::string str(str_buff); if (str.length() >= 50) { diff --git a/test/decompiler/reference/engine/ambient/ambient_REF.gc b/test/decompiler/reference/engine/ambient/ambient_REF.gc index 43c84e3d78..5eca687627 100644 --- a/test/decompiler/reference/engine/ambient/ambient_REF.gc +++ b/test/decompiler/reference/engine/ambient/ambient_REF.gc @@ -1184,7 +1184,7 @@ (add-debug-text-3d #t (bucket-id debug-draw1) - (the-as string (-> (the-as (pointer uint32) (+ #xff38 (res-lump-struct obj 'effect-name int :time 0.0))))) + (symbol->string (res-lump-struct obj 'effect-name symbol :time 0.0)) gp-0 (font-color white) (new 'static 'vector2h :y 24) diff --git a/test/decompiler/reference/engine/game/effect-control_REF.gc b/test/decompiler/reference/engine/game/effect-control_REF.gc index fe4f41490e..faecadbfd4 100644 --- a/test/decompiler/reference/engine/game/effect-control_REF.gc +++ b/test/decompiler/reference/engine/game/effect-control_REF.gc @@ -537,13 +537,8 @@ (s0-3 (-> (the-as death-info s3-0) sound)) ) (set! sv-288 (-> obj res)) - (let ((t1-11 - (string->sound-name - (the-as string (-> (the-as (pointer uint32) (+ #xff38 (the-as int (-> (the-as death-info s3-0) sound)))))) - ) - ) - ) - (s1-3 s2-5 (the-as symbol s0-3) arg1 s5-0 sv-288 t1-11) + (let ((t1-11 (string->sound-name (symbol->string (-> (the-as death-info s3-0) sound))))) + (s1-3 s2-5 s0-3 arg1 s5-0 sv-288 t1-11) ) ) ) diff --git a/test/decompiler/reference/engine/game/generic-obs_REF.gc b/test/decompiler/reference/engine/game/generic-obs_REF.gc index c73a6de8f0..05ed572bc2 100644 --- a/test/decompiler/reference/engine/game/generic-obs_REF.gc +++ b/test/decompiler/reference/engine/game/generic-obs_REF.gc @@ -1426,32 +1426,32 @@ "#f" (let ((s4-0 (the-as (pointer sparticle-launch-group) #f))) (set! sv-16 (new 'static 'res-tag)) - (let* ((s3-0 (res-lump-data arg0 'art-name (pointer (pointer sparticle-launch-group)) :tag-ptr (& sv-16))) + (let* ((s3-0 (res-lump-data arg0 'art-name (pointer symbol) :tag-ptr (& sv-16))) (s2-0 (-> s3-0 0)) ) (cond ((not s3-0) ) - ((part-group-pointer? s2-0) - (set! s4-0 (-> s3-0 0)) + ((part-group-pointer? (the-as pointer s2-0)) + (set! s4-0 (the-as (pointer sparticle-launch-group) (-> s3-0 0))) ) - ((= (-> s2-0 -1) string) + ((= (-> s2-0 type) string) (set! s4-0 (lookup-part-group-pointer-by-name (the-as string s2-0))) (if s4-0 - (set! (-> s3-0 0) s4-0) + (set! (-> s3-0 0) (the-as symbol s4-0)) ) ) - ((= (-> s2-0 -1) symbol) - (let ((a0-7 (-> (&+ s2-0 #xff38) 0))) - (set! s4-0 (lookup-part-group-pointer-by-name (the-as string a0-7))) + ((= (-> s2-0 type) symbol) + (let ((a0-7 (symbol->string s2-0))) + (set! s4-0 (lookup-part-group-pointer-by-name a0-7)) ) (if s4-0 - (set! (-> s3-0 0) s4-0) + (set! (-> s3-0 0) (the-as symbol s4-0)) ) ) ) ) - (when s4-0 + (when (the-as object s4-0) (let ((a0-8 (-> s4-0 0))) (if (and (nonzero? a0-8) (= (-> a0-8 type) sparticle-launch-group)) (set! (-> obj part) (create-launch-control a0-8 obj)) diff --git a/test/decompiler/reference/engine/gfx/merc/merc-death_REF.gc b/test/decompiler/reference/engine/gfx/merc/merc-death_REF.gc index e340026f02..aef0b0be45 100644 --- a/test/decompiler/reference/engine/gfx/merc/merc-death_REF.gc +++ b/test/decompiler/reference/engine/gfx/merc/merc-death_REF.gc @@ -10,7 +10,7 @@ (timer uint8 :offset-assert 6) (overlap uint8 :offset-assert 7) (effect uint32 :offset-assert 8) - (sound basic :offset-assert 12) + (sound symbol :offset-assert 12) ) :method-count-assert 9 :size-assert #x10 diff --git a/test/decompiler/reference/engine/level/level_REF.gc b/test/decompiler/reference/engine/level/level_REF.gc index 981ca91d67..7a15ccf1c1 100644 --- a/test/decompiler/reference/engine/level/level_REF.gc +++ b/test/decompiler/reference/engine/level/level_REF.gc @@ -1683,7 +1683,7 @@ a1-49 a2-24 (if v1-142 - (-> (the-as (pointer uint32) (+ #xff38 (the-as int v1-142)))) + (symbol->string (the-as symbol v1-142)) ) (-> *game-info* current-continue name) )