diff --git a/decompiler/CMakeLists.txt b/decompiler/CMakeLists.txt index 9fde7dc95a..9f786e2402 100644 --- a/decompiler/CMakeLists.txt +++ b/decompiler/CMakeLists.txt @@ -2,7 +2,6 @@ add_library( decomp SHARED - analysis/anonymous_function_def.cpp analysis/atomic_op_builder.cpp analysis/cfg_builder.cpp analysis/expression_build.cpp @@ -11,6 +10,7 @@ add_library( analysis/insert_lets.cpp analysis/reg_usage.cpp analysis/stack_spill.cpp + analysis/static_refs.cpp analysis/symbol_def_map.cpp analysis/type_analysis.cpp analysis/variable_naming.cpp diff --git a/decompiler/Function/Function.h b/decompiler/Function/Function.h index 755292aada..72567c7811 100644 --- a/decompiler/Function/Function.h +++ b/decompiler/Function/Function.h @@ -1,8 +1,5 @@ #pragma once -#ifndef NEXT_FUNCTION_H -#define NEXT_FUNCTION_H - #include #include #include @@ -45,7 +42,7 @@ struct FunctionName { case FunctionKind::METHOD: return "(method " + std::to_string(method_id) + " " + type_name + ")"; case FunctionKind::TOP_LEVEL_INIT: - return "(top-level-login)"; + return "(top-level-login " + object_name + ")"; case FunctionKind::UNIDENTIFIED: return "(anon-function " + std::to_string(id_in_object) + " " + object_name + ")"; default: @@ -60,7 +57,10 @@ struct FunctionName { bool empty() const { return kind == FunctionKind::UNIDENTIFIED; } - void set_as_top_level() { kind = FunctionKind::TOP_LEVEL_INIT; } + void set_as_top_level(const std::string& object_file_name) { + kind = FunctionKind::TOP_LEVEL_INIT; + object_name = object_file_name; + } void set_as_global(std::string name) { kind = FunctionKind::GLOBAL; @@ -175,4 +175,3 @@ class Function { std::unordered_map basic_op_to_instruction; }; } // namespace decompiler -#endif // NEXT_FUNCTION_H diff --git a/decompiler/IR2/Form.cpp b/decompiler/IR2/Form.cpp index 0384d45f5a..c9153f6b23 100644 --- a/decompiler/IR2/Form.cpp +++ b/decompiler/IR2/Form.cpp @@ -6,6 +6,7 @@ #include "common/goos/PrettyPrinter.h" #include "common/type_system/TypeSystem.h" #include "decompiler/util/DecompilerTypeSystem.h" +#include "decompiler/util/data_decompile.h" namespace decompiler { @@ -2118,11 +2119,16 @@ void StoreArrayAccess::get_modified_regs(RegSet& regs) const { // DecompiledDataElement ///////////////////////////// -DecompiledDataElement::DecompiledDataElement(goos::Object description) - : m_description(std::move(description)) {} +DecompiledDataElement::DecompiledDataElement(const DecompilerLabel& label, + const std::optional& type_hint) + : m_label(label), m_type_hint(type_hint) {} goos::Object DecompiledDataElement::to_form_internal(const Env&) const { - return m_description; + if (m_decompiled) { + return m_description; + } else { + return pretty_print::to_symbol(fmt::format("", m_label.name)); + } } void DecompiledDataElement::apply(const std::function& f) { @@ -2135,6 +2141,18 @@ void DecompiledDataElement::collect_vars(RegAccessSet&, bool) const {} void DecompiledDataElement::get_modified_regs(RegSet&) const {} +void DecompiledDataElement::do_decomp(const Env& env, const LinkedObjectFile* file) { + if (m_type_hint) { + m_description = decompile_at_label_with_hint(*m_type_hint, m_label, env.file->labels, + env.file->words_by_seg, *env.dts, file); + + } else { + m_description = decompile_at_label_guess_type(m_label, env.file->labels, env.file->words_by_seg, + env.dts->ts, file); + } + m_decompiled = true; +} + ///////////////////////////// // LetElement ///////////////////////////// diff --git a/decompiler/IR2/Form.h b/decompiler/IR2/Form.h index c7960d10e5..8e449c3eb9 100644 --- a/decompiler/IR2/Form.h +++ b/decompiler/IR2/Form.h @@ -8,6 +8,7 @@ #include "decompiler/IR2/AtomicOp.h" #include "common/goos/Object.h" #include "common/type_system/TypeSystem.h" +#include "decompiler/Disasm/DecompilerLabel.h" namespace decompiler { class Form; @@ -1261,17 +1262,27 @@ class StoreArrayAccess : public FormElement { std::optional m_src_cast_type; }; +/*! + * This marks some static data that will be decompiled in a later pass. + * This is done at the very end so that we can make sure all static references to lambdas work. + */ class DecompiledDataElement : public FormElement { public: - DecompiledDataElement(goos::Object description); + // DecompiledDataElement(goos::Object description); + DecompiledDataElement(const DecompilerLabel& label, + const std::optional& type_hint = {}); goos::Object to_form_internal(const Env& env) const override; void apply(const std::function& f) override; void apply_form(const std::function& f) override; void collect_vars(RegAccessSet& vars, bool recursive) const override; void get_modified_regs(RegSet& regs) const override; + void do_decomp(const Env& env, const LinkedObjectFile* file); private: + bool m_decompiled = false; goos::Object m_description; + DecompilerLabel m_label; + std::optional m_type_hint; }; class LetElement : public FormElement { diff --git a/decompiler/IR2/FormExpressionAnalysis.cpp b/decompiler/IR2/FormExpressionAnalysis.cpp index 37bf16f136..7410fe599c 100644 --- a/decompiler/IR2/FormExpressionAnalysis.cpp +++ b/decompiler/IR2/FormExpressionAnalysis.cpp @@ -454,16 +454,13 @@ void SimpleExpressionElement::update_from_stack_identity(const Env& env, auto kv = env.label_types().find(lab.name); if (kv != env.label_types().end()) { auto type_name = kv->second.type_name; + // the actual decompilation is deferred until later, once static lambdas are done. if (type_name == "_auto_") { - auto decompiled_data = decompile_at_label_guess_type(lab, env.file->labels, - env.file->words_by_seg, env.dts->ts); - result->push_back(pool.alloc_element(decompiled_data)); + result->push_back(pool.alloc_element(lab)); } else if (type_name == "_lambda_") { result->push_back(this); } else { - auto decompiled_data = decompile_at_label_with_hint(kv->second, lab, env.file->labels, - env.file->words_by_seg, *env.dts); - result->push_back(pool.alloc_element(decompiled_data)); + result->push_back(pool.alloc_element(lab, kv->second)); } } else { result->push_back(this); diff --git a/decompiler/ObjectFile/LinkedObjectFile.cpp b/decompiler/ObjectFile/LinkedObjectFile.cpp index 649a712503..cb8e8f12b9 100644 --- a/decompiler/ObjectFile/LinkedObjectFile.cpp +++ b/decompiler/ObjectFile/LinkedObjectFile.cpp @@ -104,7 +104,11 @@ Function& LinkedObjectFile::get_function_at_label(int label_id) { * Get the function starting at this label, or nullptr if there is none. */ const Function* LinkedObjectFile::try_get_function_at_label(int label_id) const { - auto& label = labels.at(label_id); + const auto& label = labels.at(label_id); + return try_get_function_at_label(label); +} + +const Function* LinkedObjectFile::try_get_function_at_label(const DecompilerLabel& label) const { for (auto& func : functions_by_seg.at(label.target_segment)) { // + 4 to skip past type tag to the first word, which is were the label points. if (func.start_word * 4 + 4 == label.offset) { diff --git a/decompiler/ObjectFile/LinkedObjectFile.h b/decompiler/ObjectFile/LinkedObjectFile.h index b8469381aa..4ab4ebd050 100644 --- a/decompiler/ObjectFile/LinkedObjectFile.h +++ b/decompiler/ObjectFile/LinkedObjectFile.h @@ -40,6 +40,7 @@ class LinkedObjectFile { void symbol_link_offset(int source_segment, int source_offset, const char* name); Function& get_function_at_label(int label_id); const Function* try_get_function_at_label(int label_id) const; + const Function* try_get_function_at_label(const DecompilerLabel& label) const; std::string get_label_name(int label_id) const; uint32_t set_ordered_label_names(); void find_code(); diff --git a/decompiler/ObjectFile/ObjectFileDB.cpp b/decompiler/ObjectFile/ObjectFileDB.cpp index ce434f9508..c14d07ba3d 100644 --- a/decompiler/ObjectFile/ObjectFileDB.cpp +++ b/decompiler/ObjectFile/ObjectFileDB.cpp @@ -160,6 +160,8 @@ ObjectFileDB::ObjectFileDB(const std::vector& _dgos, "No object files have been added. Check that there are input files and the allowed_objects " "list."); } + + dts.bad_format_strings = config.bad_format_strings; } void ObjectFileDB::load_map_file(const std::string& map_data) { @@ -639,7 +641,7 @@ void ObjectFileDB::analyze_functions_ir1(const Config& config) { auto& func = data.linked_data.functions_by_seg.at(2).front(); assert(func.guessed_name.empty()); - func.guessed_name.set_as_top_level(); + func.guessed_name.set_as_top_level(data.to_unique_name()); func.find_global_function_defs(data.linked_data, dts); func.find_type_defs(data.linked_data, dts); func.find_method_defs(data.linked_data, dts); diff --git a/decompiler/ObjectFile/ObjectFileDB_IR2.cpp b/decompiler/ObjectFile/ObjectFileDB_IR2.cpp index 90fe38ef8c..49b7eb766a 100644 --- a/decompiler/ObjectFile/ObjectFileDB_IR2.cpp +++ b/decompiler/ObjectFile/ObjectFileDB_IR2.cpp @@ -18,7 +18,7 @@ #include "decompiler/analysis/expression_build.h" #include "decompiler/analysis/inline_asm_rewrite.h" #include "decompiler/analysis/stack_spill.h" -#include "decompiler/analysis/anonymous_function_def.h" +#include "decompiler/analysis/static_refs.h" #include "decompiler/analysis/symbol_def_map.h" #include "common/goos/PrettyPrinter.h" #include "decompiler/IR2/Form.h" @@ -96,7 +96,7 @@ void ObjectFileDB::ir2_top_level_pass(const Config& config) { auto& func = data.linked_data.functions_by_seg.at(2).front(); assert(func.guessed_name.empty()); - func.guessed_name.set_as_top_level(); + func.guessed_name.set_as_top_level(data.to_unique_name()); func.find_global_function_defs(data.linked_data, dts); func.find_type_defs(data.linked_data, dts); func.find_method_defs(data.linked_data, dts); @@ -602,7 +602,7 @@ void ObjectFileDB::ir2_insert_anonymous_functions() { (void)segment_id; (void)data; if (func.ir2.top_form && func.ir2.env.has_type_analysis()) { - total += insert_anonymous_functions(func.ir2.top_form, *func.ir2.form_pool, func, dts); + total += insert_static_refs(func.ir2.top_form, *func.ir2.form_pool, func, dts); } }); diff --git a/decompiler/analysis/anonymous_function_def.h b/decompiler/analysis/anonymous_function_def.h deleted file mode 100644 index 0ee1aed78a..0000000000 --- a/decompiler/analysis/anonymous_function_def.h +++ /dev/null @@ -1,10 +0,0 @@ -#pragma once - -#include "decompiler/IR2/Form.h" - -namespace decompiler { -int insert_anonymous_functions(Form* top_level_form, - FormPool& pool, - const Function& function, - const DecompilerTypeSystem& dts); -} diff --git a/decompiler/analysis/final_output.cpp b/decompiler/analysis/final_output.cpp index 15e28ab279..032a0068ec 100644 --- a/decompiler/analysis/final_output.cpp +++ b/decompiler/analysis/final_output.cpp @@ -9,6 +9,10 @@ namespace decompiler { goos::Object get_arg_list_for_function(const Function& func, const Env& env) { std::vector argument_elts; + if (func.type.arg_count() < 1) { + throw std::runtime_error( + fmt::format("Function {} has unknown type.\n", func.guessed_name.to_string())); + } assert(func.type.arg_count() >= 1); for (size_t i = 0; i < func.type.arg_count() - 1; i++) { auto reg = Register(Reg::GPR, Reg::A0 + i); diff --git a/decompiler/analysis/anonymous_function_def.cpp b/decompiler/analysis/static_refs.cpp similarity index 74% rename from decompiler/analysis/anonymous_function_def.cpp rename to decompiler/analysis/static_refs.cpp index 2489072a87..9b6625e8d9 100644 --- a/decompiler/analysis/anonymous_function_def.cpp +++ b/decompiler/analysis/static_refs.cpp @@ -1,14 +1,14 @@ -#include "anonymous_function_def.h" +#include "static_refs.h" #include "common/goos/PrettyPrinter.h" #include "decompiler/Function/Function.h" #include "decompiler/ObjectFile/LinkedObjectFile.h" #include "decompiler/analysis/final_output.h" namespace decompiler { -int insert_anonymous_functions(Form* top_level_form, - FormPool& pool, - const Function& function, - const DecompilerTypeSystem&) { +int insert_static_refs(Form* top_level_form, + FormPool& pool, + const Function& function, + const DecompilerTypeSystem&) { int replaced = 0; top_level_form->apply_form([&](Form* f) { auto atom = form_as_atom(f); @@ -35,6 +35,13 @@ int insert_anonymous_functions(Form* top_level_form, } } }); + + top_level_form->apply([&](FormElement* fe) { + auto as_static_data = dynamic_cast(fe); + if (as_static_data) { + as_static_data->do_decomp(function.ir2.env, function.ir2.env.file); + } + }); return replaced; } } // namespace decompiler diff --git a/decompiler/analysis/static_refs.h b/decompiler/analysis/static_refs.h new file mode 100644 index 0000000000..849888e054 --- /dev/null +++ b/decompiler/analysis/static_refs.h @@ -0,0 +1,10 @@ +#pragma once + +#include "decompiler/IR2/Form.h" + +namespace decompiler { +int insert_static_refs(Form* top_level_form, + FormPool& pool, + const Function& function, + const DecompilerTypeSystem& dts); +} diff --git a/decompiler/config.cpp b/decompiler/config.cpp index 71cd1d4dd5..57c31c2bba 100644 --- a/decompiler/config.cpp +++ b/decompiler/config.cpp @@ -170,6 +170,9 @@ Config read_config_file(const std::string& path_to_config_file) { config.hacks.cond_with_else_len_by_func_name[func_name].max_length_by_start_block[cond_name] = max_len; } + + config.bad_format_strings = + hacks_json.at("bad_format_strings").get>(); return config; } diff --git a/decompiler/config.h b/decompiler/config.h index 2603b78dcc..3a87be4328 100644 --- a/decompiler/config.h +++ b/decompiler/config.h @@ -100,6 +100,8 @@ struct Config { std::unordered_map> stack_structure_hints_by_function; + std::unordered_map bad_format_strings; + DecompileHacks hacks; }; diff --git a/decompiler/config/jak1_ntsc_black_label/hacks.jsonc b/decompiler/config/jak1_ntsc_black_label/hacks.jsonc index 07ba3a0d83..716522a929 100644 --- a/decompiler/config/jak1_ntsc_black_label/hacks.jsonc +++ b/decompiler/config/jak1_ntsc_black_label/hacks.jsonc @@ -448,6 +448,8 @@ "(method 12 perf-stat)" ], + // these functions use pairs and the decompiler + // will be less picky about types related to pairs. "pair_functions_by_name": [ "ref", "last", @@ -470,5 +472,15 @@ "update-sound-banks", "(method 16 level-group)", "bg" - ] + ], + + // If format is used with the wrong number of arguments, + // it will often mess up the decompilation, as the decompiler assumes + // that they used the correct number. This will override the decompiler's + // automatic detection. + "bad_format_strings": { + "ERROR: dma tag has data in reserved bits ~X~%":0, + "#: value of symbol ~A in task-controls is not a task-control~%":0 + } } diff --git a/decompiler/util/DecompilerTypeSystem.cpp b/decompiler/util/DecompilerTypeSystem.cpp index c7a55c29d8..1065977707 100644 --- a/decompiler/util/DecompilerTypeSystem.cpp +++ b/decompiler/util/DecompilerTypeSystem.cpp @@ -383,14 +383,11 @@ bool DecompilerTypeSystem::tp_lca(TypeState* combined, const TypeState& add) { } int DecompilerTypeSystem::get_format_arg_count(const std::string& str) const { - // temporary hack, remove this. - if (str == "ERROR: dma tag has data in reserved bits ~X~%") { - return 0; + auto bad_it = bad_format_strings.find(str); + if (bad_it != bad_format_strings.end()) { + return bad_it->second; } - if (str == "# symbol_add_order; std::unordered_map type_flags; std::unordered_map type_parents; + std::unordered_map bad_format_strings; void add_symbol(const std::string& name) { if (symbols.find(name) == symbols.end()) { @@ -54,5 +54,3 @@ class DecompilerTypeSystem { mutable goos::Reader m_reader; }; } // namespace decompiler - -#endif // JAK_DECOMPILERTYPESYSTEM_H diff --git a/decompiler/util/data_decompile.cpp b/decompiler/util/data_decompile.cpp index 011470d401..8ce0b3f9ae 100644 --- a/decompiler/util/data_decompile.cpp +++ b/decompiler/util/data_decompile.cpp @@ -6,6 +6,9 @@ #include "common/goos/PrettyPrinter.h" #include "common/util/math_util.h" #include "common/log/log.h" +#include "decompiler/ObjectFile/LinkedObjectFile.h" +#include "decompiler/IR2/Form.h" +#include "decompiler/analysis/final_output.h" namespace decompiler { @@ -16,11 +19,12 @@ goos::Object decompile_at_label_with_hint(const LabelType& hint, const DecompilerLabel& label, const std::vector& labels, const std::vector>& words, - DecompilerTypeSystem& dts) { + DecompilerTypeSystem& dts, + const LinkedObjectFile* file) { auto type = dts.parse_type_spec(hint.type_name); if (!hint.array_size.has_value()) { // if we don't have an array size, treat it as just a normal type. - return decompile_at_label(type, label, labels, words, dts.ts); + return decompile_at_label(type, label, labels, words, dts.ts, file); } if (type.base_type() == "pointer") { @@ -67,7 +71,7 @@ goos::Object decompile_at_label_with_hint(const LabelType& hint, fake_label.offset = label.offset + field_type_info->get_offset() + stride * elt; fake_label.name = fmt::format("fake-label-{}-elt-{}", type.get_single_arg().print(), elt); array_def.push_back( - decompile_at_label(type.get_single_arg(), fake_label, labels, words, dts.ts)); + decompile_at_label(type.get_single_arg(), fake_label, labels, words, dts.ts, file)); } return pretty_print::build_list(array_def); } @@ -121,12 +125,29 @@ std::optional get_type_of_label(const DecompilerLabel& label, goos::Object decompile_at_label_guess_type(const DecompilerLabel& label, const std::vector& labels, const std::vector>& words, - const TypeSystem& ts) { + const TypeSystem& ts, + const LinkedObjectFile* file) { auto guessed_type = get_type_of_label(label, words); if (!guessed_type.has_value()) { throw std::runtime_error("Could not guess the type of " + label.name); } - return decompile_at_label(*guessed_type, label, labels, words, ts); + return decompile_at_label(*guessed_type, label, labels, words, ts, file); +} + +goos::Object decompile_function_at_label(const DecompilerLabel& label, + const LinkedObjectFile* file) { + if (file) { + auto other_func = file->try_get_function_at_label(label); + if (other_func) { + std::vector inline_body; + other_func->ir2.top_form->inline_forms(inline_body, other_func->ir2.env); + auto result = pretty_print::build_list( + "lambda", get_arg_list_for_function(*other_func, other_func->ir2.env)); + pretty_print::append(result, pretty_print::build_list(inline_body)); + return result; + } + } + return pretty_print::to_symbol(fmt::format("", label.name)); } /*! @@ -137,21 +158,26 @@ goos::Object decompile_at_label(const TypeSpec& type, const DecompilerLabel& label, const std::vector& labels, const std::vector>& words, - const TypeSystem& ts) { + const TypeSystem& ts, + const LinkedObjectFile* file) { if (type == TypeSpec("string")) { return decompile_string_at_label(label, words); } + if (ts.tc(TypeSpec("function"), type)) { + return decompile_function_at_label(label, file); + } + if (ts.tc(TypeSpec("array"), type)) { - return decompile_boxed_array(label, labels, words, ts); + return decompile_boxed_array(label, labels, words, ts, file); } if (ts.tc(TypeSpec("structure"), type)) { - return decompile_structure(type, label, labels, words, ts); + return decompile_structure(type, label, labels, words, ts, file); } if (type == TypeSpec("pair")) { - return decompile_pair(label, labels, words, ts, true); + return decompile_pair(label, labels, words, ts, true, file); } throw std::runtime_error("Unimplemented decompile_at_label for " + type.print()); @@ -256,7 +282,8 @@ goos::Object decompile_structure(const TypeSpec& type, const DecompilerLabel& label, const std::vector& labels, const std::vector>& words, - const TypeSystem& ts) { + const TypeSystem& ts, + const LinkedObjectFile* file) { // first step, get type info and words TypeSpec actual_type = type; auto uncast_type_info = ts.lookup_type(actual_type); @@ -265,11 +292,39 @@ goos::Object decompile_structure(const TypeSpec& type, throw std::runtime_error(fmt::format("Type {} wasn't a structure type.", actual_type.print())); } bool is_basic = dynamic_cast(uncast_type_info); + auto offset_location = label.offset - type_info->get_offset(); + + if (is_basic) { + const auto& word = words.at(label.target_segment).at((offset_location / 4)); + if (word.kind != LinkedWord::TYPE_PTR) { + throw std::runtime_error("Basic does not start with type pointer"); + } + + if (word.symbol_name != actual_type.base_type()) { + // we can specify a more specific type. + auto got_type = TypeSpec(word.symbol_name); + if (ts.tc(actual_type, got_type)) { + actual_type = got_type; + + type_info = dynamic_cast(ts.lookup_type(actual_type)); + if (!type_info) { + throw std::runtime_error( + fmt::format("Type-tag type {} wasn't a structure type.", actual_type.print())); + } + + // try again with the right type. this resets back to decompile_at_label because we may + // want to get the specific function/string/etc implementations. + return decompile_at_label(actual_type, label, labels, words, ts, file); + } else { + throw std::runtime_error(fmt::format("Basic has the wrong type pointer, got {} expected {}", + word.symbol_name, actual_type.base_type())); + } + } + } int word_count = (type_info->get_size_in_memory() + 3) / 4; // check alignment - auto offset_location = label.offset - type_info->get_offset(); if (offset_location % 8) { throw std::runtime_error(fmt::format( "Tried to decompile a structure with type type {} (type offset {}) at label {}, but it has " @@ -333,18 +388,8 @@ goos::Object decompile_structure(const TypeSpec& type, } if (word.symbol_name != actual_type.base_type()) { - // we can specify a more specific type. - auto got_type = TypeSpec(word.symbol_name); - if (ts.tc(actual_type, got_type)) { - actual_type = got_type; - if (actual_type == TypeSpec("string")) { - return decompile_string_at_label(label, words); - } - } else { - throw std::runtime_error( - fmt::format("Basic has the wrong type pointer, got {} expected {}", word.symbol_name, - actual_type.base_type())); - } + // the check above should have caught this. + assert(false); } for (int k = 0; k < 4; k++) { field_status_per_byte.at(k) = HAS_DATA_READ; @@ -420,7 +465,7 @@ goos::Object decompile_structure(const TypeSpec& type, fake_label.offset = offset_location + field.offset() + field_type_info->get_offset(); fake_label.name = fmt::format("fake-label-{}-{}", actual_type.print(), field.name()); field_defs_out.emplace_back( - field.name(), decompile_at_label(field.type(), fake_label, labels, words, ts)); + field.name(), decompile_at_label(field.type(), fake_label, labels, words, ts, file)); } else if (!field.is_dynamic() && field.is_array() && field.is_inline()) { // it's an inline array. let's figure out the len and stride auto len = field.array_size(); @@ -440,7 +485,8 @@ goos::Object decompile_structure(const TypeSpec& type, offset_location + field.offset() + field_type_info->get_offset() + stride * elt; fake_label.name = fmt::format("fake-label-{}-{}-elt-{}", actual_type.print(), field.name(), elt); - array_def.push_back(decompile_at_label(field.type(), fake_label, labels, words, ts)); + array_def.push_back( + decompile_at_label(field.type(), fake_label, labels, words, ts, file)); } field_defs_out.emplace_back(field.name(), pretty_print::build_list(array_def)); } else if (!field.is_dynamic() && field.is_array() && !field.is_inline()) { @@ -467,8 +513,8 @@ goos::Object decompile_structure(const TypeSpec& type, auto& word = obj_words.at((field_start / 4) + elt); if (word.kind == LinkedWord::PTR) { - array_def.push_back( - decompile_at_label(field.type(), labels.at(word.label_id), labels, words, ts)); + array_def.push_back(decompile_at_label(field.type(), labels.at(word.label_id), labels, + words, ts, file)); } else if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) { // do nothing, the default is zero? array_def.push_back(pretty_print::to_symbol("0")); @@ -501,7 +547,7 @@ goos::Object decompile_structure(const TypeSpec& type, if (word.kind == LinkedWord::PTR) { field_defs_out.emplace_back( field.name(), - decompile_at_label(field.type(), labels.at(word.label_id), labels, words, ts)); + decompile_at_label(field.type(), labels.at(word.label_id), labels, words, ts, file)); } else if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) { // do nothing, the default is zero? field_defs_out.emplace_back(field.name(), pretty_print::to_symbol("0")); @@ -673,7 +719,8 @@ goos::Object decompile_value(const TypeSpec& type, goos::Object decompile_boxed_array(const DecompilerLabel& label, const std::vector& labels, const std::vector>& words, - const TypeSystem& ts) { + const TypeSystem& ts, + const LinkedObjectFile* file) { TypeSpec content_type; auto type_ptr_word_idx = (label.offset / 4) - 1; if ((label.offset % 8) == 4) { @@ -723,7 +770,7 @@ goos::Object decompile_boxed_array(const DecompilerLabel& label, result.push_back(pretty_print::to_symbol("0")); } else if (word.kind == LinkedWord::PTR) { result.push_back( - decompile_at_label(content_type, labels.at(word.label_id), labels, words, ts)); + decompile_at_label(content_type, labels.at(word.label_id), labels, words, ts, file)); } else if (word.kind == LinkedWord::SYM_PTR) { result.push_back(pretty_print::to_symbol(fmt::format("'{}", word.symbol_name))); } else { @@ -765,7 +812,8 @@ namespace { goos::Object decompile_pair_elt(const LinkedWord& word, const std::vector& labels, const std::vector>& words, - const TypeSystem& ts) { + const TypeSystem& ts, + const LinkedObjectFile* file) { if (word.kind == LinkedWord::PTR) { auto& label = labels.at(word.label_id); auto guessed_type = get_type_of_label(label, words); @@ -774,10 +822,10 @@ goos::Object decompile_pair_elt(const LinkedWord& word, } if (guessed_type == TypeSpec("pair")) { - return decompile_pair(label, labels, words, ts, false); + return decompile_pair(label, labels, words, ts, false, file); } - return decompile_at_label(*guessed_type, label, labels, words, ts); + return decompile_at_label(*guessed_type, label, labels, words, ts, file); } else if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) { // do nothing, the default is zero? return pretty_print::to_symbol("0"); @@ -798,7 +846,8 @@ goos::Object decompile_pair(const DecompilerLabel& label, const std::vector& labels, const std::vector>& words, const TypeSystem& ts, - bool add_quote) { + bool add_quote, + const LinkedObjectFile* file) { if ((label.offset % 8) != 2) { if ((label.offset % 4) != 0) { throw std::runtime_error(fmt::format("Invalid alignment for pair {}\n", label.offset % 16)); @@ -825,7 +874,7 @@ goos::Object decompile_pair(const DecompilerLabel& label, if ((to_print.offset % 8) == 2) { // continue auto car_word = words.at(to_print.target_segment).at((to_print.offset - 2) / 4); - list_tokens.push_back(decompile_pair_elt(car_word, labels, words, ts)); + list_tokens.push_back(decompile_pair_elt(car_word, labels, words, ts, file)); auto cdr_word = words.at(to_print.target_segment).at((to_print.offset + 2) / 4); // if empty @@ -847,7 +896,7 @@ goos::Object decompile_pair(const DecompilerLabel& label, "because we " "could not find a test case yet."); list_tokens.push_back(pretty_print::to_symbol(".")); - list_tokens.push_back(decompile_pair_elt(cdr_word, labels, words, ts)); + list_tokens.push_back(decompile_pair_elt(cdr_word, labels, words, ts, file)); if (add_quote) { return pretty_print::build_list("quote", pretty_print::build_list(list_tokens)); } else { @@ -870,7 +919,7 @@ goos::Object decompile_pair(const DecompilerLabel& label, "could not find a test case yet."); list_tokens.push_back(pretty_print::to_symbol(".")); list_tokens.push_back(decompile_pair_elt( - words.at(to_print.target_segment).at(to_print.offset / 4), labels, words, ts)); + words.at(to_print.target_segment).at(to_print.offset / 4), labels, words, ts, file)); if (add_quote) { return pretty_print::build_list("quote", pretty_print::build_list(list_tokens)); } else { diff --git a/decompiler/util/data_decompile.h b/decompiler/util/data_decompile.h index a6a3f3fbab..0cd6eea5a4 100644 --- a/decompiler/util/data_decompile.h +++ b/decompiler/util/data_decompile.h @@ -10,6 +10,7 @@ #include "decompiler/util/DecompilerTypeSystem.h" namespace decompiler { +class LinkedObjectFile; std::optional get_type_of_label(const DecompilerLabel& label, const std::vector>& words); @@ -19,30 +20,36 @@ goos::Object decompile_at_label(const TypeSpec& type, const DecompilerLabel& label, const std::vector& labels, const std::vector>& words, - const TypeSystem& ts); + const TypeSystem& ts, + const LinkedObjectFile* file); goos::Object decompile_at_label_with_hint(const LabelType& hint, const DecompilerLabel& label, const std::vector& labels, const std::vector>& words, - DecompilerTypeSystem& dts); + DecompilerTypeSystem& dts, + const LinkedObjectFile* file); goos::Object decompile_at_label_guess_type(const DecompilerLabel& label, const std::vector& labels, const std::vector>& words, - const TypeSystem& ts); + const TypeSystem& ts, + const LinkedObjectFile* file); goos::Object decompile_structure(const TypeSpec& actual_type, const DecompilerLabel& label, const std::vector& labels, const std::vector>& words, - const TypeSystem& ts); + const TypeSystem& ts, + const LinkedObjectFile* file); goos::Object decompile_pair(const DecompilerLabel& label, const std::vector& labels, const std::vector>& words, const TypeSystem& ts, - bool add_quote); + bool add_quote, + const LinkedObjectFile* file); goos::Object decompile_boxed_array(const DecompilerLabel& label, const std::vector& labels, const std::vector>& words, - const TypeSystem& ts); + const TypeSystem& ts, + const LinkedObjectFile* file); goos::Object decompile_value(const TypeSpec& type, const std::vector& bytes, const TypeSystem& ts); diff --git a/docs/markdown/progress-notes/changelog.md b/docs/markdown/progress-notes/changelog.md index d9b32a5a76..7d74e5c08f 100644 --- a/docs/markdown/progress-notes/changelog.md +++ b/docs/markdown/progress-notes/changelog.md @@ -170,4 +170,5 @@ - Added a `defun-recursive` to make it easier to define recursive functions - Forward declared basics can be used in more places - You can now set a field which has a forward declared structure or basic type -- `cdr` now returns an object of type `pair`. \ No newline at end of file +- `cdr` now returns an object of type `pair`. +- `lambda`s can now be used inside of a static object definition. \ No newline at end of file diff --git a/goalc/compiler/StaticObject.cpp b/goalc/compiler/StaticObject.cpp index 5ab2bc0396..6df8ef3d2f 100644 --- a/goalc/compiler/StaticObject.cpp +++ b/goalc/compiler/StaticObject.cpp @@ -1,6 +1,7 @@ #include "third-party/fmt/core.h" #include "StaticObject.h" #include "common/goal_constants.h" +#include "goalc/compiler/Env.h" namespace { template @@ -102,12 +103,17 @@ void StaticStructure::generate_structure(emitter::ObjectGenerator* gen) { } for (auto& ptr : pointers) { - gen->link_static_pointer(rec, ptr.offset_in_this, ptr.dest->rec, ptr.offset_in_dest); + gen->link_static_pointer_to_data(rec, ptr.offset_in_this, ptr.dest->rec, ptr.offset_in_dest); } for (auto& type : types) { gen->link_static_type_ptr(rec, type.offset, type.name); } + + for (auto& func : functions) { + gen->link_static_pointer_to_function(rec, func.offset_in_this, + gen->get_existing_function_record(func.func->idx_in_file)); + } } void StaticStructure::generate(emitter::ObjectGenerator* gen) { @@ -138,6 +144,13 @@ void StaticStructure::add_type_record(std::string name, int offset) { types.push_back(srec); } +void StaticStructure::add_function_record(const FunctionEnv* function, int offset) { + FunctionRecord rec; + rec.func = function; + rec.offset_in_this = offset; + functions.push_back(rec); +} + /////////////////// // StaticBasic /////////////////// @@ -224,4 +237,12 @@ StaticResult StaticResult::make_type_ref(const std::string& type_name, int metho result.m_method_count = method_count; result.m_ts = TypeSpec("type"); return result; +} + +StaticResult StaticResult::make_func_ref(const FunctionEnv* func, const TypeSpec& type) { + StaticResult result; + result.m_kind = Kind::FUNCTION_REFERENCE; + result.m_func = func; + result.m_ts = type; + return result; } \ No newline at end of file diff --git a/goalc/compiler/StaticObject.h b/goalc/compiler/StaticObject.h index c968e23acb..73281997fd 100644 --- a/goalc/compiler/StaticObject.h +++ b/goalc/compiler/StaticObject.h @@ -7,6 +7,8 @@ #include "goalc/compiler/ConstantValue.h" #include "common/util/BitUtils.h" +class FunctionEnv; + class StaticObject { public: virtual std::string print() const = 0; @@ -60,13 +62,20 @@ class StaticStructure : public StaticObject { int offset_in_dest = -1; }; + struct FunctionRecord { + const FunctionEnv* func = nullptr; + int offset_in_this = -1; + }; + std::vector types; std::vector symbols; std::vector pointers; + std::vector functions; void add_symbol_record(std::string name, int offset); void add_pointer_record(int offset_in_this, StaticStructure* dest, int offset_in_dest); void add_type_record(std::string name, int offset); + void add_function_record(const FunctionEnv* function, int offset); private: int m_offset = 0; @@ -87,6 +96,8 @@ class StaticString : public StaticBasic { void generate(emitter::ObjectGenerator* gen) override; }; +class FunctionEnv; + /*! * Represents a "static value". Like a reference to a static structure (including pair, string, * basic), a symbol, or some constant (bitfield, integer, float). Cannot be used to store a static @@ -102,12 +113,14 @@ class StaticResult { static StaticResult make_constant_data(u64 data, const TypeSpec& ts); static StaticResult make_symbol(const std::string& name); static StaticResult make_type_ref(const std::string& type_name, int method_count); + static StaticResult make_func_ref(const FunctionEnv* func, const TypeSpec& ts); const TypeSpec& typespec() const { return m_ts; } bool is_reference() const { return m_kind == Kind::STRUCTURE_REFERENCE; } bool is_constant_data() const { return m_kind == Kind::CONSTANT_DATA; } bool is_symbol() const { return m_kind == Kind::SYMBOL; } bool is_type() const { return m_kind == Kind::TYPE; } + bool is_func() const { return m_kind == Kind::FUNCTION_REFERENCE; } StaticStructure* reference() const { assert(is_reference()); @@ -125,6 +138,11 @@ class StaticResult { return m_symbol; } + const FunctionEnv* function() const { + assert(is_func()); + return m_func; + } + int method_count() const { assert(is_type()); return m_method_count; @@ -147,6 +165,9 @@ class StaticResult { // used for only STRUCTURE_REFERENCE StaticStructure* m_struct = nullptr; + // used for only FUNCTION_REFERENCE + const FunctionEnv* m_func = nullptr; + // used for only constant data std::optional m_constant_data; @@ -161,6 +182,7 @@ class StaticResult { CONSTANT_DATA, SYMBOL, TYPE, + FUNCTION_REFERENCE, INVALID } m_kind = Kind::INVALID; }; diff --git a/goalc/compiler/compilation/Static.cpp b/goalc/compiler/compilation/Static.cpp index a4a77c9162..62e4e17703 100644 --- a/goalc/compiler/compilation/Static.cpp +++ b/goalc/compiler/compilation/Static.cpp @@ -211,6 +211,8 @@ void Compiler::compile_static_structure_inline(const goos::Object& form, field_info.type.print()); } structure->add_type_record(sr.symbol_name(), field_offset); + } else if (sr.is_func()) { + structure->add_function_record(sr.function(), field_offset); } else { throw_compiler_error(form, "Unsupported field value {}.", field_value.print()); } @@ -706,6 +708,13 @@ StaticResult Compiler::compile_static(const goos::Object& form_before_macro, Env m_ts.forward_declare_type_method_count(type_name, method_count); return StaticResult::make_type_ref(type_name, method_count); + } else if (first.is_symbol("lambda")) { + auto lambda = dynamic_cast(compile_lambda(form, rest, env)); + assert(lambda); + if (!lambda->func) { + throw_compiler_error(form, "Static lambda cannot be inline."); + } + return StaticResult::make_func_ref(lambda->func, lambda->type()); } else { // maybe an enum s64 int_out; diff --git a/goalc/emitter/ObjectGenerator.cpp b/goalc/emitter/ObjectGenerator.cpp index 6480e061e5..52c4c68732 100644 --- a/goalc/emitter/ObjectGenerator.cpp +++ b/goalc/emitter/ObjectGenerator.cpp @@ -279,17 +279,32 @@ void ObjectGenerator::link_static_symbol_ptr(StaticRecord rec, * Insert a pointer to other static data. This patching will happen during runtime linking. * The source and destination must be in the same segment. */ -void ObjectGenerator::link_static_pointer(const StaticRecord& source, - int source_offset, - const StaticRecord& dest, - int dest_offset) { - StaticPointerLink link; +void ObjectGenerator::link_static_pointer_to_data(const StaticRecord& source, + int source_offset, + const StaticRecord& dest, + int dest_offset) { + StaticDataPointerLink link; link.source = source; link.dest = dest; link.offset_in_source = source_offset; link.offset_in_dest = dest_offset; assert(link.source.seg == link.dest.seg); - m_static_temp_ptr_links_by_seg.at(source.seg).push_back(link); + m_static_data_temp_ptr_links_by_seg.at(source.seg).push_back(link); +} + +/*! + * Insert a pointer to a function in static data. + * The patching will happen during runtime linking. + */ +void ObjectGenerator::link_static_pointer_to_function(const StaticRecord& source, + int source_offset, + const FunctionRecord& target_func) { + StaticFunctionPointerLink link; + link.source = source; + link.offset_in_source = source_offset; + link.dest = target_func; + assert(target_func.seg == source.seg); + m_static_function_temp_ptr_links_by_seg.at(source.seg).push_back(link); } void ObjectGenerator::link_instruction_static(const InstructionRecord& instr, @@ -341,7 +356,7 @@ void ObjectGenerator::handle_temp_static_sym_links(int seg) { * m_static_temp_ptr_links_by_seg -> m_pointer_links_by_seg */ void ObjectGenerator::handle_temp_static_ptr_links(int seg) { - for (const auto& link : m_static_temp_ptr_links_by_seg.at(seg)) { + for (const auto& link : m_static_data_temp_ptr_links_by_seg.at(seg)) { const auto& source_object = m_static_data_by_seg.at(seg).at(link.source.static_id); const auto& dest_object = m_static_data_by_seg.at(seg).at(link.dest.static_id); PointerLink result_link; @@ -350,6 +365,18 @@ void ObjectGenerator::handle_temp_static_ptr_links(int seg) { result_link.dest = dest_object.location + link.offset_in_dest; m_pointer_links_by_seg.at(seg).push_back(result_link); } + + for (const auto& link : m_static_function_temp_ptr_links_by_seg.at(seg)) { + const auto& source_object = m_static_data_by_seg.at(seg).at(link.source.static_id); + const auto& dest_function = m_function_data_by_seg.at(seg).at(link.dest.func_id); + assert(link.dest.seg == seg); + int loc = dest_function.instruction_to_byte_in_data.at(0); + PointerLink result_link; + result_link.segment = seg; + result_link.source = source_object.location + link.offset_in_source; + result_link.dest = loc; + m_pointer_links_by_seg.at(seg).push_back(result_link); + } } /*! diff --git a/goalc/emitter/ObjectGenerator.h b/goalc/emitter/ObjectGenerator.h index 2857447314..ccf351595b 100644 --- a/goalc/emitter/ObjectGenerator.h +++ b/goalc/emitter/ObjectGenerator.h @@ -78,10 +78,13 @@ class ObjectGenerator { void link_instruction_symbol_mem(const InstructionRecord& rec, const std::string& name); void link_instruction_symbol_ptr(const InstructionRecord& rec, const std::string& name); void link_static_symbol_ptr(StaticRecord rec, int offset, const std::string& name); - void link_static_pointer(const StaticRecord& source, - int source_offset, - const StaticRecord& dest, - int dest_offset); + void link_static_pointer_to_data(const StaticRecord& source, + int source_offset, + const StaticRecord& dest, + int dest_offset); + void link_static_pointer_to_function(const StaticRecord& source, + int source_offset, + const FunctionRecord& target_func); void link_instruction_static(const InstructionRecord& instr, const StaticRecord& target_static, int offset); @@ -145,13 +148,19 @@ class ObjectGenerator { int offset = -1; }; - struct StaticPointerLink { + struct StaticDataPointerLink { StaticRecord source; int offset_in_source = -1; StaticRecord dest; int offset_in_dest = -1; }; + struct StaticFunctionPointerLink { + StaticRecord source; + int offset_in_source = -1; + FunctionRecord dest; + }; + struct SymbolInstrLink { InstructionRecord rec; bool is_mem_access = false; @@ -205,7 +214,8 @@ class ObjectGenerator { seg_vector m_jump_temp_links_by_seg; seg_map m_symbol_instr_temp_links_by_seg; seg_map m_static_sym_temp_links_by_seg; - seg_vector m_static_temp_ptr_links_by_seg; + seg_vector m_static_data_temp_ptr_links_by_seg; + seg_vector m_static_function_temp_ptr_links_by_seg; seg_vector m_rip_func_temp_links_by_seg; seg_vector m_rip_data_temp_links_by_seg; diff --git a/test/decompiler/test_DataParser.cpp b/test/decompiler/test_DataParser.cpp index 77e2c7383a..d762de8b6f 100644 --- a/test/decompiler/test_DataParser.cpp +++ b/test/decompiler/test_DataParser.cpp @@ -124,8 +124,8 @@ TEST_F(DataDecompTest, String) { " .word 0x0\n" " .word 0x0\n"; auto parsed = parse_data(input); - auto decomp = - decompile_at_label_guess_type(parsed.label("L62"), parsed.labels, {parsed.words}, dts->ts); + auto decomp = decompile_at_label_guess_type(parsed.label("L62"), parsed.labels, {parsed.words}, + dts->ts, nullptr); EXPECT_EQ(decomp.print(), "\"finalboss-fight\""); } @@ -147,7 +147,7 @@ TEST_F(DataDecompTest, SimpleStructure) { " .word 0x6c63\n"; auto parsed = parse_data(input); auto decomp = decompile_at_label(TypeSpec("vif-disasm-element"), parsed.label("L217"), - parsed.labels, {parsed.words}, dts->ts); + parsed.labels, {parsed.words}, dts->ts, nullptr); check_forms_equal(decomp.print(), "(new 'static 'vif-disasm-element :mask #x7f :tag (vif-cmd-32 stcycl) :print " "#x2 :string1 \"stcycl\")"); @@ -207,8 +207,8 @@ TEST_F(DataDecompTest, VifDisasmArray) { " .word 0x706f6e\n" " .word 0x0"; auto parsed = parse_data(input); - auto decomp = - decompile_at_label_guess_type(parsed.label("L148"), parsed.labels, {parsed.words}, dts->ts); + auto decomp = decompile_at_label_guess_type(parsed.label("L148"), parsed.labels, {parsed.words}, + dts->ts, nullptr); check_forms_equal(decomp.print(), "(new 'static 'boxed-array :type\n" " vif-disasm-element\n" @@ -288,8 +288,8 @@ TEST_F(DataDecompTest, ContinuePoint) { " .word 0x74732d73\n" " .word 0x747261"; auto parsed = parse_data(input); - auto decomp = - decompile_at_label_guess_type(parsed.label("L63"), parsed.labels, {parsed.words}, dts->ts); + auto decomp = decompile_at_label_guess_type(parsed.label("L63"), parsed.labels, {parsed.words}, + dts->ts, nullptr); check_forms_equal(decomp.print(), "(new 'static 'continue-point\n" " :name \"finalboss-start\"\n" @@ -343,7 +343,7 @@ TEST_F(DataDecompTest, FloatArray) { " .word 0x3f800000\n\n"; auto parsed = parse_data(input); auto decomp = decompile_at_label_with_hint({"(pointer float)", true, 7}, parsed.label("L63"), - parsed.labels, {parsed.words}, *dts); + parsed.labels, {parsed.words}, *dts, nullptr); check_forms_equal(decomp.print(), "(new 'static 'array float 7\n" "1.0 0.0 1.0 0.0 1.0 0.0 1.0)"); @@ -380,8 +380,8 @@ TEST_F(DataDecompTest, KernelContext) { " .word 0x0\n" " .symbol #t\n"; auto parsed = parse_data(input); - auto decomp = - decompile_at_label_guess_type(parsed.label("L345"), parsed.labels, {parsed.words}, dts->ts); + auto decomp = decompile_at_label_guess_type(parsed.label("L345"), parsed.labels, {parsed.words}, + dts->ts, nullptr); check_forms_equal(decomp.print(), "(new 'static 'kernel-context\n" " :prevent-from-run (process-mask execute sleep)\n" diff --git a/test/goalc/source_templates/with_game/test-static-lambda.gc b/test/goalc/source_templates/with_game/test-static-lambda.gc new file mode 100644 index 0000000000..402bfff5b5 --- /dev/null +++ b/test/goalc/source_templates/with_game/test-static-lambda.gc @@ -0,0 +1,22 @@ +(deftype type-with-func (basic) + ((add-func (function int int int)) + (sub-func (function int int int)) + ) + ) + +(define *type-with-func* + (new 'static 'type-with-func + :add-func (lambda ((x int) (y int)) + (+ x y)) + :sub-func (lambda ((x int) (y int)) + (- x y)) + ) + ) + + +(defun this-is-a-function () + "blah blah") + +(format #t "Add: ~d sub: ~d~%" + ((-> *type-with-func* add-func) 10 20) + ((-> *type-with-func* sub-func) 10 20)) diff --git a/test/goalc/test_with_game.cpp b/test/goalc/test_with_game.cpp index 3db7e73a5d..8120550156 100644 --- a/test/goalc/test_with_game.cpp +++ b/test/goalc/test_with_game.cpp @@ -771,6 +771,10 @@ TEST_F(WithGameTests, SoundName) { "0\n"}); } +TEST_F(WithGameTests, StaticLambda) { + runner.run_static_test(env, testCategory, "test-static-lambda.gc", {"Add: 30 sub: -10\n0\n"}); +} + TEST(TypeConsistency, TypeConsistency) { Compiler compiler; compiler.enable_throw_on_redefines();