From 791c4abfc0ec91ce4ed9cb82c6d72ccd3f09f08e Mon Sep 17 00:00:00 2001 From: water111 <48171810+water111@users.noreply.github.com> Date: Thu, 25 Feb 2021 09:51:28 -0500 Subject: [PATCH] [Decompiler] Static Data Decomp (#280) * update all-types * begin work on static data decompiler * working for vif disasm array * mostly working * finish static data decompilation --- common/goos/PrettyPrinter.cpp | 18 + common/goos/PrettyPrinter.h | 2 + common/type_system/Type.cpp | 17 +- common/type_system/Type.h | 3 + common/type_system/TypeSystem.cpp | 6 +- common/type_system/TypeSystem.h | 6 +- common/type_system/deftype.cpp | 5 +- decompiler/CMakeLists.txt | 2 + decompiler/Disasm/DecompilerLabel.h | 2 +- decompiler/IR2/AtomicOpTypeAnalysis.cpp | 43 +- decompiler/IR2/Form.cpp | 29 +- decompiler/IR2/Form.h | 13 + decompiler/IR2/FormExpressionAnalysis.cpp | 25 +- decompiler/IR2/FormStack.cpp | 2 +- decompiler/ObjectFile/LinkedWord.h | 24 +- decompiler/ObjectFile/ObjectFileDB_IR2.cpp | 9 +- decompiler/analysis/expression_build.cpp | 1 + decompiler/analysis/final_output.cpp | 6 +- decompiler/config/all-types.gc | 68 +- .../jak1_ntsc_black_label/label_types.jsonc | 44 +- decompiler/util/DataParser.cpp | 232 +++++++ decompiler/util/DataParser.h | 23 + decompiler/util/data_decompile.cpp | 655 ++++++++++++++++++ decompiler/util/data_decompile.h | 41 ++ goal_src/engine/dma/dma-disasm.gc | 36 + goal_src/engine/gfx/texture-h.gc | 222 ++++++ goal_src/engine/level/level-h.gc | 375 ++++++++++ goal_src/engine/math/vector-h.gc | 2 +- goalc/compiler/compilation/Static.cpp | 2 +- test/CMakeLists.txt | 1 + test/decompiler/test_DataParser.cpp | 219 ++++++ 31 files changed, 2064 insertions(+), 69 deletions(-) create mode 100644 decompiler/util/DataParser.cpp create mode 100644 decompiler/util/DataParser.h create mode 100644 decompiler/util/data_decompile.cpp create mode 100644 decompiler/util/data_decompile.h create mode 100644 test/decompiler/test_DataParser.cpp diff --git a/common/goos/PrettyPrinter.cpp b/common/goos/PrettyPrinter.cpp index afcb45dc5b..1ed70093c0 100644 --- a/common/goos/PrettyPrinter.cpp +++ b/common/goos/PrettyPrinter.cpp @@ -7,11 +7,29 @@ #include #include #include +#include #include "PrettyPrinter.h" #include "Reader.h" +#include "third-party/fmt/core.h" namespace pretty_print { +/*! + * Print a float in a nice representation if possibly, or an exact 32-bit integer constant to + * be reinterpreted. + */ +goos::Object float_representation(float value) { + int rounded = value; + bool exact_int = ((float)rounded) == value; + if (value == 0.5 || value == -0.5 || value == 0.0 || value == 1.0 || value == -1.0 || exact_int) { + return goos::Object::make_float(value); + } else { + u32 int_value; + memcpy(&int_value, &value, 4); + return pretty_print::build_list("the-as", "float", fmt::format("#x{:x}", int_value)); + } +} + /*! * A single token which cannot be split between lines. */ diff --git a/common/goos/PrettyPrinter.h b/common/goos/PrettyPrinter.h index 893e034780..a59049bddc 100644 --- a/common/goos/PrettyPrinter.h +++ b/common/goos/PrettyPrinter.h @@ -51,4 +51,6 @@ goos::Object build_list(const goos::Object& car, Args... rest) { goos::Reader& get_pretty_printer_reader(); +goos::Object float_representation(float value); + } // namespace pretty_print diff --git a/common/type_system/Type.cpp b/common/type_system/Type.cpp index 414b3324f7..9b06e8bd79 100644 --- a/common/type_system/Type.cpp +++ b/common/type_system/Type.cpp @@ -51,8 +51,10 @@ Field::Field(std::string name, TypeSpec ts, int offset) */ std::string Field::print() const { return fmt::format( - "Field: ({} {} :offset {}) inline: {:5}, dynamic: {:5}, array: {:5}, array size {:3}", m_name, - m_type.print(), m_offset, m_inline, m_dynamic, m_array, m_array_size); + "Field: ({} {} :offset {}) inline: {:5}, dynamic: {:5}, array: {:5}, array size {:3}, align " + "{:2}, skip {}", + m_name, m_type.print(), m_offset, m_inline, m_dynamic, m_array, m_array_size, m_alignment, + m_skip_in_static_decomp); } /*! @@ -90,7 +92,8 @@ bool Field::operator==(const Field& other) const { m_dynamic == other.m_dynamic && m_array == other.m_array && m_array_size == other.m_array_size && - m_alignment == other.m_alignment; + m_alignment == other.m_alignment && + m_skip_in_static_decomp == other.m_skip_in_static_decomp; // clang-format on } @@ -480,8 +483,9 @@ StructureType::StructureType(std::string parent, std::string StructureType::print() const { std::string result = fmt::format( - "[StructureType] {}\n parent: {}\n boxed: {}\n dynamic: {}\n size: {}\n pack: {}\n fields:\n", - m_name, m_parent, m_is_boxed, m_dynamic, m_size_in_mem, m_pack); + "[StructureType] {}\n parent: {}\n boxed: {}\n dynamic: {}\n size: {}\n pack: {}\n misalign: " + "{}\n fields:\n", + m_name, m_parent, m_is_boxed, m_dynamic, m_size_in_mem, m_pack, m_allow_misalign); for (auto& x : m_fields) { result += " " + x.print() + "\n"; } @@ -507,7 +511,8 @@ bool StructureType::operator==(const Type& other) const { m_fields == p_other->m_fields && m_dynamic == p_other->m_dynamic && m_size_in_mem == p_other->m_size_in_mem && - m_pack == p_other->m_pack; + m_pack == p_other->m_pack && + m_allow_misalign == p_other->m_allow_misalign; // clang-format on } diff --git a/common/type_system/Type.h b/common/type_system/Type.h index 65439afc93..6b36595e61 100644 --- a/common/type_system/Type.h +++ b/common/type_system/Type.h @@ -192,6 +192,7 @@ class Field { bool is_dynamic() const { return m_dynamic; } const std::string& name() const { return m_name; } int offset() const { return m_offset; } + bool skip_in_decomp() const { return m_skip_in_static_decomp; } bool operator==(const Field& other) const; int alignment() const { @@ -208,6 +209,7 @@ class Field { friend class TypeSystem; void set_alignment(int alignment) { m_alignment = alignment; } void set_offset(int offset) { m_offset = offset; } + void set_skip_in_static_decomp() { m_skip_in_static_decomp = true; } std::string m_name; TypeSpec m_type; @@ -218,6 +220,7 @@ class Field { bool m_array = false; int m_array_size = 0; int m_alignment = -1; + bool m_skip_in_static_decomp = false; }; class StructureType : public ReferenceType { diff --git a/common/type_system/TypeSystem.cpp b/common/type_system/TypeSystem.cpp index 1e2b7411d2..e4b403fb38 100644 --- a/common/type_system/TypeSystem.cpp +++ b/common/type_system/TypeSystem.cpp @@ -616,7 +616,8 @@ int TypeSystem::add_field_to_type(StructureType* type, bool is_inline, bool is_dynamic, int array_size, - int offset_override) { + int offset_override, + bool skip_in_static_decomp) { if (type->lookup_field(field_name, nullptr)) { fmt::print("[TypeSystem] Type {} already has a field named {}\n", type->get_name(), field_name); throw std::runtime_error("add_field_to_type duplicate field names"); @@ -656,6 +657,9 @@ int TypeSystem::add_field_to_type(StructureType* type, field.set_offset(offset); field.set_alignment(field_alignment); + if (skip_in_static_decomp) { + field.set_skip_in_static_decomp(); + } int after_field = offset + get_size_in_type(field); if (type->get_size_in_memory() < after_field) { diff --git a/common/type_system/TypeSystem.h b/common/type_system/TypeSystem.h index ed4e4b960f..ea04e285b8 100644 --- a/common/type_system/TypeSystem.h +++ b/common/type_system/TypeSystem.h @@ -145,7 +145,8 @@ class TypeSystem { bool is_inline = false, bool is_dynamic = false, int array_size = -1, - int offset_override = -1); + int offset_override = -1, + bool skip_in_static_decomp = false); void add_builtin_types(); @@ -182,6 +183,8 @@ class TypeSystem { TypeSpec lowest_common_ancestor_reg(const TypeSpec& a, const TypeSpec& b) const; TypeSpec lowest_common_ancestor(const std::vector& types) const; + int get_size_in_type(const Field& field) const; + private: bool try_reverse_lookup(const FieldReverseLookupInput& input, std::vector* path, @@ -205,7 +208,6 @@ class TypeSystem { TypeSpec* result_type) const; std::string lca_base(const std::string& a, const std::string& b) const; bool typecheck_base_types(const std::string& expected, const std::string& actual) const; - int get_size_in_type(const Field& field) const; int get_alignment_in_type(const Field& field); Field lookup_field(const std::string& type_name, const std::string& field_name) const; StructureType* add_builtin_structure(const std::string& parent, diff --git a/common/type_system/deftype.cpp b/common/type_system/deftype.cpp index b1bc118cef..5e972d1b2c 100644 --- a/common/type_system/deftype.cpp +++ b/common/type_system/deftype.cpp @@ -98,6 +98,7 @@ void add_field(StructureType* structure, TypeSystem* ts, const goos::Object& def bool is_dynamic = false; int offset_override = -1; int offset_assert = -1; + bool skip_in_decomp = false; if (!rest->is_empty_list()) { if (car(rest).is_int()) { @@ -122,6 +123,8 @@ void add_field(StructureType* structure, TypeSystem* ts, const goos::Object& def throw std::runtime_error("Cannot use -1 as offset-assert"); } rest = cdr(rest); + } else if (opt_name == ":do-not-decompile") { + skip_in_decomp = true; } else { throw std::runtime_error("Invalid option in field specification: " + opt_name); } @@ -129,7 +132,7 @@ void add_field(StructureType* structure, TypeSystem* ts, const goos::Object& def } int actual_offset = ts->add_field_to_type(structure, name, type, is_inline, is_dynamic, - array_size, offset_override); + array_size, offset_override, skip_in_decomp); if (offset_assert != -1 && actual_offset != offset_assert) { throw std::runtime_error("Field " + name + " was placed at " + std::to_string(actual_offset) + " but offset-assert was set to " + std::to_string(offset_assert)); diff --git a/decompiler/CMakeLists.txt b/decompiler/CMakeLists.txt index 1f4f8b32c9..f1b1253a10 100644 --- a/decompiler/CMakeLists.txt +++ b/decompiler/CMakeLists.txt @@ -44,6 +44,8 @@ add_library( ObjectFile/ObjectFileDB.cpp ObjectFile/ObjectFileDB_IR2.cpp + util/data_decompile.cpp + util/DataParser.cpp util/DecompilerTypeSystem.cpp util/TP_Type.cpp diff --git a/decompiler/Disasm/DecompilerLabel.h b/decompiler/Disasm/DecompilerLabel.h index a6a150ccde..fc4409c94e 100644 --- a/decompiler/Disasm/DecompilerLabel.h +++ b/decompiler/Disasm/DecompilerLabel.h @@ -9,7 +9,7 @@ namespace decompiler { */ struct DecompilerLabel { std::string name; - int target_segment; + int target_segment = 0; int offset; // in bytes }; } // namespace decompiler \ No newline at end of file diff --git a/decompiler/IR2/AtomicOpTypeAnalysis.cpp b/decompiler/IR2/AtomicOpTypeAnalysis.cpp index ba418e5f25..441980f763 100644 --- a/decompiler/IR2/AtomicOpTypeAnalysis.cpp +++ b/decompiler/IR2/AtomicOpTypeAnalysis.cpp @@ -658,14 +658,6 @@ TP_Type LoadVarOp::get_src_type(const TypeState& input, rd_in.offset = ro.offset; auto rd = dts.ts.reverse_field_lookup(rd_in); - // only error on failure if "pair" is disabled. otherwise it might be a pair. - if (!rd.success && !dts.type_prop_settings.allow_pair) { - printf("input type is %s, offset is %d, sign %d size %d\n", rd_in.base_type.print().c_str(), - rd_in.offset, rd_in.deref.value().sign_extend, rd_in.deref.value().size); - throw std::runtime_error(fmt::format("Could not get type of load: {}. Reverse Deref Failed.", - to_form(env.file->labels, env).print())); - } - if (rd.success) { // load_path_set = true; // load_path_addr_of = rd.addr_of; @@ -676,6 +668,41 @@ TP_Type LoadVarOp::get_src_type(const TypeState& input, return TP_Type::make_from_ts(coerce_to_reg_type(rd.result_type)); } + if (input_type.typespec() == TypeSpec("pointer")) { + // we got a plain pointer. let's just assume we're loading an integer. + // perhaps we should disable this feature by default on 4-byte loads if we're getting + // lots of false positives for loading pointers from plain pointers. + + switch (m_kind) { + case Kind::UNSIGNED: + switch (m_size) { + case 1: + case 2: + case 4: + case 8: + return TP_Type::make_from_ts(TypeSpec("uint")); + default: + break; + } + break; + case Kind::SIGNED: + switch (m_size) { + case 1: + case 2: + case 4: + case 8: + return TP_Type::make_from_ts(TypeSpec("int")); + default: + break; + } + break; + case Kind::FLOAT: + return TP_Type::make_from_ts(TypeSpec("float")); + default: + assert(false); + } + } + // rd failed, try as pair. if (dts.type_prop_settings.allow_pair) { // we are strict here - only permit pair-type loads from object or pair. diff --git a/decompiler/IR2/Form.cpp b/decompiler/IR2/Form.cpp index a374945ae2..7dd3eaa26f 100644 --- a/decompiler/IR2/Form.cpp +++ b/decompiler/IR2/Form.cpp @@ -1664,17 +1664,7 @@ void ConstantFloatElement::collect_vars(VariableSet&) const {} void ConstantFloatElement::get_modified_regs(RegSet&) const {} goos::Object ConstantFloatElement::to_form_internal(const Env&) const { - // return goos::Object::make_float(m_value); - int rounded = m_value; - bool exact_int = ((float)rounded) == m_value; - if (m_value == 0.5 || m_value == -0.5 || m_value == 0.0 || m_value == 1.0 || m_value == -1.0 || - exact_int) { - return goos::Object::make_float(m_value); - } else { - u32 value; - memcpy(&value, &m_value, 4); - return pretty_print::build_list("the-as", "float", fmt::format("#x{:x}", value)); - } + return pretty_print::float_representation(m_value); } StorePlainDeref::StorePlainDeref(DerefElement* dst, @@ -1742,4 +1732,21 @@ void StoreArrayAccess::get_modified_regs(RegSet& regs) const { m_dst->get_modified_regs(regs); } +DecompiledDataElement::DecompiledDataElement(goos::Object description) + : m_description(std::move(description)) {} + +goos::Object DecompiledDataElement::to_form_internal(const Env&) const { + return m_description; +} + +void DecompiledDataElement::apply(const std::function& f) { + f(this); +} + +void DecompiledDataElement::apply_form(const std::function&) {} + +void DecompiledDataElement::collect_vars(VariableSet&) const {} + +void DecompiledDataElement::get_modified_regs(RegSet&) const {} + } // namespace decompiler diff --git a/decompiler/IR2/Form.h b/decompiler/IR2/Form.h index 6e15c7461e..dbcd5314f7 100644 --- a/decompiler/IR2/Form.h +++ b/decompiler/IR2/Form.h @@ -1107,6 +1107,19 @@ class StoreArrayAccess : public FormElement { Variable m_base_var; }; +class DecompiledDataElement : public FormElement { + public: + DecompiledDataElement(goos::Object description); + goos::Object to_form_internal(const Env& env) const override; + void apply(const std::function& f) override; + void apply_form(const std::function& f) override; + void collect_vars(VariableSet& vars) const override; + void get_modified_regs(RegSet& regs) const override; + + private: + goos::Object m_description; +}; + /*! * A Form is a wrapper around one or more FormElements. * This is done for two reasons: diff --git a/decompiler/IR2/FormExpressionAnalysis.cpp b/decompiler/IR2/FormExpressionAnalysis.cpp index f99b229a4f..1643fa0230 100644 --- a/decompiler/IR2/FormExpressionAnalysis.cpp +++ b/decompiler/IR2/FormExpressionAnalysis.cpp @@ -4,6 +4,7 @@ #include "common/goos/PrettyPrinter.h" #include "decompiler/util/DecompilerTypeSystem.h" #include "decompiler/ObjectFile/LinkedObjectFile.h" +#include "decompiler/util/data_decompile.h" /* * TODO @@ -334,7 +335,23 @@ void SimpleExpressionElement::update_from_stack_identity(const Env& env, auto str = env.file->get_goal_string(lab.target_segment, lab.offset / 4 - 1, false); result->push_back(pool.alloc_element(str)); } else { - result->push_back(this); + // look for a label hint: + auto kv = env.label_types().find(lab.name); + if (kv != env.label_types().end()) { + auto type_name = kv->second.type_name; + if (type_name == "_auto_") { + auto decompiled_data = decompile_at_label_guess_type(lab, env.file->labels, + env.file->words_by_seg, env.dts->ts); + result->push_back(pool.alloc_element(decompiled_data)); + } else { + auto type = env.dts->parse_type_spec(kv->second.type_name); + auto decompiled_data = + decompile_at_label(type, lab, env.file->labels, env.file->words_by_seg, env.dts->ts); + result->push_back(pool.alloc_element(decompiled_data)); + } + } else { + result->push_back(this); + } } } else if (arg.is_sym_ptr() || arg.is_sym_val() || arg.is_int() || arg.is_empty_list()) { @@ -1194,7 +1211,11 @@ void FunctionCallElement::update_from_stack(const Env& env, function_type = tp_type.typespec(); } - assert(is_method == m_op->is_method()); + // assert(is_method == m_op->is_method()); + if (is_method != m_op->is_method()) { + lg::error("Disagreement on method!"); + throw std::runtime_error("Disagreement on method"); + } // if method, don't pop the obj arg. // Variable method_obj_var; diff --git a/decompiler/IR2/FormStack.cpp b/decompiler/IR2/FormStack.cpp index 1ed498f052..37a93eca6a 100644 --- a/decompiler/IR2/FormStack.cpp +++ b/decompiler/IR2/FormStack.cpp @@ -340,7 +340,7 @@ void rewrite_to_get_var(std::vector& default_result, std::vector result; bool first = true; - while (keep_going) { + while (keep_going && !default_result.empty()) { keep_going = false; auto last_op_as_set = dynamic_cast(default_result.back()); if (last_op_as_set && last_op_as_set->dst().reg() == var_to_get.reg() && diff --git a/decompiler/ObjectFile/LinkedWord.h b/decompiler/ObjectFile/LinkedWord.h index d01d2b52ec..744fa3dd2f 100644 --- a/decompiler/ObjectFile/LinkedWord.h +++ b/decompiler/ObjectFile/LinkedWord.h @@ -5,11 +5,11 @@ * A word (4 bytes), possibly with some linking info. */ -#ifndef JAK2_DISASSEMBLER_LINKEDWORD_H -#define JAK2_DISASSEMBLER_LINKEDWORD_H - #include #include +#include + +#include "common/common_types.h" namespace decompiler { class LinkedWord { @@ -31,7 +31,21 @@ class LinkedWord { int label_id = -1; std::string symbol_name; + + u8 get_byte(int idx) const { + assert(kind == PLAIN_DATA); + switch (idx) { + case 0: + return data & 0xff; + case 1: + return (data >> 8) & 0xff; + case 2: + return (data >> 16) & 0xff; + case 3: + return (data >> 24) & 0xff; + default: + assert(false); + } + } }; } // namespace decompiler - -#endif // JAK2_DISASSEMBLER_LINKEDWORD_H diff --git a/decompiler/ObjectFile/ObjectFileDB_IR2.cpp b/decompiler/ObjectFile/ObjectFileDB_IR2.cpp index 1e5761becd..a3a36699e3 100644 --- a/decompiler/ObjectFile/ObjectFileDB_IR2.cpp +++ b/decompiler/ObjectFile/ObjectFileDB_IR2.cpp @@ -456,7 +456,14 @@ std::string ObjectFileDB::ir2_to_file(ObjectFileData& data) { // functions for (auto& func : data.linked_data.functions_by_seg.at(seg)) { - result += ir2_function_to_string(data, func, seg); + try { + result += ir2_function_to_string(data, func, seg); + } catch (std::exception& e) { + result += "Failed to write: "; + result += e.what(); + result += "\n"; + } + if (func.ir2.top_form && func.ir2.env.has_local_vars()) { result += '\n'; if (func.ir2.env.has_local_vars()) { diff --git a/decompiler/analysis/expression_build.cpp b/decompiler/analysis/expression_build.cpp index 4cbb19150e..3c3a9cf420 100644 --- a/decompiler/analysis/expression_build.cpp +++ b/decompiler/analysis/expression_build.cpp @@ -97,6 +97,7 @@ bool convert_to_expressions(Form* top_level_form, } catch (std::exception& e) { f.warnings.expression_build_warning("In {}: {}", f.guessed_name.to_string(), e.what()); + lg::warn("In {}: {}", f.guessed_name.to_string(), e.what()); return false; } diff --git a/decompiler/analysis/final_output.cpp b/decompiler/analysis/final_output.cpp index 424e061d1a..23504b4299 100644 --- a/decompiler/analysis/final_output.cpp +++ b/decompiler/analysis/final_output.cpp @@ -26,7 +26,11 @@ std::string final_defun_out(const Function& func, const DecompilerTypeSystem& dts, FunctionDefSpecials special_mode) { std::vector inline_body; - func.ir2.top_form->inline_forms(inline_body, env); + try { + func.ir2.top_form->inline_forms(inline_body, env); + } catch (std::exception& e) { + return e.what(); + } int var_count = 0; auto var_dec = env.local_var_type_list(func.ir2.top_form, func.type.arg_count() - 1, &var_count); diff --git a/decompiler/config/all-types.gc b/decompiler/config/all-types.gc index 3d4da9f98e..9ba74dc2a0 100644 --- a/decompiler/config/all-types.gc +++ b/decompiler/config/all-types.gc @@ -983,7 +983,7 @@ (deftype vector (structure) ( - (data float 4 :offset-assert 0) + (data float 4 :do-not-decompile :offset-assert 0) (x float :offset 0) (y float :offset 4) (z float :offset 8) @@ -3315,14 +3315,14 @@ ;; texture-h ;;;;;;;;;;;;;; -; ;; texture-h -; (deftype texture-id (uint32) -; () -; :method-count-assert 9 -; :size-assert #x4 -; :flag-assert #x900000004 -; ;; likely a bitfield type -; ) +;; texture-h +(deftype texture-id (uint32) + () + :method-count-assert 9 + :size-assert #x4 + :flag-assert #x900000004 + ;; likely a bitfield type + ) (deftype texture-pool-segment (structure) ((dest uint32 :offset-assert 0) @@ -3402,6 +3402,8 @@ :flag-assert #x90000000c ) +(define-extern texture-mip->segment (function int int int)) + ;; texture-h (deftype texture-page (basic) ((info basic :offset-assert 4) @@ -3490,6 +3492,9 @@ :size-assert #x10 :flag-assert #x900000010 ) + +(define-extern *texture-relocate-later* texture-relocate-later) + ;;;;;;;;;;;;;; ;; level-h ;;;;;;;;;;;;;; @@ -3517,30 +3522,30 @@ ;; level-h (deftype level-load-info (basic) - ((name-list basic 3 :offset-assert 4) + ((name-list basic 3 :offset-assert 4) (index int32 :offset-assert 16) (name basic :offset 4) (visname basic :offset 8) (nickname basic :offset 12) - (packages basic :offset-assert 20) - (sound-banks basic :offset-assert 24) + (packages pair :offset-assert 20) + (sound-banks pair :offset-assert 24) (music-bank basic :offset-assert 28) - (ambient-sounds basic :offset-assert 32) + (ambient-sounds pair :offset-assert 32) (mood basic :offset-assert 36) (mood-func basic :offset-assert 40) (ocean basic :offset-assert 44) (sky basic :offset-assert 48) (sun-fade float :offset-assert 52) - (continues basic :offset-assert 56) - (tasks basic :offset-assert 60) + (continues pair :offset-assert 56) + (tasks pair :offset-assert 60) (priority int32 :offset-assert 64) - (load-commands basic :offset-assert 68) - (alt-load-commands basic :offset-assert 72) + (load-commands pair :offset-assert 68) + (alt-load-commands pair :offset-assert 72) (bsp-mask uint64 :offset-assert 80) (bsphere sphere :offset-assert 88) (buzzer int32 :offset-assert 92) - (bottom-height float :offset-assert 96) ;; meters - (run-packages basic :offset-assert 100) + (bottom-height float :offset-assert 96) ;; meters + (run-packages pair :offset-assert 100) (prev-level basic :offset-assert 104) (next-level basic :offset-assert 108) (wait-for-load basic :offset-assert 112) @@ -3576,6 +3581,10 @@ (texture-page basic 9 :offset-assert 60) (loaded-texture-page basic 16 :offset-assert 96) (loaded-texture-page-count int32 :offset-assert 160) + ; (foreground-sink-group-0 dma-foreground-sink-group :inline :offset-assert 176) + ; (foreground-sink-group-1 dma-foreground-sink-group :inline :offset-assert 208) + ; (foreground-sink-group-2 dma-foreground-sink-group :inline :offset-assert 240) + ; (array-pad uint8 12) (foreground-sink-group dma-foreground-sink-group 3 :inline :offset-assert 176) ;; inline basic (foreground-draw-engine basic 3 :offset-assert 272) (entity basic :offset-assert 284) @@ -3639,18 +3648,25 @@ ;; level-h (deftype level-group (basic) ((length int32 :offset-assert 4) + (unknown-field-1 basic :offset-assert 8) + (unknown-field-2 basic :offset-assert 12) (entity-link entity-links :offset 16) ;; not sure what's going on here (border? basic :offset-assert 20) (vis? basic :offset-assert 24) (want-level basic :offset-assert 28) (receiving-level basic :offset-assert 32) - (load-commands basic :offset-assert 36) + (load-commands pair :offset-assert 36) (play? basic :offset-assert 40) - ; (level UNKNOWN 3 :offset-assert 100) - ; (data UNKNOWN 3 :offset-assert 100) - (level0 level :inline :offset 96) ;; inline basic + ;; there's something? from 40 -> 96. + (hack-pad uint8 :offset 90) + + ;(level level 3 :inline :offset-assert 96) + ;(data level 3 :inline :offset-assert 100) + (level0 level :inline :offset-assert 96) ;; inline basic (level1 level :inline :offset-assert 2704) ;; inline basic (level-default level :inline :offset-assert 5312) ;; inline basic + ;; this actually went earlier, + (level level 3 :inline :offset 96) (pad uint32) ) :method-count-assert 27 @@ -6984,7 +7000,7 @@ (quat vector :inline :offset-assert 32) (camera-trans vector :inline :offset-assert 48) (camera-rot float 9 :offset-assert 64) - (load-commands basic :offset-assert 100) + (load-commands pair :offset-assert 100) (vis-nick basic :offset-assert 104) (lev0 basic :offset-assert 108) (disp0 basic :offset-assert 112) @@ -32489,14 +32505,14 @@ (define-extern texture-page type) ;;(define-extern *depth-cue-base-page* object) ;; unknown type ;;(define-extern texture-pool-segment object) ;; unknown type -;;(define-extern *texture-relocate-later* object) ;; unknown type +;; ;; unknown type ;;(define-extern *sky-base-vram-word* object) ;; unknown type ;;(define-extern texture-id object) ;; unknown type ;;(define-extern *sky-base-page* object) ;; unknown type (define-extern texture-page-dir type) ;;(define-extern texture-relocate-later object) ;; unknown type ;;(define-extern texture-link object) ;; unknown type -(define-extern texture-mip->segment function) + ;;(define-extern shader-ptr object) ;; unknown type ;;(define-extern *eyes-base-vram-word* object) ;; unknown type ;;(define-extern *eyes-base-page* object) ;; unknown type diff --git a/decompiler/config/jak1_ntsc_black_label/label_types.jsonc b/decompiler/config/jak1_ntsc_black_label/label_types.jsonc index d1342a403c..69d9562769 100644 --- a/decompiler/config/jak1_ntsc_black_label/label_types.jsonc +++ b/decompiler/config/jak1_ntsc_black_label/label_types.jsonc @@ -39,5 +39,47 @@ "loader-h":[ ["L10", "float", true] - ] + ], + + "dma-disasm":[ + ["L148", "(array vif-disasm-element)", true] + ], + "level-h":[ + ["L3", "level-group", true] + ], + + "level-info":[ + ["L964", "level-load-info", true], + ["L867", "level-load-info", true], + ["L851", "level-load-info", true], + ["L822", "level-load-info", true], + ["L812", "level-load-info", true], + ["L531", "level-load-info", true], + ["L512", "level-load-info", true], + ["L495", "level-load-info", true], + ["L479", "level-load-info", true], + ["L271", "level-load-info", true], + ["L255", "level-load-info", true], + ["L237", "level-load-info", true], + ["L215", "level-load-info", true], + ["L175", "level-load-info", true], + ["L153", "level-load-info", true], + ["L143", "level-load-info", true], + ["L131", "level-load-info", true], + ["L112", "level-load-info", true], + ["L72", "level-load-info", true], + ["L52", "level-load-info", true], + ["L48", "level-load-info", true], + ["L41", "level-load-info", true], + ["L35", "level-load-info", true], + ["L30", "level-load-info", true], + ["L28", "level-load-info", true], + ["L544", "level-load-info", true], + ["L2", "pair", true] + + + ], + + "level-h":[ + ["L3", "_auto_", true]] } \ No newline at end of file diff --git a/decompiler/util/DataParser.cpp b/decompiler/util/DataParser.cpp new file mode 100644 index 0000000000..035d225551 --- /dev/null +++ b/decompiler/util/DataParser.cpp @@ -0,0 +1,232 @@ +#include +#include +#include "DataParser.h" +#include "third-party/fmt/core.h" + +/* + * Allowable lines: + * L123: - label + * L123: (offset 2) - label with an offset (only 2 allowed) + * .word 0xbeef - a hex word + * .word L123 - a label word + * .symbol sym - a symbol + * .empty-list - the empty list + * .type typ - a type + */ + +namespace decompiler { +namespace { +std::vector string_to_lines(const std::string& str) { + std::vector result; + std::string::size_type i; + std::string::size_type start = 0; + while (true) { + i = str.find('\n', start); + if (i == std::string::npos) { + if (start < str.length()) { + result.push_back(str.substr(start)); + } + return result; + } else { + result.push_back(str.substr(start, i - start)); + start = i + 1; + } + } +} + +std::string get_until_space(std::string& instr) { + assert(!instr.empty()); + size_t i; + for (i = 0; i < instr.length(); i++) { + if (instr[i] == ' ') { + break; + } + } + auto name = instr.substr(0, i); + if (i == instr.length()) { + instr.clear(); + } else { + instr = instr.substr(i + 1); + } + return name; +} + +} // namespace +ParsedData parse_data(const std::string& str) { + ParsedData result; + struct LabelInfo { + int idx = -1; + bool defined = false; + }; + + std::unordered_map label_map; + + const std::string offset_2 = ": (offset 2)"; + auto lines = string_to_lines(str); + int byte_offset = 0; + for (auto& line : lines) { + // strip off leading white space + size_t i; + for (i = 0; i < line.length(); i++) { + if (line[i] != ' ') { + line = line.substr(i); + break; + } + } + + if (line.empty()) { + continue; + } + + // try as label definition. + if (line.front() == 'L') { + int offset = 0; + if (line.back() == ':') { + line.pop_back(); + } else { + if (line.length() >= (2 + offset_2.length()) && + line.substr(line.length() - offset_2.length()) == offset_2) { + line = line.substr(0, line.length() - offset_2.length()); + offset = 2; + } else { + throw std::runtime_error(fmt::format("Invalid label line: {}", line)); + } + } + + auto& l = label_map[line]; + if (l.defined) { + throw std::runtime_error(fmt::format("Label {} is multiply defined.", line)); + } + + l.defined = true; + if (l.idx == -1) { + l.idx = result.labels.size(); + result.labels.emplace_back(); + } + + auto& label = result.labels.at(l.idx); + label.target_segment = 0; + label.offset = byte_offset + offset; + label.name = line; + continue; + } + + auto first_thing = get_until_space(line); + + // try as .type + if (first_thing == ".type") { + LinkedWord word(0); + word.kind = LinkedWord::TYPE_PTR; + word.symbol_name = line; + result.words.push_back(word); + byte_offset += 4; + continue; + } + + if (first_thing == ".symbol") { + LinkedWord word(0); + word.kind = LinkedWord::SYM_PTR; + word.symbol_name = line; + result.words.push_back(word); + byte_offset += 4; + continue; + } + + if (first_thing == ".empty-list") { + if (!line.empty()) { + throw std::runtime_error("Got something after .empty-list, this is not allowed"); + } + LinkedWord word(0); + word.kind = LinkedWord::EMPTY_PTR; + result.words.push_back(word); + byte_offset += 4; + continue; + } + + if (first_thing == ".word") { + if (!line.empty() && line.at(0) == 'L') { + auto& l = label_map[line]; + if (l.idx == -1) { + l.idx = result.labels.size(); + result.labels.emplace_back(); + } + LinkedWord word(0); + word.kind = LinkedWord::PTR; + word.label_id = l.idx; + result.words.push_back(word); + byte_offset += 4; + continue; + } else { + auto val = std::stoull(line, nullptr, 16); + assert(val <= UINT32_MAX); + LinkedWord word(val); + word.kind = LinkedWord::PLAIN_DATA; + result.words.push_back(word); + byte_offset += 4; + continue; + } + } + } + + for (auto& kv : label_map) { + if (!kv.second.defined) { + throw std::runtime_error(fmt::format("Label {} was used but not defined.", kv.first)); + } + } + + return result; +} + +std::string ParsedData::print() const { + std::string result; + std::unordered_map label_map; + for (const auto& x : labels) { + label_map[x.offset] = &x; + } + + for (size_t idx = 0; idx < words.size(); idx++) { + // print label + auto kv = label_map.find(idx * 4); + if (kv != label_map.end()) { + result += fmt::format("{}:\n", kv->second->name); + } + auto kv_offset = label_map.find(idx * 4 + 2); + if (kv_offset != label_map.end()) { + result += fmt::format("{}: (offset 2)\n", kv_offset->second->name); + } + + // print word + auto& word = words.at(idx); + switch (word.kind) { + case LinkedWord::PLAIN_DATA: + result += fmt::format(" .word 0x{:x}\n", word.data); + break; + case LinkedWord::PTR: + result += fmt::format(" .word {}\n", labels.at(word.label_id).name); + break; + case LinkedWord::SYM_PTR: + result += fmt::format(" .symbol {}\n", word.symbol_name); + break; + case LinkedWord::TYPE_PTR: + result += fmt::format(" .type {}\n", word.symbol_name); + break; + case LinkedWord::EMPTY_PTR: + result += " .empty-list\n"; + break; + default: + assert(false); + } + } + + return result; +} + +const DecompilerLabel& ParsedData::label(const std::string& name) const { + for (auto& x : labels) { + if (x.name == name) { + return x; + } + } + throw std::runtime_error("Could not find label " + name); +} +} // namespace decompiler diff --git a/decompiler/util/DataParser.h b/decompiler/util/DataParser.h new file mode 100644 index 0000000000..ae6d8e529b --- /dev/null +++ b/decompiler/util/DataParser.h @@ -0,0 +1,23 @@ +#pragma once + +/*! + * @file DataParser.h + * A parser for the decompiled GOAL data format. + */ + +#include +#include +#include "decompiler/ObjectFile/LinkedWord.h" +#include "decompiler/Disasm/DecompilerLabel.h" + +namespace decompiler { +struct ParsedData { + std::vector words; + std::vector labels; + std::string print() const; + const DecompilerLabel& label(const std::string& name) const; +}; + +ParsedData parse_data(const std::string& str); + +} // namespace decompiler \ No newline at end of file diff --git a/decompiler/util/data_decompile.cpp b/decompiler/util/data_decompile.cpp new file mode 100644 index 0000000000..216a1b3b6c --- /dev/null +++ b/decompiler/util/data_decompile.cpp @@ -0,0 +1,655 @@ +#include "data_decompile.h" +#include "third-party/fmt/core.h" +#include "common/goos/PrettyPrinter.h" +#include "common/util/math_util.h" +#include "common/log/log.h" + +namespace decompiler { + +/*! + * Attempt to determine the type of this label. This does not make sure that the type system + * actually knows about the type. If the thing is not a basic or pair, it will fail. + */ +std::optional get_type_of_label(const DecompilerLabel& label, + const std::vector>& words) { + if ((label.offset % 8) == 2) { + return TypeSpec("pair"); + } + + // try to guess the type by looking for a type pointer. + if (label.offset < 4) { + return {}; + } + + if ((label.offset % 8) == 4) { + auto type_ptr_word_idx = (label.offset / 4) - 1; + auto& type_ptr = words.at(label.target_segment).at(type_ptr_word_idx); + if (type_ptr.kind != LinkedWord::TYPE_PTR) { + return {}; + } + if (type_ptr.symbol_name == "array") { + auto content_type_ptr_word_idx = type_ptr_word_idx + 3; + auto& content_type_ptr = words.at(label.target_segment).at(content_type_ptr_word_idx); + if (content_type_ptr.kind != LinkedWord::TYPE_PTR) { + return {}; + } + return TypeSpec("array", {TypeSpec(content_type_ptr.symbol_name)}); + } + return TypeSpec(type_ptr.symbol_name); + } else { + return {}; + } +} + +/*! + * Attempt to decompile data at the given label, without knowing the type. This can only succeed + * if the object is a basic or pair, and is intended to save the user time in these cases, + * or even be run automatically. + */ +goos::Object decompile_at_label_guess_type(const DecompilerLabel& label, + const std::vector& labels, + const std::vector>& words, + const TypeSystem& ts) { + auto guessed_type = get_type_of_label(label, words); + if (!guessed_type.has_value()) { + throw std::runtime_error("Couldn't guess the type of " + label.name); + } + return decompile_at_label(*guessed_type, label, labels, words, ts); +} + +/*! + * Attempt to decompile data of the given type at the given label. If the decompiler thinks the + * types do not line up, it will fail. + */ +goos::Object decompile_at_label(const TypeSpec& type, + const DecompilerLabel& label, + const std::vector& labels, + const std::vector>& words, + const TypeSystem& ts) { + if (type == TypeSpec("string")) { + return decompile_string_at_label(label, words); + } + + if (ts.typecheck(TypeSpec("array"), type, "", false, false)) { + return decompile_boxed_array(label, labels, words, ts); + } + + if (ts.typecheck(TypeSpec("structure"), type, "", false, false)) { + return decompile_structure(type, label, labels, words, ts); + } + + if (type == TypeSpec("pair")) { + return decompile_pair(label, labels, words, ts); + } + + throw std::runtime_error("Unimplemented decompile_at_label for " + type.print()); +} + +/*! + * Special case to decompile a string into a string constant. + */ +goos::Object decompile_string_at_label(const DecompilerLabel& label, + const std::vector>& words) { + // first, check that it's actually a string. + if (label.offset % 4) { + throw std::runtime_error(fmt::format("Cannot get string at label {}, alignment of label is {}", + label.name, label.offset)); + } + assert(label.offset >= 4); + + const auto& type_ptr = words.at(label.target_segment).at((label.offset - 4) / 4); + if (type_ptr.kind != LinkedWord::TYPE_PTR) { + throw std::runtime_error(fmt::format( + "Cannot get string at label {}, word before is not a type pointer.", label.name)); + } + + if (type_ptr.symbol_name != "string") { + throw std::runtime_error(fmt::format("Cannot get string at label {}, type pointer is for a {}.", + label.name, type_ptr.symbol_name)); + } + + std::string result; + + auto word_idx = (label.offset / 4) - 1; + // next should be the size + if (word_idx + 1 >= int(words.at(label.target_segment).size())) { + throw std::runtime_error( + fmt::format("Cannot get string at label {}, not enough room", label.name)); + } + const LinkedWord& size_word = words.at(label.target_segment).at(word_idx + 1); + if (size_word.kind != LinkedWord::PLAIN_DATA) { + // sometimes an array of string pointer triggers this! + throw std::runtime_error( + fmt::format("Cannot get string at label {}, size is not plain data.", label.name)); + } + + // now characters... + for (size_t i = 0; i < size_word.data; i++) { + int word_offset = word_idx + 2 + (i / 4); + int byte_offset = i % 4; + auto& word = words.at(label.target_segment).at(word_offset); + if (word.kind != LinkedWord::PLAIN_DATA) { + throw std::runtime_error( + fmt::format("Cannot get string at label {}, character is not plain data.", label.name)); + } + char cword[4]; + memcpy(cword, &word.data, 4); + result += cword[byte_offset]; + assert(result.back() != 0); + } + return goos::StringObject::make_new(result); +} + +goos::Object decompile_structure(const TypeSpec& type, + const DecompilerLabel& label, + const std::vector& labels, + const std::vector>& words, + const TypeSystem& ts) { + // first step, get type info and words + TypeSpec actual_type = type; + auto uncast_type_info = ts.lookup_type(actual_type); + auto type_info = dynamic_cast(uncast_type_info); + if (!type_info) { + throw std::runtime_error(fmt::format("Type {} wasn't a structure type.", actual_type.print())); + } + bool is_basic = dynamic_cast(uncast_type_info); + + int word_count = (type_info->get_size_in_memory() + 3) / 4; + + // check alignment + auto offset_location = label.offset - type_info->get_offset(); + if ((offset_location % 8) == 2) { + // TEMP HACK + lg::error("Data decompile was looking for a structure, but it looks like a pair instead."); + return decompile_pair(label, labels, words, ts); + } + if (offset_location % 8) { + throw std::runtime_error( + fmt::format("Structure type {} (type offset {}) has alignment {}, which is not valid.", + type_info->get_name(), type_info->get_offset(), (offset_location % 8))); + } + + // check enough room + if (int(words.at(label.target_segment).size()) < word_count + offset_location / 4) { + throw std::runtime_error(fmt::format("Structure type {} takes up {} bytes and doesn't fit.", + type_info->get_name(), type_info->get_size_in_memory())); + } + + // get words for real + std::vector obj_words; + obj_words.insert(obj_words.begin(), + words.at(label.target_segment).begin() + (offset_location / 4), + words.at(label.target_segment).begin() + (offset_location / 4) + word_count); + + // status of each byte. + enum ByteStatus : u8 { ZERO_UNREAD, HAS_DATA_UNREAD, ZERO_READ, HAS_DATA_READ }; + std::vector field_status_per_byte; + for (int i = 0; i < word_count; i++) { + auto& w = obj_words.at(i); + switch (w.kind) { + case LinkedWord::TYPE_PTR: + case LinkedWord::PTR: + case LinkedWord::SYM_PTR: + case LinkedWord::EMPTY_PTR: + field_status_per_byte.push_back(HAS_DATA_UNREAD); + field_status_per_byte.push_back(HAS_DATA_UNREAD); + field_status_per_byte.push_back(HAS_DATA_UNREAD); + field_status_per_byte.push_back(HAS_DATA_UNREAD); + break; + case LinkedWord::PLAIN_DATA: { + u8 bytes[4]; + memcpy(bytes, &w.data, 4); + for (auto b : bytes) { + field_status_per_byte.push_back(b ? HAS_DATA_UNREAD : ZERO_UNREAD); + } + } break; + default: + throw std::runtime_error("Unsupported word in static data"); + } + } + + std::vector> field_defs_out; + // now iterate over fields: + int idx = 0; + for (auto& field : type_info->fields()) { + if (field.skip_in_decomp()) { + idx++; + continue; + } + if (is_basic && idx == 0) { + assert(field.name() == "type" && field.offset() == 0); + auto& word = obj_words.at(0); + if (word.kind != LinkedWord::TYPE_PTR) { + throw std::runtime_error("Basic doesn't start with type pointer"); + } + + if (word.symbol_name != actual_type.base_type()) { + // we can specify a more specific type. + auto got_type = TypeSpec(word.symbol_name); + if (ts.typecheck(actual_type, got_type, "", false, false)) { + lg::info("For type {}, got more specific type {}\n", actual_type.print(), + got_type.print()); + actual_type = got_type; + if (actual_type == TypeSpec("string")) { + return decompile_string_at_label(label, words); + } + } else { + throw std::runtime_error( + fmt::format("Basic has the wrong type pointer, got {} expected {}", word.symbol_name, + actual_type.base_type())); + } + } + for (int k = 0; k < 4; k++) { + field_status_per_byte.at(k) = HAS_DATA_READ; + } + idx++; + continue; + } + idx++; + // first, let's see if this overlaps with anything: + auto field_start = field.offset(); + auto field_end = field_start + ts.get_size_in_type(field); + bool all_zero = true; + bool any_overlap = false; + for (int i = field_start; i < field_end; i++) { + auto status = field_status_per_byte.at(i); + if (status != ZERO_UNREAD && status != ZERO_READ) { + all_zero = false; + } + if (status == HAS_DATA_READ || status == ZERO_READ) { + any_overlap = true; + } + } + + if (all_zero) { + // field has nothing in it, just skip it. + continue; + } + + if (any_overlap) { + // for now, let's just skip fields that overlapped with the previous. + // eventually we should do something smarter here... + continue; + } + + // OK - READ THE FIELD: + for (int i = field_start; i < field_end; i++) { + // even if our field was partially zero, we mark those zero bytes as "has data". + field_status_per_byte.at(i) = HAS_DATA_READ; + } + + // first, let's see if it's a value or reference + auto field_type_info = ts.lookup_type(field.type()); + if (!field_type_info->is_reference()) { + // value type. need to get bytes. + assert(!field.is_inline()); + if (field.is_array()) { + // array of values. + auto len = field.array_size(); + auto stride = ts.get_size_in_type(field) / len; + assert(stride == field_type_info->get_size_in_memory()); + + std::vector array_def = {pretty_print::to_symbol( + fmt::format("new 'static 'array '{} {}", field.type().print(), field.array_size()))}; + + for (int i = 0; i < len; i++) { + auto start = field_start + stride * i; + auto end = start + field_type_info->get_size_in_memory(); + std::vector elt_bytes; + for (int j = start; j < end; j++) { + auto& word = obj_words.at(j / 4); + if (word.kind != LinkedWord::PLAIN_DATA) { + throw std::runtime_error("Got bad word in kind in array of values"); + } + elt_bytes.push_back(word.get_byte(j % 4)); + } + array_def.push_back(decompile_value(field.type(), elt_bytes, ts)); + } + + field_defs_out.emplace_back(field.name(), pretty_print::build_list(array_def)); + } else if (field.is_dynamic()) { + throw std::runtime_error( + fmt::format("Dynamic value field {} in static data type {} not yet implemented", + field.name(), actual_type.print())); + } else { + std::vector bytes_out; + for (int byte_idx = field_start; byte_idx < field_end; byte_idx++) { + bytes_out.push_back(obj_words.at(byte_idx / 4).get_byte(byte_idx % 4)); + } + field_defs_out.emplace_back(field.name(), decompile_value(field.type(), bytes_out, ts)); + } + + } else { + if (!field.is_dynamic() && !field.is_array() && field.is_inline()) { + // inline structure! + DecompilerLabel fake_label; + fake_label.target_segment = label.target_segment; + // offset from real start of outer + field offset + tag, we want to fake that. + fake_label.offset = offset_location + field.offset() + field_type_info->get_offset(); + fake_label.name = fmt::format("fake-label-{}-{}", actual_type.print(), field.name()); + field_defs_out.emplace_back( + field.name(), decompile_at_label(field.type(), fake_label, labels, words, ts)); + } else if (!field.is_dynamic() && field.is_array() && field.is_inline()) { + // it's an inline array. let's figure out the len and stride + auto len = field.array_size(); + auto total_size = ts.get_size_in_type(field); + auto stride = total_size / len; + assert(stride * len == total_size); + assert(stride == align(field_type_info->get_size_in_memory(), + field_type_info->get_inline_array_stride_alignment())); + + std::vector array_def = {pretty_print::to_symbol(fmt::format( + "new 'static 'inline-array '{} {}", field.type().print(), field.array_size()))}; + for (int elt = 0; elt < len; elt++) { + DecompilerLabel fake_label; + fake_label.target_segment = label.target_segment; + // offset from real start of outer + field offset + tag, we want to fake that. + fake_label.offset = + offset_location + field.offset() + field_type_info->get_offset() + stride * elt; + fake_label.name = + fmt::format("fake-label-{}-{}-elt-{}", actual_type.print(), field.name(), elt); + array_def.push_back(decompile_at_label(field.type(), fake_label, labels, words, ts)); + } + field_defs_out.emplace_back(field.name(), pretty_print::build_list(array_def)); + } else if (!field.is_dynamic() && field.is_array() && !field.is_inline()) { + auto len = field.array_size(); + auto total_size = ts.get_size_in_type(field); + auto stride = total_size / len; + assert(stride * len == total_size); + assert(stride == 4); + + std::vector array_def = {pretty_print::to_symbol( + fmt::format("new 'static 'array '{} {}", field.type().print(), field.array_size()))}; + for (int elt = 0; elt < len; elt++) { + auto& word = obj_words.at((field_start / 4) + elt); + + if (word.kind == LinkedWord::PTR) { + array_def.push_back( + decompile_at_label(field.type(), labels.at(word.label_id), labels, words, ts)); + } else if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) { + // do nothing, the default is zero? + array_def.push_back(pretty_print::to_symbol("0")); + } else if (word.kind == LinkedWord::SYM_PTR) { + if (word.symbol_name == "#f" || word.symbol_name == "#t") { + array_def.push_back(pretty_print::to_symbol(fmt::format("{}", word.symbol_name))); + } else { + array_def.push_back(pretty_print::to_symbol(fmt::format("'{}", word.symbol_name))); + } + } else if (word.kind == LinkedWord::EMPTY_PTR) { + array_def.push_back(pretty_print::to_symbol("'()")); + } else { + throw std::runtime_error( + fmt::format("Field {} in type {} offset {} did not have a proper reference for " + "array element {}", + field.name(), actual_type.print(), field.offset(), elt)); + } + } + field_defs_out.emplace_back(field.name(), pretty_print::build_list(array_def)); + + } else if (field.is_dynamic() || field.is_array() || field.is_inline()) { + throw std::runtime_error(fmt::format( + "Dynamic/array/inline reference field {} type {} in static data not yet implemented", + field.name(), actual_type.print())); + } else { + // then we expect a label. + assert(field_end - field_start == 4); + auto& word = obj_words.at(field_start / 4); + + if (word.kind == LinkedWord::PTR) { + field_defs_out.emplace_back( + field.name(), + decompile_at_label(field.type(), labels.at(word.label_id), labels, words, ts)); + } else if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) { + // do nothing, the default is zero? + field_defs_out.emplace_back(field.name(), pretty_print::to_symbol("0")); + } else if (word.kind == LinkedWord::SYM_PTR) { + if (word.symbol_name == "#f" || word.symbol_name == "#t") { + field_defs_out.emplace_back( + field.name(), pretty_print::to_symbol(fmt::format("{}", word.symbol_name))); + } else { + field_defs_out.emplace_back( + field.name(), pretty_print::to_symbol(fmt::format("'{}", word.symbol_name))); + } + } else if (word.kind == LinkedWord::EMPTY_PTR) { + field_defs_out.emplace_back(field.name(), pretty_print::to_symbol("'()")); + } else { + throw std::runtime_error( + fmt::format("Field {} in type {} offset {} did not have a proper reference", + field.name(), actual_type.print(), field.offset())); + } + } + } + } + + for (size_t i = 0; i < field_status_per_byte.size(); i++) { + if (field_status_per_byte.at(i) == HAS_DATA_UNREAD) { + throw std::runtime_error( + fmt::format("In structure of type {} at label {} offset {}, there was unknown data.", + actual_type.print(), label.name, i)); + } + } + + std::vector result_def = { + pretty_print::to_symbol("new"), pretty_print::to_symbol("'static"), + pretty_print::to_symbol(fmt::format("'{}", actual_type.print()))}; + for (auto& f : field_defs_out) { + result_def.push_back(pretty_print::to_symbol(fmt::format(":{}", f.first))); + result_def.push_back(f.second); + } + return pretty_print::build_list(result_def); +} + +goos::Object decompile_value(const TypeSpec& type, + const std::vector& bytes, + const TypeSystem& ts) { + // try as common integer types: + if (ts.typecheck(TypeSpec("uint32"), type, "", false, false)) { + assert(bytes.size() == 4); + u32 value; + memcpy(&value, bytes.data(), 4); + return pretty_print::to_symbol(fmt::format("#x{:x}", u64(value))); + } else if (ts.typecheck(TypeSpec("int32"), type, "", false, false)) { + assert(bytes.size() == 4); + s32 value; + memcpy(&value, bytes.data(), 4); + if (value > 100 && value <= INT32_MAX) { + return pretty_print::to_symbol(fmt::format("#x{:x}", value)); + } else { + return pretty_print::to_symbol(fmt::format("{}", value)); + } + } else if (ts.typecheck(TypeSpec("int8"), type, "", false, false)) { + assert(bytes.size() == 1); + s8 value; + memcpy(&value, bytes.data(), 1); + if (value > 5) { + return pretty_print::to_symbol(fmt::format("#x{:x}", value)); + } else { + return pretty_print::to_symbol(fmt::format("{}", value)); + } + } else if (ts.typecheck(TypeSpec("uint64"), type, "", false, false)) { + assert(bytes.size() == 8); + u64 value; + memcpy(&value, bytes.data(), 8); + return pretty_print::to_symbol(fmt::format("#x{:x}", value)); + } else if (ts.typecheck(TypeSpec("float"), type, "", false, false)) { + assert(bytes.size() == 4); + float value; + memcpy(&value, bytes.data(), 4); + return pretty_print::float_representation(value); + } + + else { + throw std::runtime_error(fmt::format("decompile_value failed on a {}", type.print())); + } +} + +goos::Object decompile_boxed_array(const DecompilerLabel& label, + const std::vector& labels, + const std::vector>& words, + const TypeSystem& ts) { + TypeSpec content_type; + auto type_ptr_word_idx = (label.offset / 4) - 1; + if ((label.offset % 8) == 4) { + auto& type_ptr = words.at(label.target_segment).at(type_ptr_word_idx); + if (type_ptr.kind != LinkedWord::TYPE_PTR) { + throw std::runtime_error("Invalid basic in decompile_boxed_array"); + } + if (type_ptr.symbol_name == "array") { + auto content_type_ptr_word_idx = type_ptr_word_idx + 3; + auto& content_type_ptr = words.at(label.target_segment).at(content_type_ptr_word_idx); + if (content_type_ptr.kind != LinkedWord::TYPE_PTR) { + throw std::runtime_error("Invalid content in decompile_boxed_array"); + } + content_type = TypeSpec(content_type_ptr.symbol_name); + } else { + throw std::runtime_error("Wrong basic type in decompile_boxed_array"); + } + } else { + throw std::runtime_error("Invalid alignment in decompile_boxed_array"); + } + + // now get the size + auto& size_word_1 = words.at(label.target_segment).at(type_ptr_word_idx + 1); + auto& size_word_2 = words.at(label.target_segment).at(type_ptr_word_idx + 2); + auto first_elt_word_idx = type_ptr_word_idx + 4; + + if (size_word_1.kind != LinkedWord::PLAIN_DATA || size_word_2.kind != LinkedWord::PLAIN_DATA) { + throw std::runtime_error("Invalid size in decompile_boxed_array"); + } + + if (size_word_1.data != size_word_2.data) { + throw std::runtime_error("Inconsistent size in decompile_boxed_array"); + } + + int array_length = size_word_1.data; + + auto content_type_info = ts.lookup_type(content_type); + if (content_type_info->is_reference()) { + // easy, stride of 4. + std::vector result = { + pretty_print::to_symbol("new"), pretty_print::to_symbol("'static"), + pretty_print::to_symbol("'boxed-array"), pretty_print::to_symbol(content_type.print()), + pretty_print::to_symbol(fmt::format("{}", array_length))}; + + for (int elt = 0; elt < array_length; elt++) { + auto& word = words.at(label.target_segment).at(first_elt_word_idx + elt); + if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) { + result.push_back(pretty_print::to_symbol("0")); + } else if (word.kind == LinkedWord::PTR) { + result.push_back( + decompile_at_label(content_type, labels.at(word.label_id), labels, words, ts)); + } else { + throw std::runtime_error( + fmt::format("Unknown content type in boxed array of references, word idx {}", + first_elt_word_idx + elt)); + } + } + + return pretty_print::build_list(result); + } else { + // value type. + throw std::runtime_error("boxed value type array decompile not yet implemented."); + } +} + +namespace { +goos::Object decompile_pair_elt(const LinkedWord& word, + const std::vector& labels, + const std::vector>& words, + const TypeSystem& ts) { + if (word.kind == LinkedWord::PTR) { + return decompile_at_label_guess_type(labels.at(word.label_id), labels, words, ts); + } else if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) { + // do nothing, the default is zero? + return pretty_print::to_symbol("0"); + } else if (word.kind == LinkedWord::SYM_PTR) { + if (word.symbol_name == "#f" || word.symbol_name == "#t") { + return pretty_print::to_symbol(fmt::format("{}", word.symbol_name)); + } else { + return pretty_print::to_symbol(fmt::format("'{}", word.symbol_name)); + } + } else if (word.kind == LinkedWord::EMPTY_PTR) { + return pretty_print::to_symbol("'()"); + } else if (word.kind == LinkedWord::PLAIN_DATA && (word.data & 0b111) == 0) { + return pretty_print::to_symbol(fmt::format("(the binteger {})", word.data >> 3)); + } else { + throw std::runtime_error(fmt::format("Pair elt did not have a good word kind")); + } +} +} // namespace + +goos::Object decompile_pair(const DecompilerLabel& label, + const std::vector& labels, + const std::vector>& words, + const TypeSystem& ts) { + if ((label.offset % 8) != 2) { + if ((label.offset % 4) != 0) { + throw std::runtime_error(fmt::format("Invalid alignment for pair {}\n", label.offset % 16)); + } else { + auto& word = words.at(label.target_segment).at(label.offset / 4); + if (word.kind != LinkedWord::EMPTY_PTR) { + throw std::runtime_error( + fmt::format("Based on alignment, expected to get empty list for pair, but didn't")); + } + return pretty_print::to_symbol("'()"); + } + } + + std::vector list_tokens; + auto to_print = label; + + for (int iter = 0;; iter++) { + if (iter > 10000) { + throw std::runtime_error( + "Exceeded 10,000 look ups while trying to follow a linked list. Giving up, the list is " + "possibly circular. Increase the limit in data_decompile.cpp if you really need more."); + } + + if ((to_print.offset % 8) == 2) { + // continue + auto car_word = words.at(to_print.target_segment).at((to_print.offset - 2) / 4); + list_tokens.push_back(decompile_pair_elt(car_word, labels, words, ts)); + + auto cdr_word = words.at(to_print.target_segment).at((to_print.offset + 2) / 4); + // if empty + if (cdr_word.kind == LinkedWord::EMPTY_PTR) { + return pretty_print::build_list(list_tokens); + } + // if pointer + if (cdr_word.kind == LinkedWord::PTR) { + to_print = labels.at(cdr_word.label_id); + continue; + } + // invalid. + lg::error( + "There is an improper list. This is probably okay, but should be checked manually " + "because we " + "could not find a test case yet."); + list_tokens.push_back(pretty_print::to_symbol(".")); + list_tokens.push_back(decompile_pair_elt(cdr_word, labels, words, ts)); + return pretty_print::build_list(list_tokens); + } else { + if ((to_print.offset % 4) != 0) { + throw std::runtime_error( + fmt::format("Invalid alignment for pair {}\n", to_print.offset % 16)); + } else { + auto& word = words.at(to_print.target_segment).at(to_print.offset / 4); + if (word.kind != LinkedWord::EMPTY_PTR) { + throw std::runtime_error( + fmt::format("Based on alignment, expected to get empty list for pair, but didn't")); + } + // improper list + lg::error( + "There is an improper list. This is probably okay, but should be checked manually " + "because we " + "could not find a test case yet."); + list_tokens.push_back(pretty_print::to_symbol(".")); + list_tokens.push_back(decompile_pair_elt( + words.at(to_print.target_segment).at(to_print.offset / 4), labels, words, ts)); + return pretty_print::build_list(list_tokens); + } + } + } +} + +} // namespace decompiler \ No newline at end of file diff --git a/decompiler/util/data_decompile.h b/decompiler/util/data_decompile.h new file mode 100644 index 0000000000..c9c4a35a75 --- /dev/null +++ b/decompiler/util/data_decompile.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include "common/goos/Object.h" +#include "decompiler/Disasm/DecompilerLabel.h" +#include "decompiler/ObjectFile/LinkedWord.h" +#include "common/type_system/TypeSpec.h" +#include "common/type_system/TypeSystem.h" + +namespace decompiler { +std::optional get_type_of_label(const DecompilerLabel& label, + const std::vector>& words); + +goos::Object decompile_string_at_label(const DecompilerLabel& label, + const std::vector>& words); +goos::Object decompile_at_label(const TypeSpec& type, + const DecompilerLabel& label, + const std::vector& labels, + const std::vector>& words, + const TypeSystem& ts); +goos::Object decompile_at_label_guess_type(const DecompilerLabel& label, + const std::vector& labels, + const std::vector>& words, + const TypeSystem& ts); +goos::Object decompile_structure(const TypeSpec& actual_type, + const DecompilerLabel& label, + const std::vector& labels, + const std::vector>& words, + const TypeSystem& ts); +goos::Object decompile_pair(const DecompilerLabel& label, + const std::vector& labels, + const std::vector>& words, + const TypeSystem& ts); +goos::Object decompile_boxed_array(const DecompilerLabel& label, + const std::vector& labels, + const std::vector>& words, + const TypeSystem& ts); +goos::Object decompile_value(const TypeSpec& type, + const std::vector& bytes, + const TypeSystem& ts); +} // namespace decompiler diff --git a/goal_src/engine/dma/dma-disasm.gc b/goal_src/engine/dma/dma-disasm.gc index 3d3f473364..b10a280e4d 100644 --- a/goal_src/engine/dma/dma-disasm.gc +++ b/goal_src/engine/dma/dma-disasm.gc @@ -18,6 +18,42 @@ :flag-assert #x900000018 ) +(define *vif-disasm-table* + (new 'static 'boxed-array vif-disasm-element 34 + (new 'static 'vif-disasm-element :mask #x7f :string1 "nop") + (new 'static 'vif-disasm-element :mask #x7f :tag #x1 :print #x2 :string1 "stcycl") + (new 'static 'vif-disasm-element :mask #x7f :tag #x2 :print #x1 :string1 "offset" :string2 "offset") + (new 'static 'vif-disasm-element :mask #x7f :tag #x3 :print #x1 :string1 "base" :string2 "base") + (new 'static 'vif-disasm-element :mask #x7f :tag #x4 :print #x1 :string1 "itop" :string2 "addr") + (new 'static 'vif-disasm-element :mask #x7f :tag #x5 :print #x1 :string1 "stmod" :string2 "mode") + (new 'static 'vif-disasm-element :mask #x7f :tag #x6 :print #x1 :string1 "mskpath3" :string2 "mask") + (new 'static 'vif-disasm-element :mask #x7f :tag #x7 :print #x1 :string1 "mark" :string2 "mark") + (new 'static 'vif-disasm-element :mask #x7f :tag #x10 :string1 "flushe") + (new 'static 'vif-disasm-element :mask #x7f :tag #x11 :string1 "flush") + (new 'static 'vif-disasm-element :mask #x7f :tag #x13 :string1 "flusha") + (new 'static 'vif-disasm-element :mask #x7f :tag #x14 :print #x1 :string1 "mscal" :string2 "addr") + (new 'static 'vif-disasm-element :mask #x7f :tag #x17 :string1 "mscnt") + (new 'static 'vif-disasm-element :mask #x7f :tag #x15 :print #x1 :string1 "mscalf" :string2 "addr") + (new 'static 'vif-disasm-element :mask #x7f :tag #x20 :print #x3 :string1 "stmask" :string2 "mask") + (new 'static 'vif-disasm-element :mask #x7f :tag #x30 :print #x4 :string1 "strow" :string2 "row") + (new 'static 'vif-disasm-element :mask #x7f :tag #x31 :print #x4 :string1 "stcol" :string2 "col") + (new 'static 'vif-disasm-element :mask #x7f :tag #x4a :print #x5 :string1 "mpg") + (new 'static 'vif-disasm-element :mask #x7f :tag #x50 :print #x6 :string1 "direct") + (new 'static 'vif-disasm-element :mask #x7f :tag #x51 :print #x6 :string1 "directhl") + (new 'static 'vif-disasm-element :mask #x6f :tag #x60 :val #x10 :print #x7 :string1 "unpack-s-32") + (new 'static 'vif-disasm-element :mask #x6f :tag #x61 :val #x8 :print #x7 :string1 "unpack-s-16") + (new 'static 'vif-disasm-element :mask #x6f :tag #x62 :val #x4 :print #x7 :string1 "unpack-s-8") + (new 'static 'vif-disasm-element :mask #x6f :tag #x64 :val #x8 :print #x7 :string1 "unpack-v2-32") + (new 'static 'vif-disasm-element :mask #x6f :tag #x65 :val #x4 :print #x7 :string1 "unpack-v2-16") + (new 'static 'vif-disasm-element :mask #x6f :tag #x66 :val #x2 :print #x7 :string1 "unpack-v2-8") + (new 'static 'vif-disasm-element :mask #x6f :tag #x68 :val #xc :print #x7 :string1 "unpack-v3-32") + (new 'static 'vif-disasm-element :mask #x6f :tag #x69 :val #x6 :print #x7 :string1 "unpack-v3-16") + (new 'static 'vif-disasm-element :mask #x6f :tag #x6a :val #x3 :print #x7 :string1 "unpack-v3-8") + (new 'static 'vif-disasm-element :mask #x6f :tag #x6c :val #x10 :print #x7 :string1 "unpack-v4-32") + (new 'static 'vif-disasm-element :mask #x6f :tag #x6d :val #x8 :print #x7 :string1 "unpack-v4-16") + (new 'static 'vif-disasm-element :mask #x6f :tag #x6e :val #x4 :print #x7 :string1 "unpack-v4-8") + (new 'static 'vif-disasm-element :mask #x6f :tag #x6f :val #x2 :print #x7 :string1 "unpack-v4-5") + (new 'static 'vif-disasm-element :print #x8))) (defun disasm-vif-details ((stream symbol) (data (pointer uint8)) (kind int) (count int)) (local-vars diff --git a/goal_src/engine/gfx/texture-h.gc b/goal_src/engine/gfx/texture-h.gc index 0854a1d0b8..fcc8bd9287 100644 --- a/goal_src/engine/gfx/texture-h.gc +++ b/goal_src/engine/gfx/texture-h.gc @@ -5,3 +5,225 @@ ;; name in dgo: texture-h ;; dgos: GAME, ENGINE +;; mask for different texture things. +;; these are the ones that will be displayed in the menu +(define *texture-enable-user-menu* 31) + +;; enabled textures. +(define *texture-enable-user* 0) + +(deftype texture-id (uint32) + () + :method-count-assert 9 + :size-assert #x4 + :flag-assert #x900000004 + ) + +(defmethod inspect texture-id ((obj texture-id)) + (format #t "[~8x] ~A~%" obj 'texture-id) + (format #t "~Tindex: ~D~%" (shr (shl (the-as int obj) 44) 52)) + + ;; you aren't supposed to do this. + (format #t "~Tpage: ~D~%" (shr (logand #xffffffff obj) 20)) + ;; (set! t9-2 format) + ;; (set! a0-3 '#t) + ;; (set! a1-2 "~Tpage: ~D~%") + ;; (.srl a2-2 obj 20) + ;; (t9-2 a0-3 a1-2 a2-2) + obj + ) + +(deftype texture-pool-segment (structure) + ((dest uint32 :offset-assert 0) + (size uint32 :offset-assert 4) + ) + :pack-me + :method-count-assert 9 + :size-assert #x8 + :flag-assert #x900000008 + ) + +(deftype texture-pool (basic) + ((top int32 :offset-assert 4) + (cur int32 :offset-assert 8) + (allocate-func basic :offset-assert 12) + (font-palette int32 :offset-assert 16) + (segment texture-pool-segment 4 :inline :offset-assert 20) + (segment-near texture-pool-segment :inline :offset 20) + (segment-common texture-pool-segment :inline :offset 28) + (common-page int32 32 :offset-assert 52) + (common-page-mask int32 :offset-assert 180) + (ids int32 126 :offset-assert 184) + ) + :method-count-assert 23 + :size-assert #x2b0 + :flag-assert #x17000002b0 + (:methods + (dummy-9 () none 9) + (dummy-10 () none 10) + (dummy-11 () none 11) + (dummy-12 () none 12) + (dummy-13 () none 13) + (dummy-14 () none 14) + (dummy-15 () none 15) + (dummy-16 () none 16) + (dummy-17 () none 17) + (dummy-18 () none 18) + (dummy-19 () none 19) + (dummy-20 () none 20) + (dummy-21 () none 21) + (dummy-22 () none 22) + ) + ) + +(deftype texture (basic) + ((w int16 :offset-assert 4) + (h int16 :offset-assert 6) + (num-mips uint8 :offset-assert 8) + (tex1-control uint8 :offset-assert 9) + (psm uint8 :offset-assert 10) + (mip-shift uint8 :offset-assert 11) + (clutpsm uint16 :offset-assert 12) + (dest uint16 7 :offset-assert 14) + (clutdest uint16 :offset-assert 28) + (width uint8 7 :offset-assert 30) + (name basic :offset-assert 40) + (size uint32 :offset-assert 44) + (uv-dist float :offset-assert 48) + (masks uint32 3 :offset-assert 52) + ) + :method-count-assert 9 + :size-assert #x40 + :flag-assert #x900000040 + ) + +(deftype texture-page-segment (structure) + ((block-data pointer :offset-assert 0) + (size uint32 :offset-assert 4) + (dest uint32 :offset-assert 8) + ) + :pack-me + :method-count-assert 9 + :size-assert #xc + :flag-assert #x90000000c + ) + +(defun texture-mip->segment ((arg0 int) (arg1 int)) + (if (>= 2 arg1) (+ (- -1 arg0) arg1) (max 0 (- 2 arg0))) + ) + +(deftype texture-page (basic) + ((info basic :offset-assert 4) + (name basic :offset-assert 8) + (id uint32 :offset-assert 12) + (length int32 :offset-assert 16) + (mip0-size uint32 :offset-assert 20) + (size uint32 :offset-assert 24) + (segment texture-page-segment 3 :inline :offset-assert 28) + (pad uint32 16 :offset-assert 64) + (data uint8 :dynamic :offset-assert 128) + ) + :method-count-assert 15 + :size-assert #x80 + :flag-assert #xf00000080 + (:methods + (dummy-9 () none 9) + (dummy-10 () none 10) + (dummy-11 () none 11) + (dummy-12 () none 12) + (dummy-13 () none 13) + (dummy-14 () none 14) + ) + ) + +(deftype shader-ptr (uint32) + () + :method-count-assert 9 + :size-assert #x4 + :flag-assert #x900000004 + ) + +(deftype texture-link (structure) + ((next uint32 :offset-assert 0) + ) + :method-count-assert 9 + :size-assert #x4 + :flag-assert #x900000004 + ) + +(deftype texture-page-dir-entry (structure) + ((length int16 :offset-assert 0) + (status uint16 :offset-assert 2) + (page basic :offset-assert 4) + (link uint32 :offset-assert 8) + ) + :method-count-assert 9 + :size-assert #xc + :flag-assert #x90000000c + ) + +(deftype texture-page-dir (basic) + ((pad uint8 #x10)) + (:methods + (dummy-9 () none 9) + ) + :flag-assert #xa00000014 + ) + +(deftype texture-relocate-later (basic) + ((memcpy basic :offset-assert 4) + (dest uint32 :offset-assert 8) + (source uint32 :offset-assert 12) + (move uint32 :offset-assert 16) + (entry texture-page-dir-entry :offset-assert 20) + (page basic :offset-assert 24) + ) + :method-count-assert 9 + :size-assert #x1c + :flag-assert #x90000001c + ) + +(define *texture-relocate-later* (new 'global 'texture-relocate-later)) +(set! (-> *texture-relocate-later* memcpy) #f) +(define *texture-page-dir* #f) + +(deftype adgif-shader (structure) + ( + ;;(quad UNKNOWN 5 :offset-assert 0) + ;;(prims UNKNOWN 10 :offset-assert 0) + (tex0 uint64 :offset-assert 0) + (tex1 uint64 :offset 16) + (miptbp1 uint64 :offset 32) + (clamp uint64 :offset 48) + (clamp-reg uint64 :offset 56) + (alpha uint64 :offset 64) + (link-test uint32 :offset 8) + (texture-id uint32 :offset 24) + (next uint32 :offset 40) + (_pad uint64) + ) + :method-count-assert 9 + :size-assert #x50 + :flag-assert #x900000050 + ) + +(deftype adgif-shader-array (inline-array-class) + () + :method-count-assert 9 + :size-assert #x10 + :flag-assert #x900000010 + ) +(set! (-> adgif-shader-array heap-base) 80) + +(define *sky-base-vram-word* 0) +(define *sky-base-block* 0) +(define *sky-base-page* 0) +(define *depth-cue-base-vram-word* 0) +(define *depth-cue-base-block* 0) +(define *depth-cue-base-page* 0) +(define *eyes-base-vram-word* 0) +(define *eyes-base-block* 0) +(define *eyes-base-page* 0) +(define *ocean-base-vram-word* 0) +(define *ocean-base-block* 0) +(define *ocean-base-page* 0) diff --git a/goal_src/engine/level/level-h.gc b/goal_src/engine/level/level-h.gc index 0c92b3dab3..7c6392b6a8 100644 --- a/goal_src/engine/level/level-h.gc +++ b/goal_src/engine/level/level-h.gc @@ -5,3 +5,378 @@ ;; name in dgo: level-h ;; dgos: GAME, ENGINE +;; Information related to visibility data for a level. +;; Unclear why there are 8 of these per level. +;; Perhaps there are up to 8 "chunks" of the visibility loaded at a single time? +;; The full visibility data lives on the IOP. +(deftype level-vis-info (basic) + ((level basic :offset-assert 4) + (from-level basic :offset-assert 8) + (from-bsp basic :offset-assert 12) + (flags uint32 :offset-assert 16) + (length uint32 :offset-assert 20) + (allocated-length uint32 :offset-assert 24) + (dictionary-length uint32 :offset-assert 28) + (dictionary uint32 :offset-assert 32) + (string-block uint32 :offset-assert 36) + (ramdisk uint32 :offset-assert 40) + (vis-bits uint32 :offset-assert 44) + (current-vis-string uint32 :offset-assert 48) + (vis-string uint8 :dynamic :offset-assert 52) + ) + :method-count-assert 9 + :size-assert #x34 + :flag-assert #x900000034 + ) + + +(defmethod asize-of level-vis-info ((obj level-vis-info)) + "Get the size of a level-vis-info in memory" + (the-as int (+ (-> level-vis-info size) (-> obj dictionary-length))) + ) + +;; Per level information related to how to load the level. +;; These are stored in level-info.gc +(deftype level-load-info (basic) + ((name-list basic 3 :offset-assert 4) + (index int32 :offset-assert 16) ;; the level number (starting with 1?) + (name basic :offset 4) ;; symbol with full name, like "misty" + (visname basic :offset 8) ;; symbol with vis file name, like "misty-vis" + (nickname basic :offset 12) ;; 3 letter name for DGO, like "mis" + (packages pair :offset-assert 20) ;; list of symbols, usually empty or the level name + (sound-banks pair :offset-assert 24) ;; require sound bank files (list of symbols) + (music-bank basic :offset-assert 28) ;; name of level music + (ambient-sounds pair :offset-assert 32) ;; always empty list. + (mood basic :offset-assert 36) ;; mood object name + (mood-func basic :offset-assert 40) ;; mood update function name + (ocean basic :offset-assert 44) ;; ocean map object + (sky basic :offset-assert 48) ;; boolean to enable sky + (sun-fade float :offset-assert 52) ;; sun/sky setting + (continues pair :offset-assert 56) ;; list of checkpoints + (tasks pair :offset-assert 60) ;; list of boxed integers for tasks + (priority int32 :offset-assert 64) ;; either 100 or 200 + (load-commands pair :offset-assert 68) ;; ?? + (alt-load-commands pair :offset-assert 72) ;; ?? + (bsp-mask uint64 :offset-assert 80) ;; ?? + (bsphere sphere :offset-assert 88) ;; boundings sphere of level? + (buzzer int32 :offset-assert 92) ;; which task is the scout fly? + (bottom-height float :offset-assert 96) ;; meters + (run-packages pair :offset-assert 100) ;; possibly unused? + (prev-level basic :offset-assert 104) + (next-level basic :offset-assert 108) + (wait-for-load basic :offset-assert 112) + ) + :method-count-assert 9 + :size-assert #x74 + :flag-assert #x900000074 + ) + +(deftype login-state (basic) + ((state int32 :offset-assert 4) + (pos uint32 :offset-assert 8) + (elts uint32 :offset-assert 12) + (elt uint32 16 :offset-assert 16) + ) + :method-count-assert 9 + :size-assert #x50 + :flag-assert #x900000050 + ) + +;; Per level-buffer info about the current loaded level. +(deftype level (basic) + ((name basic :offset-assert 4) + (load-name basic :offset-assert 8) + (nickname basic :offset-assert 12) + (index int32 :offset-assert 16) + (status basic :offset-assert 20) + (other basic :offset-assert 24) + (heap kheap :inline :offset-assert 32) + (bsp basic :offset-assert 48) + (art-group basic :offset-assert 52) + (info basic :offset-assert 56) + (texture-page basic 9 :offset-assert 60) + (loaded-texture-page basic 16 :offset-assert 96) + (loaded-texture-page-count int32 :offset-assert 160) + (foreground-sink-group dma-foreground-sink-group 3 :inline :offset-assert 176) ;; inline basic, todo check stride. + (foreground-draw-engine basic 3 :offset-assert 272) + (entity basic :offset-assert 284) + (ambient basic :offset-assert 288) + (closest-object basic 9 :offset-assert 292) + (upload-size uint32 9 :offset-assert 328) + (level-distance float :offset-assert 364) ; meters + (inside-sphere? basic :offset-assert 368) + (inside-boxes? basic :offset-assert 372) + (display? basic :offset-assert 376) + (meta-inside? basic :offset-assert 380) + (mood basic :offset-assert 384) + (mood-func basic :offset-assert 388) + (vis-bits uint32 :offset-assert 392) + (all-visible? basic :offset-assert 396) + (force-all-visible? basic :offset-assert 400) + (linking basic :offset-assert 404) + (vis-info level-vis-info 8 :offset-assert 408) + (vis-self-index int32 :offset-assert 440) + (vis-adj-index int32 :offset-assert 444) + (vis-buffer uint8 2048 :offset-assert 448) + (mem-usage-block basic :offset-assert 2496) + (mem-usage int32 :offset-assert 2500) + (code-memory-start uint32 :offset-assert 2504) + (code-memory-end uint32 :offset-assert 2508) + (texture-mask uint32 9 :offset-assert 2512) + (force-inside? basic :offset-assert 2548) + (pad uint8 56) + ) + :method-count-assert 29 + :size-assert #xa30 + :flag-assert #x1d00000a30 + (:methods + (dummy-9 () none 9) + (dummy-10 () none 10) + (dummy-11 () none 11) + (dummy-12 () none 12) + (dummy-13 () none 13) + (dummy-14 () none 14) + (dummy-15 () none 15) + (dummy-16 () none 16) + (dummy-17 () none 17) + (dummy-18 () none 18) + (dummy-19 () none 19) + (dummy-20 () none 20) + (dummy-21 () none 21) + (dummy-22 () none 22) + (dummy-23 () none 23) + (dummy-24 () none 24) + (dummy-25 () none 25) + (dummy-26 () none 26) + (dummy-27 () none 27) + (dummy-28 () none 28) + ) + ) + +(declare-type entity-links structure) + +;; Main *level* object. +;; There are actually three levels. level0 and level1 correspond to the actual buffered levels +;; The level-default seems to be a fake level that can possibly be used by renderers that +;; don't belong to any level, for example to render Jak. +(deftype level-group (basic) + ((length int32 :offset-assert 4) + (unknown-field-1 basic :offset-assert 8) + (unknown-field-2 basic :offset-assert 12) + (entity-link entity-links :offset 16) ;; not sure what's going on here + (border? basic :offset-assert 20) + (vis? basic :offset-assert 24) + (want-level basic :offset-assert 28) + (receiving-level basic :offset-assert 32) + (load-commands pair :offset-assert 36) + (play? basic :offset-assert 40) + ;; there's something? from 40 -> 96. + (hack-pad uint8 :offset 90) + + ;(level level 3 :inline :offset-assert 96) + ;(data level 3 :inline :offset-assert 100) + (level0 level :inline :offset-assert 96) ;; inline basic + (level1 level :inline :offset-assert 2704) ;; inline basic + (level-default level :inline :offset-assert 5312) ;; inline basic + (level level 3 :inline :offset 96) + (pad uint32) + ) + :method-count-assert 27 + :size-assert #x1ef4 + :flag-assert #x1b00001ef4 + (:methods + (dummy-9 () none 9) + (dummy-10 () none 10) + (dummy-11 () none 11) + (dummy-12 () none 12) + (dummy-13 () none 13) + (dummy-14 () none 14) + (dummy-15 () none 15) + (dummy-16 () none 16) + (dummy-17 () none 17) + (dummy-18 () none 18) + (dummy-19 () none 19) + (dummy-20 () none 20) + (dummy-21 () none 21) + (dummy-22 () none 22) + (dummy-23 () none 23) + (dummy-24 () none 24) + (dummy-25 () none 25) + (dummy-26 () none 26) + ) + ) + +(define-extern *level* level-group) +;; OpenGOAL doesn not yet support setting fields to arrays in statics. +#| +(if (zero? *level*) + (set! *level* + (new 'static 'level-group + :length 2 + :unknown-field-1 #f + :unknown-field-2 #f + :entity-link #f + :border? #f + :want-level #f + :load-commands '() + :play? #f + :level0 + (new 'static 'level + :name #f + :status 'inactive + :foreground-sink-group + (new 'static 'inline-array 'dma-foreground-sink-group 3 + (new 'static 'dma-foreground-sink-group + :sink (new 'static 'array 'dma-foreground-sink 3 + (new 'static 'dma-foreground-sink :bucket 10) + (new 'static 'generic-dma-foreground-sink + :bucket 11 :foreground-output-bucket 1 + ) + 0) + ) + (new 'static 'dma-foreground-sink-group + :sink (new 'static 'array 'dma-foreground-sink 3 + (new 'static 'dma-foreground-sink + :bucket 49 + :foreground-texture-page 1 + ) + (new 'static 'generic-dma-foreground-sink : + bucket 50 :foreground-texture-page 1 + :foreground-output-bucket 1) + 0) + ) + (new 'static 'dma-foreground-sink-group + :sink (new 'static 'array 'dma-foreground-sink 3 + (new 'static 'dma-foreground-sink + :bucket 58 + :foreground-texture-page 2 + ) + (new 'static 'generic-dma-foreground-sink + :bucket 59 + :foreground-texture-page 2 + :foreground-output-bucket 1 + ) + 0) + ) + ) + :inside-sphere? #f + :inside-boxes? #f + :force-inside? #f + ) + :level1 + (new 'static 'level + :name #f + :index 1 + :status 'inactive + :foreground-sink-group + (new 'static 'inline-array 'dma-foreground-sink-group 3 + (new 'static 'dma-foreground-sink-group + :sink (new 'static 'array 'dma-foreground-sink 3 + (new 'static 'dma-foreground-sink + :bucket 17 + :foreground-texture-level 1 + ) + (new 'static 'generic-dma-foreground-sink + :bucket 18 + :foreground-texture-level 1 + :foreground-output-bucket 1 + ) + 0) + ) + (new 'static 'dma-foreground-sink-group + :sink (new 'static 'array 'dma-foreground-sink 3 + (new 'static 'dma-foreground-sink + :bucket 52 + :foreground-texture-page 1 + :foreground-texture-level 1 + ) + (new 'static 'generic-dma-foreground-sink + :bucket 53 + :foreground-texture-page 1 + :foreground-texture-level 1 + :foreground-output-bucket 1 + ) + 0 + ) + ) + (new 'static 'dma-foreground-sink-group + :sink (new 'static 'array 'dma-foreground-sink 3 + (new 'static 'dma-foreground-sink + :bucket 61 + :foreground-texture-page 2 + :foreground-texture-level 1 + ) + (new 'static 'generic-dma-foreground-sink + :bucket 62 + :foreground-texture-page 2 + :foreground-texture-level 1 + :foreground-output-bucket 1 + ) + 0) + ) + ) + :inside-sphere? #f + :inside-boxes? #f + :force-inside? #f + ) + :level-default + (new 'static 'level + :name 'default + :index 2 + :status 'reserved + :foreground-sink-group + (new 'static 'inline-array 'dma-foreground-sink-group 3 + (new 'static 'dma-foreground-sink-group + :sink (new 'static 'array 'dma-foreground-sink 3 + (new 'static 'dma-foreground-sink + :bucket 45 + :foreground-texture-level 2 + ) + (new 'static 'generic-dma-foreground-sink + :bucket 46 + :foreground-texture-level 2 + :foreground-output-bucket 1 + ) + 0) + ) + (new 'static 'dma-foreground-sink-group : + sink (new 'static 'array 'dma-foreground-sink 3 + (new 'static 'dma-foreground-sink + :bucket 55 + :foreground-texture-page 1 + :foreground-texture-level 2 + ) + (new 'static 'generic-dma-foreground-sink + :bucket 56 + :foreground-texture-page 1 + :foreground-texture-level 2 + :foreground-output-bucket 1 + ) + 0) + ) + (new 'static 'dma-foreground-sink-group + :sink (new 'static 'array 'dma-foreground-sink 3 + (new 'static 'dma-foreground-sink + :bucket 58 + :foreground-texture-page 2 + :foreground-texture-level 2 + ) + (new 'static 'generic-dma-foreground-sink + :bucket 59 + :foreground-texture-page 2 + :foreground-texture-level 2 + :foreground-output-bucket 1 + ) + 0 + ) + ) + ) + :inside-sphere? #f + :inside-boxes? #f + :force-inside? #f + ) + ) + ) + ) +|# + diff --git a/goal_src/engine/math/vector-h.gc b/goal_src/engine/math/vector-h.gc index 5da43425bf..317af87f0a 100644 --- a/goal_src/engine/math/vector-h.gc +++ b/goal_src/engine/math/vector-h.gc @@ -294,7 +294,7 @@ ;; Vector of 4 floats. Shortened to "vector" because it is commonly used. (deftype vector (structure) - ((data float 4 :offset-assert 0) + ((data float 4 :do-not-decompile :offset-assert 0) (x float :offset 0) (y float :offset 4) (z float :offset 8) diff --git a/goalc/compiler/compilation/Static.cpp b/goalc/compiler/compilation/Static.cpp index c2d55304dc..1e60c02874 100644 --- a/goalc/compiler/compilation/Static.cpp +++ b/goalc/compiler/compilation/Static.cpp @@ -408,7 +408,7 @@ StaticResult Compiler::compile_static(const goos::Object& form, Env* env) { if (!rest.as_pair()->cdr.is_empty_list()) { throw_compiler_error(form, "The form {} is an invalid quoted form.", form.print()); } - if (second.is_pair()) { + if (second.is_pair() || second.is_empty_list()) { return compile_static_no_eval_for_pairs(second, env); } else { throw_compiler_error(form, "Could not evaluate the quoted form {} at compile time.", diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 49f7e4ce2e..8e530a7e3c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -26,6 +26,7 @@ add_executable(goalc-test ${CMAKE_CURRENT_LIST_DIR}/decompiler/test_InstructionParser.cpp ${CMAKE_CURRENT_LIST_DIR}/decompiler/test_gkernel_decomp.cpp ${CMAKE_CURRENT_LIST_DIR}/decompiler/test_math_decomp.cpp + ${CMAKE_CURRENT_LIST_DIR}/decompiler/test_DataParser.cpp ${GOALC_TEST_FRAMEWORK_SOURCES} ${GOALC_TEST_CASES}) diff --git a/test/decompiler/test_DataParser.cpp b/test/decompiler/test_DataParser.cpp new file mode 100644 index 0000000000..cb05cfbf44 --- /dev/null +++ b/test/decompiler/test_DataParser.cpp @@ -0,0 +1,219 @@ + +#include "gtest/gtest.h" +#include "decompiler/util/DataParser.h" +#include "decompiler/Disasm/DecompilerLabel.h" +#include "decompiler/util/data_decompile.h" +#include "decompiler/util/DecompilerTypeSystem.h" +#include "third-party/fmt/core.h" +#include "common/goos/PrettyPrinter.h" + +using namespace decompiler; + +class DataDecompTest : public ::testing::Test { + protected: + static std::unique_ptr dts; + + static void SetUpTestCase() { + dts = std::make_unique(); + dts->parse_type_defs({"decompiler", "config", "all-types.gc"}); + } + + static void TearDownTestCase() { dts.reset(); } + + void check_forms_equal(const std::string& expected, const std::string& actual) { + auto expected_form = + pretty_print::get_pretty_printer_reader().read_from_string(expected, false).as_pair()->car; + auto actual_form = + pretty_print::get_pretty_printer_reader().read_from_string(actual, false).as_pair()->car; + if (expected_form != actual_form) { + printf("Got:\n%s\n\nExpected\n%s\n", pretty_print::to_string(actual_form).c_str(), + pretty_print::to_string(expected_form).c_str()); + } + EXPECT_TRUE(expected_form == actual_form); + } +}; + +std::unique_ptr DataDecompTest::dts; + +TEST_F(DataDecompTest, Basic) { + // just test we can parse a very simple data. + std::string input = + "L123: (offset 2)\n" + " .type test\n" + " .symbol #t\n" + "L234:\n" + " .empty-list\n" + " .word L123\n" + " .word L555\n" + "L555:\n" + " .symbol asdf\n" + " .word 0x0\n" + " .word 0xdeadbeef\n" + " .word 0xffffffff\n"; + auto result = parse_data(input); + EXPECT_EQ(input, result.print()); +} + +TEST_F(DataDecompTest, FromDecomp) { + std::string input = + " .symbol finalboss\n" + " .empty-list\n" + " .symbol *finalboss-mood*\n" + " .symbol update-mood-finalboss\n" + " .symbol #f\n" + " .symbol #t\n" + " .word 0x3f800000\n" + " .word L55\n" + " .empty-list\n" + " .word 0x64\n" + " .empty-list\n" + " .empty-list\n" + " .word 0x0\n" + "L63:\n" + " .word 0xffffffff\n" + "L62:\n" + " .word 0xffffffff\n" + "L968:\n" + " .word L54\n" + " .word 0x5b\n" + " .word 0xc8e40000\n" + " .word L53\n" + " .word 0x0\n" + " .word 0x0\n" + " .symbol #f\n" + " .word 0x0\n" + "L53: (offset 2)\n" + " .word L968\n" + " .empty-list\n" + "L54:\n" + " .word 0x4b34a000\n" + " .word 0x4a020000\n" + " .word 0xcb956000\n" + " .word 0x493e0000\n" + "L55: (offset 2)\n" + " .word L63\n" + " .word L56\n" + "L56: (offset 2)\n" + " .word L57\n" + " .empty-list\n" + " .type continue-point\n" + "L57:\n" + " .word L62\n" + " .symbol finalboss\n" + " .word 0x0\n" + " .word 0x4b3b814f\n" + " .word 0x49f088ef\n" + " .word 0xcb976ea5\n" + " .word 0x3f800000\n" + " .word 0x0\n" + " .word 0xbf0930be\n" + " .word 0x0\n"; + auto result = parse_data(input); + EXPECT_EQ(input, result.print()); +} + +TEST_F(DataDecompTest, String) { + std::string input = + " .type string\n" + "L62:\n" + " .word 0xf\n" + " .word 0x616e6966\n" + " .word 0x736f626c\n" + " .word 0x69662d73\n" + " .word 0x746867\n" + " .word 0x0\n" + " .word 0x0\n"; + auto parsed = parse_data(input); + auto decomp = + decompile_at_label_guess_type(parsed.label("L62"), parsed.labels, {parsed.words}, dts->ts); + EXPECT_EQ(decomp.print(), "\"finalboss-fight\""); +} + +TEST_F(DataDecompTest, SimpleStructure) { + std::string input = + "L217:\n" + " .word 0x7f\n" + " .word 0x1\n" + " .word 0x0\n" + " .word 0x2\n" + " .word L218\n" + " .word 0x0\n" + " .word 0x0\n" + " .word 0x0\n" + " .type string\n" + "L218:\n" + " .word 0x6\n" + " .word 0x79637473\n" + " .word 0x6c63\n"; + auto parsed = parse_data(input); + auto decomp = decompile_at_label(TypeSpec("vif-disasm-element"), parsed.label("L217"), + parsed.labels, {parsed.words}, dts->ts); + check_forms_equal( + decomp.print(), + "(new 'static 'vif-disasm-element :mask #x7f :tag #x1 :print #x2 :string1 \"stcycl\")"); +} + +TEST_F(DataDecompTest, VifDisasmArray) { + std::string input = + ".type array\n" + "L148:\n" + " .word 0x3\n" + " .word 0x3\n" + " .type vif-disasm-element\n" + " .word L219\n" + " .word L217\n" + " .word L215\n" + " .word 0x0\n" + "L215:\n" + " .word 0x7f\n" + " .word 0x2\n" + " .word 0x0\n" + " .word 0x1\n" + " .word L216\n" + " .word L216\n" + " .word 0x0\n" + " .word 0x0\n" + " .type string\n" + "L216:\n" + " .word 0x6\n" + " .word 0x7366666f\n" + " .word 0x7465\n" + "L217:\n" + " .word 0x7f\n" + " .word 0x1\n" + " .word 0x0\n" + " .word 0x2\n" + " .word L218\n" + " .word 0x0\n" + " .word 0x0\n" + " .word 0x0\n" + " .type string\n" + "L218:\n" + " .word 0x6\n" + " .word 0x79637473\n" + " .word 0x6c63\n" + "L219:\n" + " .word 0x7f\n" + " .word 0x0\n" + " .word 0x0\n" + " .word 0x0\n" + " .word L220\n" + " .word 0x0\n" + " .word 0x0\n" + " .word 0x0\n" + " .type string\n" + "L220:\n" + " .word 0x3\n" + " .word 0x706f6e\n" + " .word 0x0"; + auto parsed = parse_data(input); + auto decomp = + decompile_at_label_guess_type(parsed.label("L148"), parsed.labels, {parsed.words}, dts->ts); + check_forms_equal( + decomp.print(), + "(new 'static 'boxed-array vif-disasm-element 3\n" + " (new 'static 'vif-disasm-element :mask #x7f :string1 \"nop\")\n" + " (new 'static 'vif-disasm-element :mask #x7f :tag #x1 :print #x2 :string1 \"stcycl\")\n" + " (new 'static 'vif-disasm-element :mask #x7f :tag #x2 :print #x1 :string1 \"offset\" " + " :string2 \"offset\"))"); +} \ No newline at end of file