mirror/jak-project
1
0
Fork 0
mirror of https://github.com/open-goal/jak-project synced 2026-09-03 02:14:07 -04:00
Code Issues Packages Projects Releases Wiki Activity

[decompiler/compiler] Fixes for task-control (#668)

Browse Source 
* fix decompiler for task control

* support in compiler

* changelog

* typo
This commit is contained in:
water111
2021-07-01 21:38:19 -04:00
committed by GitHub
parent d4e68e7ef4
commit 4bea175140
29 changed files with 343 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
decompiler/CMakeLists.txt
+1 -1
View File
@@ -2,7 +2,6 @@ add_library(
decomp
SHARED
analysis/anonymous_function_def.cpp
analysis/atomic_op_builder.cpp
analysis/cfg_builder.cpp
analysis/expression_build.cpp
@@ -11,6 +10,7 @@ add_library(
analysis/insert_lets.cpp
analysis/reg_usage.cpp
analysis/stack_spill.cpp
analysis/static_refs.cpp
analysis/symbol_def_map.cpp
analysis/type_analysis.cpp
analysis/variable_naming.cpp
decompiler/Function/Function.h
+5 -6
View File
@@ -1,8 +1,5 @@
#pragma once
#ifndef NEXT_FUNCTION_H
#define NEXT_FUNCTION_H
#include <string>
#include <vector>
#include <unordered_map>
@@ -45,7 +42,7 @@ struct FunctionName {
case FunctionKind::METHOD:
return "(method " + std::to_string(method_id) + " " + type_name + ")";
case FunctionKind::TOP_LEVEL_INIT:
return "(top-level-login)";
return "(top-level-login " + object_name + ")";
case FunctionKind::UNIDENTIFIED:
return "(anon-function " + std::to_string(id_in_object) + " " + object_name + ")";
default:
@@ -60,7 +57,10 @@ struct FunctionName {
bool empty() const { return kind == FunctionKind::UNIDENTIFIED; }
void set_as_top_level() { kind = FunctionKind::TOP_LEVEL_INIT; }
void set_as_top_level(const std::string& object_file_name) {
kind = FunctionKind::TOP_LEVEL_INIT;
object_name = object_file_name;
}
void set_as_global(std::string name) {
kind = FunctionKind::GLOBAL;
@@ -175,4 +175,3 @@ class Function {
std::unordered_map<int, int> basic_op_to_instruction;
};
} // namespace decompiler
#endif // NEXT_FUNCTION_H
decompiler/IR2/Form.cpp
+21 -3
View File
@@ -6,6 +6,7 @@
#include "common/goos/PrettyPrinter.h"
#include "common/type_system/TypeSystem.h"
#include "decompiler/util/DecompilerTypeSystem.h"
#include "decompiler/util/data_decompile.h"
namespace decompiler {
@@ -2118,11 +2119,16 @@ void StoreArrayAccess::get_modified_regs(RegSet& regs) const {
// DecompiledDataElement
/////////////////////////////
DecompiledDataElement::DecompiledDataElement(goos::Object description)
: m_description(std::move(description)) {}
DecompiledDataElement::DecompiledDataElement(const DecompilerLabel& label,
const std::optional<LabelType>& type_hint)
: m_label(label), m_type_hint(type_hint) {}
goos::Object DecompiledDataElement::to_form_internal(const Env&) const {
return m_description;
if (m_decompiled) {
return m_description;
} else {
return pretty_print::to_symbol(fmt::format("<static-data {}>", m_label.name));
}
}
void DecompiledDataElement::apply(const std::function<void(FormElement*)>& f) {
@@ -2135,6 +2141,18 @@ void DecompiledDataElement::collect_vars(RegAccessSet&, bool) const {}
void DecompiledDataElement::get_modified_regs(RegSet&) const {}
void DecompiledDataElement::do_decomp(const Env& env, const LinkedObjectFile* file) {
if (m_type_hint) {
m_description = decompile_at_label_with_hint(*m_type_hint, m_label, env.file->labels,
env.file->words_by_seg, *env.dts, file);
} else {
m_description = decompile_at_label_guess_type(m_label, env.file->labels, env.file->words_by_seg,
env.dts->ts, file);
}
m_decompiled = true;
}
/////////////////////////////
// LetElement
/////////////////////////////
decompiler/IR2/Form.h
+12 -1
View File
@@ -8,6 +8,7 @@
#include "decompiler/IR2/AtomicOp.h"
#include "common/goos/Object.h"
#include "common/type_system/TypeSystem.h"
#include "decompiler/Disasm/DecompilerLabel.h"
namespace decompiler {
class Form;
@@ -1261,17 +1262,27 @@ class StoreArrayAccess : public FormElement {
std::optional<TypeSpec> m_src_cast_type;
};
/*!
* This marks some static data that will be decompiled in a later pass.
* This is done at the very end so that we can make sure all static references to lambdas work.
*/
class DecompiledDataElement : public FormElement {
public:
DecompiledDataElement(goos::Object description);
// DecompiledDataElement(goos::Object description);
DecompiledDataElement(const DecompilerLabel& label,
const std::optional<LabelType>& type_hint = {});
goos::Object to_form_internal(const Env& env) const override;
void apply(const std::function<void(FormElement*)>& f) override;
void apply_form(const std::function<void(Form*)>& f) override;
void collect_vars(RegAccessSet& vars, bool recursive) const override;
void get_modified_regs(RegSet& regs) const override;
void do_decomp(const Env& env, const LinkedObjectFile* file);
private:
bool m_decompiled = false;
goos::Object m_description;
DecompilerLabel m_label;
std::optional<LabelType> m_type_hint;
};
class LetElement : public FormElement {
decompiler/IR2/FormExpressionAnalysis.cpp
+3 -6
View File
@@ -454,16 +454,13 @@ void SimpleExpressionElement::update_from_stack_identity(const Env& env,
auto kv = env.label_types().find(lab.name);
if (kv != env.label_types().end()) {
auto type_name = kv->second.type_name;
// the actual decompilation is deferred until later, once static lambdas are done.
if (type_name == "_auto_") {
auto decompiled_data = decompile_at_label_guess_type(lab, env.file->labels,
env.file->words_by_seg, env.dts->ts);
result->push_back(pool.alloc_element<DecompiledDataElement>(decompiled_data));
result->push_back(pool.alloc_element<DecompiledDataElement>(lab));
} else if (type_name == "_lambda_") {
result->push_back(this);
} else {
auto decompiled_data = decompile_at_label_with_hint(kv->second, lab, env.file->labels,
env.file->words_by_seg, *env.dts);
result->push_back(pool.alloc_element<DecompiledDataElement>(decompiled_data));
result->push_back(pool.alloc_element<DecompiledDataElement>(lab, kv->second));
}
} else {
result->push_back(this);
decompiler/ObjectFile/LinkedObjectFile.cpp
+5 -1
View File
@@ -104,7 +104,11 @@ Function& LinkedObjectFile::get_function_at_label(int label_id) {
* Get the function starting at this label, or nullptr if there is none.
*/
const Function* LinkedObjectFile::try_get_function_at_label(int label_id) const {
auto& label = labels.at(label_id);
const auto& label = labels.at(label_id);
return try_get_function_at_label(label);
}
const Function* LinkedObjectFile::try_get_function_at_label(const DecompilerLabel& label) const {
for (auto& func : functions_by_seg.at(label.target_segment)) {
// + 4 to skip past type tag to the first word, which is were the label points.
if (func.start_word * 4 + 4 == label.offset) {
decompiler/ObjectFile/LinkedObjectFile.h
+1
View File
@@ -40,6 +40,7 @@ class LinkedObjectFile {
void symbol_link_offset(int source_segment, int source_offset, const char* name);
Function& get_function_at_label(int label_id);
const Function* try_get_function_at_label(int label_id) const;
const Function* try_get_function_at_label(const DecompilerLabel& label) const;
std::string get_label_name(int label_id) const;
uint32_t set_ordered_label_names();
void find_code();
decompiler/ObjectFile/ObjectFileDB.cpp
+3 -1
View File
@@ -160,6 +160,8 @@ ObjectFileDB::ObjectFileDB(const std::vector<std::string>& _dgos,
"No object files have been added. Check that there are input files and the allowed_objects "
"list.");
}
dts.bad_format_strings = config.bad_format_strings;
}
void ObjectFileDB::load_map_file(const std::string& map_data) {
@@ -639,7 +641,7 @@ void ObjectFileDB::analyze_functions_ir1(const Config& config) {
auto& func = data.linked_data.functions_by_seg.at(2).front();
assert(func.guessed_name.empty());
func.guessed_name.set_as_top_level();
func.guessed_name.set_as_top_level(data.to_unique_name());
func.find_global_function_defs(data.linked_data, dts);
func.find_type_defs(data.linked_data, dts);
func.find_method_defs(data.linked_data, dts);
decompiler/ObjectFile/ObjectFileDB_IR2.cpp
+3 -3
View File
@@ -18,7 +18,7 @@
#include "decompiler/analysis/expression_build.h"
#include "decompiler/analysis/inline_asm_rewrite.h"
#include "decompiler/analysis/stack_spill.h"
#include "decompiler/analysis/anonymous_function_def.h"
#include "decompiler/analysis/static_refs.h"
#include "decompiler/analysis/symbol_def_map.h"
#include "common/goos/PrettyPrinter.h"
#include "decompiler/IR2/Form.h"
@@ -96,7 +96,7 @@ void ObjectFileDB::ir2_top_level_pass(const Config& config) {
auto& func = data.linked_data.functions_by_seg.at(2).front();
assert(func.guessed_name.empty());
func.guessed_name.set_as_top_level();
func.guessed_name.set_as_top_level(data.to_unique_name());
func.find_global_function_defs(data.linked_data, dts);
func.find_type_defs(data.linked_data, dts);
func.find_method_defs(data.linked_data, dts);
@@ -602,7 +602,7 @@ void ObjectFileDB::ir2_insert_anonymous_functions() {
(void)segment_id;
(void)data;
if (func.ir2.top_form && func.ir2.env.has_type_analysis()) {
total += insert_anonymous_functions(func.ir2.top_form, *func.ir2.form_pool, func, dts);
total += insert_static_refs(func.ir2.top_form, *func.ir2.form_pool, func, dts);
}
});
decompiler/analysis/anonymous_function_def.h
-10
View File
@@ -1,10 +0,0 @@
#pragma once
#include "decompiler/IR2/Form.h"
namespace decompiler {
int insert_anonymous_functions(Form* top_level_form,
FormPool& pool,
const Function& function,
const DecompilerTypeSystem& dts);
}
decompiler/analysis/final_output.cpp
+4
View File
@@ -9,6 +9,10 @@ namespace decompiler {
goos::Object get_arg_list_for_function(const Function& func, const Env& env) {
std::vector<goos::Object> argument_elts;
if (func.type.arg_count() < 1) {
throw std::runtime_error(
fmt::format("Function {} has unknown type.\n", func.guessed_name.to_string()));
}
assert(func.type.arg_count() >= 1);
for (size_t i = 0; i < func.type.arg_count() - 1; i++) {
auto reg = Register(Reg::GPR, Reg::A0 + i);
decompiler/analysis/anonymous_function_def.cpp → decompiler/analysis/static_refs.cpp
+12 -5
View File
@@ -1,14 +1,14 @@
#include "anonymous_function_def.h"
#include "static_refs.h"
#include "common/goos/PrettyPrinter.h"
#include "decompiler/Function/Function.h"
#include "decompiler/ObjectFile/LinkedObjectFile.h"
#include "decompiler/analysis/final_output.h"
namespace decompiler {
int insert_anonymous_functions(Form* top_level_form,
FormPool& pool,
const Function& function,
const DecompilerTypeSystem&) {
int insert_static_refs(Form* top_level_form,
FormPool& pool,
const Function& function,
const DecompilerTypeSystem&) {
int replaced = 0;
top_level_form->apply_form([&](Form* f) {
auto atom = form_as_atom(f);
@@ -35,6 +35,13 @@ int insert_anonymous_functions(Form* top_level_form,
}
}
});
top_level_form->apply([&](FormElement* fe) {
auto as_static_data = dynamic_cast<DecompiledDataElement*>(fe);
if (as_static_data) {
as_static_data->do_decomp(function.ir2.env, function.ir2.env.file);
}
});
return replaced;
}
} // namespace decompiler
decompiler/analysis/static_refs.h
+10
View File
@@ -0,0 +1,10 @@
#pragma once
#include "decompiler/IR2/Form.h"
namespace decompiler {
int insert_static_refs(Form* top_level_form,
FormPool& pool,
const Function& function,
const DecompilerTypeSystem& dts);
}
decompiler/config.cpp
+3
View File
@@ -170,6 +170,9 @@ Config read_config_file(const std::string& path_to_config_file) {
config.hacks.cond_with_else_len_by_func_name[func_name].max_length_by_start_block[cond_name] =
max_len;
}
config.bad_format_strings =
hacks_json.at("bad_format_strings").get<std::unordered_map<std::string, int>>();
return config;
}
decompiler/config.h
+2
View File
@@ -100,6 +100,8 @@ struct Config {
std::unordered_map<std::string, std::vector<StackStructureHint>>
stack_structure_hints_by_function;
std::unordered_map<std::string, int> bad_format_strings;
DecompileHacks hacks;
};
decompiler/config/jak1_ntsc_black_label/hacks.jsonc
+13 -1
View File
@@ -448,6 +448,8 @@
"(method 12 perf-stat)"
],
// these functions use pairs and the decompiler
// will be less picky about types related to pairs.
"pair_functions_by_name": [
"ref",
"last",
@@ -470,5 +472,15 @@
"update-sound-banks",
"(method 16 level-group)",
"bg"
]
],
// If format is used with the wrong number of arguments,
// it will often mess up the decompilation, as the decompiler assumes
// that they used the correct number. This will override the decompiler's
// automatic detection.
"bad_format_strings": {
"ERROR: dma tag has data in reserved bits ~X~%":0,
"#<surface f0:~m f1:~f tf+:~f tf-:~f sf:~f tvv:~m":5,
"ERROR<GMJ>: value of symbol ~A in task-controls is not a task-control~%":0
}
}
decompiler/util/DecompilerTypeSystem.cpp
+3 -6
View File
@@ -383,14 +383,11 @@ bool DecompilerTypeSystem::tp_lca(TypeState* combined, const TypeState& add) {
}
int DecompilerTypeSystem::get_format_arg_count(const std::string& str) const {
// temporary hack, remove this.
if (str == "ERROR: dma tag has data in reserved bits ~X~%") {
return 0;
auto bad_it = bad_format_strings.find(str);
if (bad_it != bad_format_strings.end()) {
return bad_it->second;
}
if (str == "#<surface f0:~m f1:~f tf+:~f tf-:~f sf:~f tvv:~m") {
return 5;
}
int arg_count = 0;
for (size_t i = 0; i < str.length(); i++) {
if (str.at(i) == '~') {
decompiler/util/DecompilerTypeSystem.h
+2 -4
View File
@@ -1,5 +1,4 @@
#ifndef JAK_DECOMPILERTYPESYSTEM_H
#define JAK_DECOMPILERTYPESYSTEM_H
#pragma once
#include "common/type_system/TypeSystem.h"
#include "decompiler/Disasm/Register.h"
@@ -18,6 +17,7 @@ class DecompilerTypeSystem {
std::vector<std::string> symbol_add_order;
std::unordered_map<std::string, u64> type_flags;
std::unordered_map<std::string, std::string> type_parents;
std::unordered_map<std::string, int> bad_format_strings;
void add_symbol(const std::string& name) {
if (symbols.find(name) == symbols.end()) {
@@ -54,5 +54,3 @@ class DecompilerTypeSystem {
mutable goos::Reader m_reader;
};
} // namespace decompiler
#endif // JAK_DECOMPILERTYPESYSTEM_H
decompiler/util/data_decompile.cpp
+86 -37
View File
@@ -6,6 +6,9 @@
#include "common/goos/PrettyPrinter.h"
#include "common/util/math_util.h"
#include "common/log/log.h"
#include "decompiler/ObjectFile/LinkedObjectFile.h"
#include "decompiler/IR2/Form.h"
#include "decompiler/analysis/final_output.h"
namespace decompiler {
@@ -16,11 +19,12 @@ goos::Object decompile_at_label_with_hint(const LabelType& hint,
const DecompilerLabel& label,
const std::vector<DecompilerLabel>& labels,
const std::vector<std::vector<LinkedWord>>& words,
DecompilerTypeSystem& dts) {
DecompilerTypeSystem& dts,
const LinkedObjectFile* file) {
auto type = dts.parse_type_spec(hint.type_name);
if (!hint.array_size.has_value()) {
// if we don't have an array size, treat it as just a normal type.
return decompile_at_label(type, label, labels, words, dts.ts);
return decompile_at_label(type, label, labels, words, dts.ts, file);
}
if (type.base_type() == "pointer") {
@@ -67,7 +71,7 @@ goos::Object decompile_at_label_with_hint(const LabelType& hint,
fake_label.offset = label.offset + field_type_info->get_offset() + stride * elt;
fake_label.name = fmt::format("fake-label-{}-elt-{}", type.get_single_arg().print(), elt);
array_def.push_back(
decompile_at_label(type.get_single_arg(), fake_label, labels, words, dts.ts));
decompile_at_label(type.get_single_arg(), fake_label, labels, words, dts.ts, file));
}
return pretty_print::build_list(array_def);
}
@@ -121,12 +125,29 @@ std::optional<TypeSpec> get_type_of_label(const DecompilerLabel& label,
goos::Object decompile_at_label_guess_type(const DecompilerLabel& label,
const std::vector<DecompilerLabel>& labels,
const std::vector<std::vector<LinkedWord>>& words,
const TypeSystem& ts) {
const TypeSystem& ts,
const LinkedObjectFile* file) {
auto guessed_type = get_type_of_label(label, words);
if (!guessed_type.has_value()) {
throw std::runtime_error("Could not guess the type of " + label.name);
}
return decompile_at_label(*guessed_type, label, labels, words, ts);
return decompile_at_label(*guessed_type, label, labels, words, ts, file);
}
goos::Object decompile_function_at_label(const DecompilerLabel& label,
const LinkedObjectFile* file) {
if (file) {
auto other_func = file->try_get_function_at_label(label);
if (other_func) {
std::vector<goos::Object> inline_body;
other_func->ir2.top_form->inline_forms(inline_body, other_func->ir2.env);
auto result = pretty_print::build_list(
"lambda", get_arg_list_for_function(*other_func, other_func->ir2.env));
pretty_print::append(result, pretty_print::build_list(inline_body));
return result;
}
}
return pretty_print::to_symbol(fmt::format("<lambda at {}>", label.name));
}
/*!
@@ -137,21 +158,26 @@ goos::Object decompile_at_label(const TypeSpec& type,
const DecompilerLabel& label,
const std::vector<DecompilerLabel>& labels,
const std::vector<std::vector<LinkedWord>>& words,
const TypeSystem& ts) {
const TypeSystem& ts,
const LinkedObjectFile* file) {
if (type == TypeSpec("string")) {
return decompile_string_at_label(label, words);
}
if (ts.tc(TypeSpec("function"), type)) {
return decompile_function_at_label(label, file);
}
if (ts.tc(TypeSpec("array"), type)) {
return decompile_boxed_array(label, labels, words, ts);
return decompile_boxed_array(label, labels, words, ts, file);
}
if (ts.tc(TypeSpec("structure"), type)) {
return decompile_structure(type, label, labels, words, ts);
return decompile_structure(type, label, labels, words, ts, file);
}
if (type == TypeSpec("pair")) {
return decompile_pair(label, labels, words, ts, true);
return decompile_pair(label, labels, words, ts, true, file);
}
throw std::runtime_error("Unimplemented decompile_at_label for " + type.print());
@@ -256,7 +282,8 @@ goos::Object decompile_structure(const TypeSpec& type,
const DecompilerLabel& label,
const std::vector<DecompilerLabel>& labels,
const std::vector<std::vector<LinkedWord>>& words,
const TypeSystem& ts) {
const TypeSystem& ts,
const LinkedObjectFile* file) {
// first step, get type info and words
TypeSpec actual_type = type;
auto uncast_type_info = ts.lookup_type(actual_type);
@@ -265,11 +292,39 @@ goos::Object decompile_structure(const TypeSpec& type,
throw std::runtime_error(fmt::format("Type {} wasn't a structure type.", actual_type.print()));
}
bool is_basic = dynamic_cast<BasicType*>(uncast_type_info);
auto offset_location = label.offset - type_info->get_offset();
if (is_basic) {
const auto& word = words.at(label.target_segment).at((offset_location / 4));
if (word.kind != LinkedWord::TYPE_PTR) {
throw std::runtime_error("Basic does not start with type pointer");
}
if (word.symbol_name != actual_type.base_type()) {
// we can specify a more specific type.
auto got_type = TypeSpec(word.symbol_name);
if (ts.tc(actual_type, got_type)) {
actual_type = got_type;
type_info = dynamic_cast<StructureType*>(ts.lookup_type(actual_type));
if (!type_info) {
throw std::runtime_error(
fmt::format("Type-tag type {} wasn't a structure type.", actual_type.print()));
}
// try again with the right type. this resets back to decompile_at_label because we may
// want to get the specific function/string/etc implementations.
return decompile_at_label(actual_type, label, labels, words, ts, file);
} else {
throw std::runtime_error(fmt::format("Basic has the wrong type pointer, got {} expected {}",
word.symbol_name, actual_type.base_type()));
}
}
}
int word_count = (type_info->get_size_in_memory() + 3) / 4;
// check alignment
auto offset_location = label.offset - type_info->get_offset();
if (offset_location % 8) {
throw std::runtime_error(fmt::format(
"Tried to decompile a structure with type type {} (type offset {}) at label {}, but it has "
@@ -333,18 +388,8 @@ goos::Object decompile_structure(const TypeSpec& type,
}
if (word.symbol_name != actual_type.base_type()) {
// we can specify a more specific type.
auto got_type = TypeSpec(word.symbol_name);
if (ts.tc(actual_type, got_type)) {
actual_type = got_type;
if (actual_type == TypeSpec("string")) {
return decompile_string_at_label(label, words);
}
} else {
throw std::runtime_error(
fmt::format("Basic has the wrong type pointer, got {} expected {}", word.symbol_name,
actual_type.base_type()));
}
// the check above should have caught this.
assert(false);
}
for (int k = 0; k < 4; k++) {
field_status_per_byte.at(k) = HAS_DATA_READ;
@@ -420,7 +465,7 @@ goos::Object decompile_structure(const TypeSpec& type,
fake_label.offset = offset_location + field.offset() + field_type_info->get_offset();
fake_label.name = fmt::format("fake-label-{}-{}", actual_type.print(), field.name());
field_defs_out.emplace_back(
field.name(), decompile_at_label(field.type(), fake_label, labels, words, ts));
field.name(), decompile_at_label(field.type(), fake_label, labels, words, ts, file));
} else if (!field.is_dynamic() && field.is_array() && field.is_inline()) {
// it's an inline array. let's figure out the len and stride
auto len = field.array_size();
@@ -440,7 +485,8 @@ goos::Object decompile_structure(const TypeSpec& type,
offset_location + field.offset() + field_type_info->get_offset() + stride * elt;
fake_label.name =
fmt::format("fake-label-{}-{}-elt-{}", actual_type.print(), field.name(), elt);
array_def.push_back(decompile_at_label(field.type(), fake_label, labels, words, ts));
array_def.push_back(
decompile_at_label(field.type(), fake_label, labels, words, ts, file));
}
field_defs_out.emplace_back(field.name(), pretty_print::build_list(array_def));
} else if (!field.is_dynamic() && field.is_array() && !field.is_inline()) {
@@ -467,8 +513,8 @@ goos::Object decompile_structure(const TypeSpec& type,
auto& word = obj_words.at((field_start / 4) + elt);
if (word.kind == LinkedWord::PTR) {
array_def.push_back(
decompile_at_label(field.type(), labels.at(word.label_id), labels, words, ts));
array_def.push_back(decompile_at_label(field.type(), labels.at(word.label_id), labels,
words, ts, file));
} else if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) {
// do nothing, the default is zero?
array_def.push_back(pretty_print::to_symbol("0"));
@@ -501,7 +547,7 @@ goos::Object decompile_structure(const TypeSpec& type,
if (word.kind == LinkedWord::PTR) {
field_defs_out.emplace_back(
field.name(),
decompile_at_label(field.type(), labels.at(word.label_id), labels, words, ts));
decompile_at_label(field.type(), labels.at(word.label_id), labels, words, ts, file));
} else if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) {
// do nothing, the default is zero?
field_defs_out.emplace_back(field.name(), pretty_print::to_symbol("0"));
@@ -673,7 +719,8 @@ goos::Object decompile_value(const TypeSpec& type,
goos::Object decompile_boxed_array(const DecompilerLabel& label,
const std::vector<DecompilerLabel>& labels,
const std::vector<std::vector<LinkedWord>>& words,
const TypeSystem& ts) {
const TypeSystem& ts,
const LinkedObjectFile* file) {
TypeSpec content_type;
auto type_ptr_word_idx = (label.offset / 4) - 1;
if ((label.offset % 8) == 4) {
@@ -723,7 +770,7 @@ goos::Object decompile_boxed_array(const DecompilerLabel& label,
result.push_back(pretty_print::to_symbol("0"));
} else if (word.kind == LinkedWord::PTR) {
result.push_back(
decompile_at_label(content_type, labels.at(word.label_id), labels, words, ts));
decompile_at_label(content_type, labels.at(word.label_id), labels, words, ts, file));
} else if (word.kind == LinkedWord::SYM_PTR) {
result.push_back(pretty_print::to_symbol(fmt::format("'{}", word.symbol_name)));
} else {
@@ -765,7 +812,8 @@ namespace {
goos::Object decompile_pair_elt(const LinkedWord& word,
const std::vector<DecompilerLabel>& labels,
const std::vector<std::vector<LinkedWord>>& words,
const TypeSystem& ts) {
const TypeSystem& ts,
const LinkedObjectFile* file) {
if (word.kind == LinkedWord::PTR) {
auto& label = labels.at(word.label_id);
auto guessed_type = get_type_of_label(label, words);
@@ -774,10 +822,10 @@ goos::Object decompile_pair_elt(const LinkedWord& word,
}
if (guessed_type == TypeSpec("pair")) {
return decompile_pair(label, labels, words, ts, false);
return decompile_pair(label, labels, words, ts, false, file);
}
return decompile_at_label(*guessed_type, label, labels, words, ts);
return decompile_at_label(*guessed_type, label, labels, words, ts, file);
} else if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) {
// do nothing, the default is zero?
return pretty_print::to_symbol("0");
@@ -798,7 +846,8 @@ goos::Object decompile_pair(const DecompilerLabel& label,
const std::vector<DecompilerLabel>& labels,
const std::vector<std::vector<LinkedWord>>& words,
const TypeSystem& ts,
bool add_quote) {
bool add_quote,
const LinkedObjectFile* file) {
if ((label.offset % 8) != 2) {
if ((label.offset % 4) != 0) {
throw std::runtime_error(fmt::format("Invalid alignment for pair {}\n", label.offset % 16));
@@ -825,7 +874,7 @@ goos::Object decompile_pair(const DecompilerLabel& label,
if ((to_print.offset % 8) == 2) {
// continue
auto car_word = words.at(to_print.target_segment).at((to_print.offset - 2) / 4);
list_tokens.push_back(decompile_pair_elt(car_word, labels, words, ts));
list_tokens.push_back(decompile_pair_elt(car_word, labels, words, ts, file));
auto cdr_word = words.at(to_print.target_segment).at((to_print.offset + 2) / 4);
// if empty
@@ -847,7 +896,7 @@ goos::Object decompile_pair(const DecompilerLabel& label,
"because we "
"could not find a test case yet.");
list_tokens.push_back(pretty_print::to_symbol("."));
list_tokens.push_back(decompile_pair_elt(cdr_word, labels, words, ts));
list_tokens.push_back(decompile_pair_elt(cdr_word, labels, words, ts, file));
if (add_quote) {
return pretty_print::build_list("quote", pretty_print::build_list(list_tokens));
} else {
@@ -870,7 +919,7 @@ goos::Object decompile_pair(const DecompilerLabel& label,
"could not find a test case yet.");
list_tokens.push_back(pretty_print::to_symbol("."));
list_tokens.push_back(decompile_pair_elt(
words.at(to_print.target_segment).at(to_print.offset / 4), labels, words, ts));
words.at(to_print.target_segment).at(to_print.offset / 4), labels, words, ts, file));
if (add_quote) {
return pretty_print::build_list("quote", pretty_print::build_list(list_tokens));
} else {
decompiler/util/data_decompile.h
+13 -6
View File
@@ -10,6 +10,7 @@
#include "decompiler/util/DecompilerTypeSystem.h"
namespace decompiler {
class LinkedObjectFile;
std::optional<TypeSpec> get_type_of_label(const DecompilerLabel& label,
const std::vector<std::vector<LinkedWord>>& words);
@@ -19,30 +20,36 @@ goos::Object decompile_at_label(const TypeSpec& type,
const DecompilerLabel& label,
const std::vector<DecompilerLabel>& labels,
const std::vector<std::vector<LinkedWord>>& words,
const TypeSystem& ts);
const TypeSystem& ts,
const LinkedObjectFile* file);
goos::Object decompile_at_label_with_hint(const LabelType& hint,
const DecompilerLabel& label,
const std::vector<DecompilerLabel>& labels,
const std::vector<std::vector<LinkedWord>>& words,
DecompilerTypeSystem& dts);
DecompilerTypeSystem& dts,
const LinkedObjectFile* file);
goos::Object decompile_at_label_guess_type(const DecompilerLabel& label,
const std::vector<DecompilerLabel>& labels,
const std::vector<std::vector<LinkedWord>>& words,
const TypeSystem& ts);
const TypeSystem& ts,
const LinkedObjectFile* file);
goos::Object decompile_structure(const TypeSpec& actual_type,
const DecompilerLabel& label,
const std::vector<DecompilerLabel>& labels,
const std::vector<std::vector<LinkedWord>>& words,
const TypeSystem& ts);
const TypeSystem& ts,
const LinkedObjectFile* file);
goos::Object decompile_pair(const DecompilerLabel& label,
const std::vector<DecompilerLabel>& labels,
const std::vector<std::vector<LinkedWord>>& words,
const TypeSystem& ts,
bool add_quote);
bool add_quote,
const LinkedObjectFile* file);
goos::Object decompile_boxed_array(const DecompilerLabel& label,
const std::vector<DecompilerLabel>& labels,
const std::vector<std::vector<LinkedWord>>& words,
const TypeSystem& ts);
const TypeSystem& ts,
const LinkedObjectFile* file);
goos::Object decompile_value(const TypeSpec& type,
const std::vector<u8>& bytes,
const TypeSystem& ts);
docs/markdown/progress-notes/changelog.md
+2 -1
View File
@@ -170,4 +170,5 @@
- Added a `defun-recursive` to make it easier to define recursive functions
- Forward declared basics can be used in more places
- You can now set a field which has a forward declared structure or basic type
- `cdr` now returns an object of type `pair`.
- `cdr` now returns an object of type `pair`.
- `lambda`s can now be used inside of a static object definition.
goalc/compiler/StaticObject.cpp
+22 -1
View File
@@ -1,6 +1,7 @@
#include "third-party/fmt/core.h"
#include "StaticObject.h"
#include "common/goal_constants.h"
#include "goalc/compiler/Env.h"
namespace {
template <typename T>
@@ -102,12 +103,17 @@ void StaticStructure::generate_structure(emitter::ObjectGenerator* gen) {
}
for (auto& ptr : pointers) {
gen->link_static_pointer(rec, ptr.offset_in_this, ptr.dest->rec, ptr.offset_in_dest);
gen->link_static_pointer_to_data(rec, ptr.offset_in_this, ptr.dest->rec, ptr.offset_in_dest);
}
for (auto& type : types) {
gen->link_static_type_ptr(rec, type.offset, type.name);
}
for (auto& func : functions) {
gen->link_static_pointer_to_function(rec, func.offset_in_this,
gen->get_existing_function_record(func.func->idx_in_file));
}
}
void StaticStructure::generate(emitter::ObjectGenerator* gen) {
@@ -138,6 +144,13 @@ void StaticStructure::add_type_record(std::string name, int offset) {
types.push_back(srec);
}
void StaticStructure::add_function_record(const FunctionEnv* function, int offset) {
FunctionRecord rec;
rec.func = function;
rec.offset_in_this = offset;
functions.push_back(rec);
}
///////////////////
// StaticBasic
///////////////////
@@ -224,4 +237,12 @@ StaticResult StaticResult::make_type_ref(const std::string& type_name, int metho
result.m_method_count = method_count;
result.m_ts = TypeSpec("type");
return result;
}
StaticResult StaticResult::make_func_ref(const FunctionEnv* func, const TypeSpec& type) {
StaticResult result;
result.m_kind = Kind::FUNCTION_REFERENCE;
result.m_func = func;
result.m_ts = type;
return result;
}
goalc/compiler/StaticObject.h
+22
View File
@@ -7,6 +7,8 @@
#include "goalc/compiler/ConstantValue.h"
#include "common/util/BitUtils.h"
class FunctionEnv;
class StaticObject {
public:
virtual std::string print() const = 0;
@@ -60,13 +62,20 @@ class StaticStructure : public StaticObject {
int offset_in_dest = -1;
};
struct FunctionRecord {
const FunctionEnv* func = nullptr;
int offset_in_this = -1;
};
std::vector<SymbolRecord> types;
std::vector<SymbolRecord> symbols;
std::vector<PointerRecord> pointers;
std::vector<FunctionRecord> functions;
void add_symbol_record(std::string name, int offset);
void add_pointer_record(int offset_in_this, StaticStructure* dest, int offset_in_dest);
void add_type_record(std::string name, int offset);
void add_function_record(const FunctionEnv* function, int offset);
private:
int m_offset = 0;
@@ -87,6 +96,8 @@ class StaticString : public StaticBasic {
void generate(emitter::ObjectGenerator* gen) override;
};
class FunctionEnv;
/*!
* Represents a "static value". Like a reference to a static structure (including pair, string,
* basic), a symbol, or some constant (bitfield, integer, float). Cannot be used to store a static
@@ -102,12 +113,14 @@ class StaticResult {
static StaticResult make_constant_data(u64 data, const TypeSpec& ts);
static StaticResult make_symbol(const std::string& name);
static StaticResult make_type_ref(const std::string& type_name, int method_count);
static StaticResult make_func_ref(const FunctionEnv* func, const TypeSpec& ts);
const TypeSpec& typespec() const { return m_ts; }
bool is_reference() const { return m_kind == Kind::STRUCTURE_REFERENCE; }
bool is_constant_data() const { return m_kind == Kind::CONSTANT_DATA; }
bool is_symbol() const { return m_kind == Kind::SYMBOL; }
bool is_type() const { return m_kind == Kind::TYPE; }
bool is_func() const { return m_kind == Kind::FUNCTION_REFERENCE; }
StaticStructure* reference() const {
assert(is_reference());
@@ -125,6 +138,11 @@ class StaticResult {
return m_symbol;
}
const FunctionEnv* function() const {
assert(is_func());
return m_func;
}
int method_count() const {
assert(is_type());
return m_method_count;
@@ -147,6 +165,9 @@ class StaticResult {
// used for only STRUCTURE_REFERENCE
StaticStructure* m_struct = nullptr;
// used for only FUNCTION_REFERENCE
const FunctionEnv* m_func = nullptr;
// used for only constant data
std::optional<ConstantValue> m_constant_data;
@@ -161,6 +182,7 @@ class StaticResult {
CONSTANT_DATA,
SYMBOL,
TYPE,
FUNCTION_REFERENCE,
INVALID
} m_kind = Kind::INVALID;
};
goalc/compiler/compilation/Static.cpp
+9
View File
@@ -211,6 +211,8 @@ void Compiler::compile_static_structure_inline(const goos::Object& form,
field_info.type.print());
}
structure->add_type_record(sr.symbol_name(), field_offset);
} else if (sr.is_func()) {
structure->add_function_record(sr.function(), field_offset);
} else {
throw_compiler_error(form, "Unsupported field value {}.", field_value.print());
}
@@ -706,6 +708,13 @@ StaticResult Compiler::compile_static(const goos::Object& form_before_macro, Env
m_ts.forward_declare_type_method_count(type_name, method_count);
return StaticResult::make_type_ref(type_name, method_count);
} else if (first.is_symbol("lambda")) {
auto lambda = dynamic_cast<LambdaVal*>(compile_lambda(form, rest, env));
assert(lambda);
if (!lambda->func) {
throw_compiler_error(form, "Static lambda cannot be inline.");
}
return StaticResult::make_func_ref(lambda->func, lambda->type());
} else {
// maybe an enum
s64 int_out;
goalc/emitter/ObjectGenerator.cpp
+34 -7
View File
@@ -279,17 +279,32 @@ void ObjectGenerator::link_static_symbol_ptr(StaticRecord rec,
* Insert a pointer to other static data. This patching will happen during runtime linking.
* The source and destination must be in the same segment.
*/
void ObjectGenerator::link_static_pointer(const StaticRecord& source,
int source_offset,
const StaticRecord& dest,
int dest_offset) {
StaticPointerLink link;
void ObjectGenerator::link_static_pointer_to_data(const StaticRecord& source,
int source_offset,
const StaticRecord& dest,
int dest_offset) {
StaticDataPointerLink link;
link.source = source;
link.dest = dest;
link.offset_in_source = source_offset;
link.offset_in_dest = dest_offset;
assert(link.source.seg == link.dest.seg);
m_static_temp_ptr_links_by_seg.at(source.seg).push_back(link);
m_static_data_temp_ptr_links_by_seg.at(source.seg).push_back(link);
}
/*!
* Insert a pointer to a function in static data.
* The patching will happen during runtime linking.
*/
void ObjectGenerator::link_static_pointer_to_function(const StaticRecord& source,
int source_offset,
const FunctionRecord& target_func) {
StaticFunctionPointerLink link;
link.source = source;
link.offset_in_source = source_offset;
link.dest = target_func;
assert(target_func.seg == source.seg);
m_static_function_temp_ptr_links_by_seg.at(source.seg).push_back(link);
}
void ObjectGenerator::link_instruction_static(const InstructionRecord& instr,
@@ -341,7 +356,7 @@ void ObjectGenerator::handle_temp_static_sym_links(int seg) {
* m_static_temp_ptr_links_by_seg -> m_pointer_links_by_seg
*/
void ObjectGenerator::handle_temp_static_ptr_links(int seg) {
for (const auto& link : m_static_temp_ptr_links_by_seg.at(seg)) {
for (const auto& link : m_static_data_temp_ptr_links_by_seg.at(seg)) {
const auto& source_object = m_static_data_by_seg.at(seg).at(link.source.static_id);
const auto& dest_object = m_static_data_by_seg.at(seg).at(link.dest.static_id);
PointerLink result_link;
@@ -350,6 +365,18 @@ void ObjectGenerator::handle_temp_static_ptr_links(int seg) {
result_link.dest = dest_object.location + link.offset_in_dest;
m_pointer_links_by_seg.at(seg).push_back(result_link);
}
for (const auto& link : m_static_function_temp_ptr_links_by_seg.at(seg)) {
const auto& source_object = m_static_data_by_seg.at(seg).at(link.source.static_id);
const auto& dest_function = m_function_data_by_seg.at(seg).at(link.dest.func_id);
assert(link.dest.seg == seg);
int loc = dest_function.instruction_to_byte_in_data.at(0);
PointerLink result_link;
result_link.segment = seg;
result_link.source = source_object.location + link.offset_in_source;
result_link.dest = loc;
m_pointer_links_by_seg.at(seg).push_back(result_link);
}
}
/*!
goalc/emitter/ObjectGenerator.h
+16 -6
View File
@@ -78,10 +78,13 @@ class ObjectGenerator {
void link_instruction_symbol_mem(const InstructionRecord& rec, const std::string& name);
void link_instruction_symbol_ptr(const InstructionRecord& rec, const std::string& name);
void link_static_symbol_ptr(StaticRecord rec, int offset, const std::string& name);
void link_static_pointer(const StaticRecord& source,
int source_offset,
const StaticRecord& dest,
int dest_offset);
void link_static_pointer_to_data(const StaticRecord& source,
int source_offset,
const StaticRecord& dest,
int dest_offset);
void link_static_pointer_to_function(const StaticRecord& source,
int source_offset,
const FunctionRecord& target_func);
void link_instruction_static(const InstructionRecord& instr,
const StaticRecord& target_static,
int offset);
@@ -145,13 +148,19 @@ class ObjectGenerator {
int offset = -1;
};
struct StaticPointerLink {
struct StaticDataPointerLink {
StaticRecord source;
int offset_in_source = -1;
StaticRecord dest;
int offset_in_dest = -1;
};
struct StaticFunctionPointerLink {
StaticRecord source;
int offset_in_source = -1;
FunctionRecord dest;
};
struct SymbolInstrLink {
InstructionRecord rec;
bool is_mem_access = false;
@@ -205,7 +214,8 @@ class ObjectGenerator {
seg_vector<JumpLink> m_jump_temp_links_by_seg;
seg_map<SymbolInstrLink> m_symbol_instr_temp_links_by_seg;
seg_map<StaticSymbolLink> m_static_sym_temp_links_by_seg;
seg_vector<StaticPointerLink> m_static_temp_ptr_links_by_seg;
seg_vector<StaticDataPointerLink> m_static_data_temp_ptr_links_by_seg;
seg_vector<StaticFunctionPointerLink> m_static_function_temp_ptr_links_by_seg;
seg_vector<RipFuncLink> m_rip_func_temp_links_by_seg;
seg_vector<RipDataLink> m_rip_data_temp_links_by_seg;
test/decompiler/test_DataParser.cpp
+10 -10
View File
@@ -124,8 +124,8 @@ TEST_F(DataDecompTest, String) {
" .word 0x0\n"
" .word 0x0\n";
auto parsed = parse_data(input);
auto decomp =
decompile_at_label_guess_type(parsed.label("L62"), parsed.labels, {parsed.words}, dts->ts);
auto decomp = decompile_at_label_guess_type(parsed.label("L62"), parsed.labels, {parsed.words},
dts->ts, nullptr);
EXPECT_EQ(decomp.print(), "\"finalboss-fight\"");
}
@@ -147,7 +147,7 @@ TEST_F(DataDecompTest, SimpleStructure) {
" .word 0x6c63\n";
auto parsed = parse_data(input);
auto decomp = decompile_at_label(TypeSpec("vif-disasm-element"), parsed.label("L217"),
parsed.labels, {parsed.words}, dts->ts);
parsed.labels, {parsed.words}, dts->ts, nullptr);
check_forms_equal(decomp.print(),
"(new 'static 'vif-disasm-element :mask #x7f :tag (vif-cmd-32 stcycl) :print "
"#x2 :string1 \"stcycl\")");
@@ -207,8 +207,8 @@ TEST_F(DataDecompTest, VifDisasmArray) {
" .word 0x706f6e\n"
" .word 0x0";
auto parsed = parse_data(input);
auto decomp =
decompile_at_label_guess_type(parsed.label("L148"), parsed.labels, {parsed.words}, dts->ts);
auto decomp = decompile_at_label_guess_type(parsed.label("L148"), parsed.labels, {parsed.words},
dts->ts, nullptr);
check_forms_equal(decomp.print(),
"(new 'static 'boxed-array :type\n"
" vif-disasm-element\n"
@@ -288,8 +288,8 @@ TEST_F(DataDecompTest, ContinuePoint) {
" .word 0x74732d73\n"
" .word 0x747261";
auto parsed = parse_data(input);
auto decomp =
decompile_at_label_guess_type(parsed.label("L63"), parsed.labels, {parsed.words}, dts->ts);
auto decomp = decompile_at_label_guess_type(parsed.label("L63"), parsed.labels, {parsed.words},
dts->ts, nullptr);
check_forms_equal(decomp.print(),
"(new 'static 'continue-point\n"
" :name \"finalboss-start\"\n"
@@ -343,7 +343,7 @@ TEST_F(DataDecompTest, FloatArray) {
" .word 0x3f800000\n\n";
auto parsed = parse_data(input);
auto decomp = decompile_at_label_with_hint({"(pointer float)", true, 7}, parsed.label("L63"),
parsed.labels, {parsed.words}, *dts);
parsed.labels, {parsed.words}, *dts, nullptr);
check_forms_equal(decomp.print(),
"(new 'static 'array float 7\n"
"1.0 0.0 1.0 0.0 1.0 0.0 1.0)");
@@ -380,8 +380,8 @@ TEST_F(DataDecompTest, KernelContext) {
" .word 0x0\n"
" .symbol #t\n";
auto parsed = parse_data(input);
auto decomp =
decompile_at_label_guess_type(parsed.label("L345"), parsed.labels, {parsed.words}, dts->ts);
auto decomp = decompile_at_label_guess_type(parsed.label("L345"), parsed.labels, {parsed.words},
dts->ts, nullptr);
check_forms_equal(decomp.print(),
"(new 'static 'kernel-context\n"
" :prevent-from-run (process-mask execute sleep)\n"
test/goalc/source_templates/with_game/test-static-lambda.gc
+22
View File
@@ -0,0 +1,22 @@
(deftype type-with-func (basic)
((add-func (function int int int))
(sub-func (function int int int))
)
)
(define *type-with-func*
(new 'static 'type-with-func
:add-func (lambda ((x int) (y int))
(+ x y))
:sub-func (lambda ((x int) (y int))
(- x y))
)
)
(defun this-is-a-function ()
"blah blah")
(format #t "Add: ~d sub: ~d~%"
((-> *type-with-func* add-func) 10 20)
((-> *type-with-func* sub-func) 10 20))
test/goalc/test_with_game.cpp
+4
View File
@@ -771,6 +771,10 @@ TEST_F(WithGameTests, SoundName) {
"0\n"});
}
TEST_F(WithGameTests, StaticLambda) {
runner.run_static_test(env, testCategory, "test-static-lambda.gc", {"Add: 30 sub: -10\n0\n"});
}
TEST(TypeConsistency, TypeConsistency) {
Compiler compiler;
compiler.enable_throw_on_redefines();