mirror of
https://github.com/open-goal/jak-project
synced 2026-09-02 10:11:44 -04:00
all-types: Improve Jak 2's all-types (#1728)
* all-types: improve all-types generation * all-types: re-generate all-types * tests: remove the test reporting feature the format indeed doesn't work, and all current actions require too many permissions for forked PRs. I'll make my own eventually that works properly (use the new markdown feature) * all-types: put the states in the method table instead * all-types: replace all `*time*...uint64` fields with `time-frame` type * all-types: address feedback
This commit is contained in:
Tyler Wilding
@@ -56,14 +56,6 @@ jobs:
|
||||
GTEST_OUTPUT: "xml:opengoal-test-report.xml"
|
||||
run: ./test.sh
|
||||
|
||||
- name: Generate Test Report
|
||||
uses: dorny/test-reporter@v1
|
||||
if: success() || failure() # run this step even if previous step failed
|
||||
with:
|
||||
name: Linux Clang - Test Report
|
||||
path: ${{ github.workspace }}/**/opengoal-test-report.xml
|
||||
reporter: jest-junit
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
|
||||
@@ -58,14 +58,6 @@ jobs:
|
||||
GTEST_OUTPUT: "xml:opengoal-test-report.xml"
|
||||
run: ninja goalc-test_coverage -w dupbuild=warn
|
||||
|
||||
- name: Generate Test Report
|
||||
uses: dorny/test-reporter@v1
|
||||
if: success() || failure() # run this step even if previous step failed
|
||||
with:
|
||||
name: Linux GCC - Test Report
|
||||
path: ${{ github.workspace }}/**/opengoal-test-report.xml
|
||||
reporter: jest-junit
|
||||
|
||||
- name: Submit Coverage Report to Codacy
|
||||
uses: codacy/codacy-coverage-reporter-action@v1
|
||||
continue-on-error: true
|
||||
|
||||
@@ -52,17 +52,3 @@ jobs:
|
||||
GTEST_OUTPUT: "xml:opengoal-test-report.xml"
|
||||
run: ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=1 --gtest_filter="-*MANUAL_TEST*"
|
||||
|
||||
- name: Generate Test Report
|
||||
uses: dorny/test-reporter@v1
|
||||
if: success() || failure() # run this step even if previous step failed
|
||||
with:
|
||||
name: Windows Clang - Test Report
|
||||
path: ./opengoal-test-report.xml
|
||||
reporter: jest-junit
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: opengoal-windows-${{ inputs.cachePrefix }}
|
||||
if-no-files-found: error
|
||||
path: ./build/bin
|
||||
|
||||
@@ -54,10 +54,3 @@ jobs:
|
||||
run: |
|
||||
./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=1 --gtest_filter="-*MANUAL_TEST*"
|
||||
|
||||
- name: Generate Test Report
|
||||
uses: dorny/test-reporter@v1
|
||||
if: success() || failure() # run this step even if previous step failed
|
||||
with:
|
||||
name: Windows MSVC - Test Report
|
||||
path: ./opengoal-test-report.xml
|
||||
reporter: jest-junit
|
||||
|
||||
@@ -9,6 +9,8 @@ dotenv:
|
||||
|
||||
tasks:
|
||||
# SETTINGS / CONFIGURATION
|
||||
settings:
|
||||
- 'python ./scripts/tasks/update-env.py --info'
|
||||
set-game-jak1:
|
||||
- 'python ./scripts/tasks/update-env.py --game jak1'
|
||||
set-game-jak2:
|
||||
|
||||
@@ -192,6 +192,18 @@ class ObjectFileDB {
|
||||
void ir2_do_segment_analysis_phase2(int seg, const Config& config, ObjectFileData& data);
|
||||
void ir2_setup_labels(const Config& config, ObjectFileData& data);
|
||||
void ir2_run_mips2c(const Config& config, ObjectFileData& data);
|
||||
struct PerObjectAllTypeInfo {
|
||||
std::string object_name;
|
||||
std::unordered_set<std::string> already_seen_symbols;
|
||||
|
||||
// type-name : { method id : state name }
|
||||
std::unordered_map<std::string, std::unordered_map<int, std::string>> state_methods;
|
||||
// symbol-name : type-name
|
||||
std::unordered_map<std::string, std::string> symbol_types;
|
||||
|
||||
std::vector<std::string> type_defs;
|
||||
std::string symbol_defs;
|
||||
};
|
||||
void ir2_analyze_all_types(const fs::path& output_file,
|
||||
const std::optional<std::string>& previous_game_types,
|
||||
const std::unordered_set<std::string>& bad_types);
|
||||
|
||||
@@ -301,40 +301,47 @@ void ObjectFileDB::ir2_top_level_pass(const Config& config) {
|
||||
void ObjectFileDB::ir2_analyze_all_types(const fs::path& output_file,
|
||||
const std::optional<std::string>& previous_game_types,
|
||||
const std::unordered_set<std::string>& bad_types) {
|
||||
struct PerObject {
|
||||
std::string object_name;
|
||||
std::vector<std::string> type_defs;
|
||||
std::string symbol_defs;
|
||||
};
|
||||
|
||||
std::vector<PerObject> per_object;
|
||||
std::vector<PerObjectAllTypeInfo> per_object;
|
||||
|
||||
DecompilerTypeSystem previous_game_ts(GameVersion::Jak1); // version here doesn't matter.
|
||||
if (previous_game_types) {
|
||||
previous_game_ts.parse_type_defs({*previous_game_types});
|
||||
}
|
||||
|
||||
std::unordered_set<std::string> already_seen;
|
||||
TypeInspectorCache ti_cache;
|
||||
|
||||
for_each_obj([&](ObjectFileData& data) {
|
||||
if (data.obj_version != 3) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto& object_result = per_object.emplace_back();
|
||||
object_result.object_name = data.to_unique_name();
|
||||
|
||||
// Go through the top-level segment first to identify the type names associated with each symbol
|
||||
// def
|
||||
for_each_function_in_seg_in_obj(TOP_LEVEL_SEGMENT, data, [&](Function& f) {
|
||||
inspect_top_level_for_metadata(f, data.linked_data, dts, previous_game_ts, object_result);
|
||||
});
|
||||
|
||||
// Handle the top level last, which is fine as all symbol_defs are always written after typedefs
|
||||
for_each_function_def_order_in_obj(data, [&](Function& f, int seg) {
|
||||
if (seg == TOP_LEVEL_SEGMENT) {
|
||||
object_result.symbol_defs += inspect_top_level_symbol_defines(
|
||||
already_seen, f, data.linked_data, dts, previous_game_ts);
|
||||
} else {
|
||||
if (seg != TOP_LEVEL_SEGMENT) {
|
||||
if (f.is_inspect_method && bad_types.find(f.guessed_name.type_name) == bad_types.end()) {
|
||||
object_result.type_defs.push_back(inspect_inspect_method(
|
||||
f, f.guessed_name.type_name, dts, data.linked_data, previous_game_ts.ts, ti_cache));
|
||||
object_result.type_defs.push_back(
|
||||
inspect_inspect_method(f, f.guessed_name.type_name, dts, data.linked_data,
|
||||
previous_game_ts, ti_cache, object_result));
|
||||
} else {
|
||||
// no inspect methods
|
||||
// - can we solve custom print methods in a generic way? ie `entity-links`
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
for_each_function_in_seg_in_obj(TOP_LEVEL_SEGMENT, data, [&](Function& f) {
|
||||
object_result.symbol_defs += inspect_top_level_symbol_defines(
|
||||
f, data.linked_data, dts, previous_game_ts, object_result);
|
||||
});
|
||||
});
|
||||
|
||||
std::string result;
|
||||
|
||||
@@ -526,8 +526,7 @@ int get_start_idx(Function& function,
|
||||
if (!type_name_str) {
|
||||
fmt::print("[iim] op7 bad in {}: {} (bad string)\n", aos.ops.at(op_idx)->to_string(env),
|
||||
function.name());
|
||||
}
|
||||
if (type_name_str != "[~8x] ~A~%") {
|
||||
} else if (type_name_str != "[~8x] ~A~%") {
|
||||
fmt::print("[iim] op7 bad in {}: {} (bad string: {})\n", aos.ops.at(op_idx)->to_string(env),
|
||||
function.name(), *type_name_str);
|
||||
}
|
||||
@@ -907,8 +906,9 @@ std::string inspect_inspect_method(Function& inspect_method,
|
||||
const std::string& type_name,
|
||||
DecompilerTypeSystem& dts,
|
||||
LinkedObjectFile& file,
|
||||
TypeSystem& previous_game_ts,
|
||||
TypeInspectorCache& ti_cache) {
|
||||
DecompilerTypeSystem& previous_game_ts,
|
||||
TypeInspectorCache& ti_cache,
|
||||
ObjectFileDB::PerObjectAllTypeInfo& object_file_meta) {
|
||||
fmt::print(" iim: {}\n", inspect_method.name());
|
||||
TypeInspectorResult result;
|
||||
ASSERT(type_name == inspect_method.guessed_name.type_name);
|
||||
@@ -920,7 +920,17 @@ std::string inspect_inspect_method(Function& inspect_method,
|
||||
result.flags = flags.flag;
|
||||
result.type_size = flags.size;
|
||||
result.type_method_count = flags.methods;
|
||||
result.type_heap_base = flags.heap_base;
|
||||
|
||||
// Only set heap-base if it's different from the automatic one
|
||||
// A child (or child of a child) of process ALWAYS has heap-base set.
|
||||
if (flags.heap_base > 0) {
|
||||
auto process_type = dts.ts.get_type_of_type<BasicType>("process");
|
||||
auto auto_hb = (flags.size - process_type->size() + 0xf) & ~0xf;
|
||||
|
||||
if (auto_hb != flags.heap_base) {
|
||||
result.type_heap_base = std::make_optional(flags.heap_base);
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
TypeFlags parent_flags;
|
||||
@@ -942,25 +952,27 @@ std::string inspect_inspect_method(Function& inspect_method,
|
||||
idx = get_start_idx_process(inspect_method, result.parent_type_name, inspect_method.ir2.env);
|
||||
}
|
||||
StructureType* old_game_type = nullptr;
|
||||
if (previous_game_ts.fully_defined_type_exists(type_name)) {
|
||||
old_game_type = dynamic_cast<StructureType*>(previous_game_ts.lookup_type(type_name));
|
||||
if (previous_game_ts.ts.fully_defined_type_exists(type_name)) {
|
||||
old_game_type = dynamic_cast<StructureType*>(previous_game_ts.ts.lookup_type(type_name));
|
||||
}
|
||||
if (idx <= 0) {
|
||||
// can't get any field...
|
||||
result.warnings += "Failed to read fields. ";
|
||||
result.warnings += "Failed to read fields.";
|
||||
idx = -2;
|
||||
ti_cache.previous_results[type_name] = result;
|
||||
return result.print_as_deftype(old_game_type, ti_cache.previous_results);
|
||||
return result.print_as_deftype(old_game_type, ti_cache.previous_results, previous_game_ts,
|
||||
object_file_meta);
|
||||
}
|
||||
while (idx < int(inspect_method.ir2.atomic_ops->ops.size()) - 2 && idx != -1) {
|
||||
idx = detect(idx, inspect_method, file, &result);
|
||||
}
|
||||
|
||||
if (idx == -1) {
|
||||
result.warnings += "Failed to read some fields. ";
|
||||
result.warnings += "Failed to read some fields.";
|
||||
}
|
||||
ti_cache.previous_results[type_name] = result;
|
||||
return result.print_as_deftype(old_game_type, ti_cache.previous_results);
|
||||
return result.print_as_deftype(old_game_type, ti_cache.previous_results, previous_game_ts,
|
||||
object_file_meta);
|
||||
}
|
||||
|
||||
std::string old_method_string(const MethodInfo& info) {
|
||||
@@ -1013,7 +1025,9 @@ bool allow_guess(const Field& field) {
|
||||
*/
|
||||
std::string TypeInspectorResult::print_as_deftype(
|
||||
StructureType* old_game_type,
|
||||
std::unordered_map<std::string, TypeInspectorResult>& previous_results) {
|
||||
std::unordered_map<std::string, TypeInspectorResult>& previous_results,
|
||||
DecompilerTypeSystem& previous_game_ts,
|
||||
ObjectFileDB::PerObjectAllTypeInfo& object_file_meta) {
|
||||
std::string result;
|
||||
|
||||
result += "#|\n";
|
||||
@@ -1148,6 +1162,9 @@ std::string TypeInspectorResult::print_as_deftype(
|
||||
|
||||
result.append(fmt::format(" :method-count-assert {}\n", type_method_count));
|
||||
result.append(fmt::format(" :size-assert #x{:x}\n", type_size));
|
||||
if (type_heap_base.has_value()) {
|
||||
result.append(fmt::format(" :heap-base #x{:x}\n", type_heap_base.value()));
|
||||
}
|
||||
result.append(fmt::format(" :flag-assert #x{:x}\n ", flags));
|
||||
if (!warnings.empty()) {
|
||||
result.append(";; ");
|
||||
@@ -1155,6 +1172,11 @@ std::string TypeInspectorResult::print_as_deftype(
|
||||
result.append("\n ");
|
||||
}
|
||||
|
||||
std::unordered_map<int, std::string> method_states = {};
|
||||
if (object_file_meta.state_methods.count(type_name) != 0) {
|
||||
method_states = object_file_meta.state_methods.at(type_name);
|
||||
}
|
||||
|
||||
if (type_method_count > 9) {
|
||||
result.append("(:methods\n ");
|
||||
MethodInfo old_new_method;
|
||||
@@ -1163,7 +1185,12 @@ std::string TypeInspectorResult::print_as_deftype(
|
||||
result.append("\n ");
|
||||
}
|
||||
for (int i = parent_method_count; i < type_method_count; i++) {
|
||||
result.append(fmt::format("(dummy-{} () none {})", i, i));
|
||||
// If the method is actually a state, skip it!
|
||||
if (method_states.count(i) != 0) {
|
||||
result.append(fmt::format("({} () _type_ :state {})", method_states.at(i), i));
|
||||
} else {
|
||||
result.append(fmt::format("({}-method-{} () none {})", type_name, i, i));
|
||||
}
|
||||
if (old_game_type) {
|
||||
MethodInfo info;
|
||||
if (old_game_type->get_my_method(i, &info)) {
|
||||
@@ -1174,17 +1201,165 @@ std::string TypeInspectorResult::print_as_deftype(
|
||||
}
|
||||
result.append(")\n ");
|
||||
}
|
||||
|
||||
// Print out states if we have em
|
||||
// - Could probably assume the process name comes first and associate it with the right type
|
||||
// but that may or may not be risky so, edit the types yourself...
|
||||
// if (method_states.size() > 0) {
|
||||
// result.append("(:states\n ");
|
||||
// for (const auto& [id, name] : method_states) {
|
||||
// result.append(name);
|
||||
// // Append old symbol def if we have it
|
||||
// auto it = previous_game_ts.symbol_types.find(name);
|
||||
// if (it != previous_game_ts.symbol_types.end()) {
|
||||
// result.append(fmt::format(" ;; {}", it->second.print()));
|
||||
// }
|
||||
// // Add symbol name to `already_seen_symbols`
|
||||
// object_file_meta.already_seen_symbols.insert(name);
|
||||
// result.append("\n ");
|
||||
// }
|
||||
// result.append(")\n ");
|
||||
//}
|
||||
|
||||
result.append(")\n");
|
||||
result += "|#\n";
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string inspect_top_level_symbol_defines(std::unordered_set<std::string>& already_seen,
|
||||
Function& top_level,
|
||||
std::string get_regex_match(std::string form, std::regex regex) {
|
||||
std::smatch matches;
|
||||
if (std::regex_search(form, matches, regex)) {
|
||||
if (matches.size() == 2) {
|
||||
return matches[1];
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
std::string get_state_symbol_name(LinkedObjectFile& file, std::string label_name) {
|
||||
try {
|
||||
auto& label = file.get_label_by_name(label_name);
|
||||
auto& label_words = file.words_by_seg.at(label.target_segment);
|
||||
int start_word_idx = (label.offset / 4) - 1;
|
||||
|
||||
auto& first_word = label_words.at(start_word_idx);
|
||||
if (first_word.kind() != LinkedWord::TYPE_PTR || first_word.symbol_name() != "state") {
|
||||
return "";
|
||||
}
|
||||
|
||||
auto& name_word = label_words.at(start_word_idx + 1);
|
||||
if (name_word.kind() != LinkedWord::SYM_PTR) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return name_word.symbol_name();
|
||||
} catch (std::exception& e) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
std::string get_label_type_name(LinkedObjectFile& file, std::string label_name) {
|
||||
try {
|
||||
auto& label = file.get_label_by_name(label_name);
|
||||
auto& label_words = file.words_by_seg.at(label.target_segment);
|
||||
int start_word_idx = (label.offset / 4) - 1;
|
||||
|
||||
auto& first_word = label_words.at(start_word_idx);
|
||||
if (first_word.kind() != LinkedWord::TYPE_PTR) {
|
||||
return "";
|
||||
}
|
||||
return first_word.symbol_name();
|
||||
} catch (std::exception& e) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
std::string inspect_top_level_for_metadata(Function& top_level,
|
||||
LinkedObjectFile& file,
|
||||
DecompilerTypeSystem& dts,
|
||||
DecompilerTypeSystem& previous_game_ts,
|
||||
ObjectFileDB::PerObjectAllTypeInfo& objectFile) {
|
||||
// State as a method:
|
||||
/*
|
||||
lui v1, L267 ;; [ 77] (set! gp-0 L267) [] -> [gp: <uninitialized> ]
|
||||
ori gp, v1, L267
|
||||
lw t9, method-set!(s7) ;; [ 78] (set! t9-12 method-set!) [] -> [t9: <uninitialized> ]
|
||||
lw a0, com-airlock(s7) ;; [ 79] (set! a0-12 com-airlock) [] -> [a0: <uninitialized> ]
|
||||
addiu a1, r0, 21 ;; [ 80] (set! a1-10 21) [] -> [a1: <uninitialized> ]
|
||||
or a2, gp, r0 ;; [ 81] (set! a2-10 gp-0) [gp: <uninitialized> ] -> [a2:
|
||||
<uninitialized> ]
|
||||
*/
|
||||
// State as symbol:
|
||||
/*
|
||||
lui v1, L753 ;; [354] (set! v1-38 L753) [] -> [v1: <uninitialized> ]
|
||||
ori v1, v1, L753
|
||||
sw v1, target-roll(s7) ;; [355] (s.w! target-roll v1-38) [v1: <uninitialized> ] -> []
|
||||
*/
|
||||
if (!top_level.ir2.atomic_ops) {
|
||||
return "";
|
||||
}
|
||||
std::string result;
|
||||
std::string last_seen_label = "";
|
||||
// TODO - safely increment op number
|
||||
for (int i = 0; i < top_level.ir2.atomic_ops->ops.size(); i++) {
|
||||
const auto& aop = top_level.ir2.atomic_ops->ops.at(i);
|
||||
const std::string as_str = aop.get()->to_string(top_level.ir2.env);
|
||||
|
||||
// Keep track of the last seen label so we can easily reference it if a later operation uses it
|
||||
auto label_match = get_regex_match(as_str, std::regex("\\(set!\\s[^\\s]*\\s(L.*)\\)"));
|
||||
if (!label_match.empty()) {
|
||||
last_seen_label = label_match;
|
||||
|
||||
// Check if the next operation is storing the label
|
||||
std::string curr_op =
|
||||
top_level.ir2.atomic_ops->ops.at(i + 1).get()->to_string(top_level.ir2.env);
|
||||
auto symbol_name = get_regex_match(curr_op, std::regex("\\(s\\.w!\\s([^\\(\\)\\s]*)\\s"));
|
||||
if (symbol_name.empty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check that the label is a state
|
||||
auto label_type_name = get_label_type_name(file, last_seen_label);
|
||||
if (label_type_name.empty()) {
|
||||
continue;
|
||||
}
|
||||
objectFile.symbol_types[symbol_name] = label_type_name;
|
||||
}
|
||||
|
||||
if (as_str.find("method-set!") != std::string::npos) {
|
||||
// The next operation should have the type name
|
||||
i++;
|
||||
std::string curr_op = top_level.ir2.atomic_ops->ops.at(i).get()->to_string(top_level.ir2.env);
|
||||
auto type_match = get_regex_match(curr_op, std::regex("\\(set!\\s[^\\s]*\\s(.*)\\)"));
|
||||
if (type_match.empty()) {
|
||||
continue;
|
||||
}
|
||||
i++;
|
||||
// The next operation should have the method id
|
||||
curr_op = top_level.ir2.atomic_ops->ops.at(i).get()->to_string(top_level.ir2.env);
|
||||
auto method_id_match = get_regex_match(curr_op, std::regex("\\(set!\\s[^\\s]*\\s(\\d*)\\)"));
|
||||
if (method_id_match.empty()) {
|
||||
continue;
|
||||
}
|
||||
int method_id = std::stoi(method_id_match);
|
||||
|
||||
// Now check the last seen label to see if it's a state
|
||||
auto state_name = get_state_symbol_name(file, last_seen_label);
|
||||
if (state_name.empty()) {
|
||||
continue;
|
||||
}
|
||||
objectFile.state_methods[type_match][method_id] = state_name;
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
std::string inspect_top_level_symbol_defines(Function& top_level,
|
||||
LinkedObjectFile& /*file*/,
|
||||
DecompilerTypeSystem& dts,
|
||||
DecompilerTypeSystem& previous_game_ts) {
|
||||
DecompilerTypeSystem& previous_game_ts,
|
||||
ObjectFileDB::PerObjectAllTypeInfo& object_file_meta) {
|
||||
if (!top_level.ir2.atomic_ops) {
|
||||
return {};
|
||||
}
|
||||
@@ -1194,12 +1369,18 @@ std::string inspect_top_level_symbol_defines(std::unordered_set<std::string>& al
|
||||
if (as_store && as_store->addr().kind() == SimpleExpression::Kind::IDENTITY &&
|
||||
as_store->addr().get_arg(0).is_sym_val()) {
|
||||
auto& sym_name = as_store->addr().get_arg(0).get_str();
|
||||
if (already_seen.find(sym_name) == already_seen.end()) {
|
||||
already_seen.insert(sym_name);
|
||||
if (object_file_meta.already_seen_symbols.find(sym_name) ==
|
||||
object_file_meta.already_seen_symbols.end()) {
|
||||
object_file_meta.already_seen_symbols.insert(sym_name);
|
||||
if (dts.ts.partially_defined_type_exists(sym_name)) {
|
||||
continue;
|
||||
}
|
||||
result += fmt::format(";; (define-extern {} object)", sym_name);
|
||||
std::string type_name = "object";
|
||||
// Look to see if we know the type name
|
||||
if (object_file_meta.symbol_types.count(sym_name) != 0) {
|
||||
type_name = object_file_meta.symbol_types.at(sym_name);
|
||||
}
|
||||
result += fmt::format(";; (define-extern {} {})", sym_name, type_name);
|
||||
auto it = previous_game_ts.symbol_types.find(sym_name);
|
||||
if (it != previous_game_ts.symbol_types.end()) {
|
||||
result += fmt::format(" ;; {}", it->second.print());
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <decompiler/ObjectFile/ObjectFileDB.h>
|
||||
#include "decompiler/Function/Function.h"
|
||||
#include "decompiler/util/DecompilerTypeSystem.h"
|
||||
|
||||
@@ -13,7 +14,7 @@ struct TypeInspectorResult {
|
||||
int type_size = -1;
|
||||
int type_method_count = -1;
|
||||
int parent_method_count = 9;
|
||||
int type_heap_base = -1;
|
||||
std::optional<int> type_heap_base = {};
|
||||
|
||||
std::string warnings;
|
||||
std::vector<Field> fields_of_type;
|
||||
@@ -22,11 +23,13 @@ struct TypeInspectorResult {
|
||||
|
||||
std::string type_name;
|
||||
std::string parent_type_name;
|
||||
u64 flags = 0;
|
||||
u64 flags;
|
||||
|
||||
std::string print_as_deftype(
|
||||
StructureType* old_game_type,
|
||||
std::unordered_map<std::string, TypeInspectorResult>& previous_results);
|
||||
std::unordered_map<std::string, TypeInspectorResult>& previous_results,
|
||||
DecompilerTypeSystem& previous_game_ts,
|
||||
ObjectFileDB::PerObjectAllTypeInfo& object_file_meta);
|
||||
};
|
||||
|
||||
struct TypeInspectorCache {
|
||||
@@ -37,13 +40,20 @@ std::string inspect_inspect_method(Function& inspect_method,
|
||||
const std::string& type_name,
|
||||
DecompilerTypeSystem& dts,
|
||||
LinkedObjectFile& file,
|
||||
TypeSystem& previous_game_ts,
|
||||
TypeInspectorCache& ti_cache);
|
||||
DecompilerTypeSystem& previous_game_ts,
|
||||
TypeInspectorCache& ti_cache,
|
||||
ObjectFileDB::PerObjectAllTypeInfo& object_file_meta);
|
||||
|
||||
std::string inspect_top_level_symbol_defines(std::unordered_set<std::string>& already_seen,
|
||||
Function& top_level,
|
||||
std::string inspect_top_level_for_metadata(Function& top_level,
|
||||
LinkedObjectFile& file,
|
||||
DecompilerTypeSystem& dts,
|
||||
DecompilerTypeSystem& previous_game_ts,
|
||||
ObjectFileDB::PerObjectAllTypeInfo& object_file_meta);
|
||||
|
||||
std::string inspect_top_level_symbol_defines(Function& top_level,
|
||||
LinkedObjectFile& file,
|
||||
DecompilerTypeSystem& dts,
|
||||
DecompilerTypeSystem& previous_game_ts);
|
||||
DecompilerTypeSystem& previous_game_ts,
|
||||
ObjectFileDB::PerObjectAllTypeInfo& object_file_meta);
|
||||
|
||||
} // namespace decompiler
|
||||
} // namespace decompiler
|
||||
|
||||
+7966
-7914
File diff suppressed because it is too large
Load Diff
@@ -212,7 +212,7 @@
|
||||
(deactivate (_type_) none 10)
|
||||
(init-from-entity! (_type_ entity-actor) none 11) ;; todo check
|
||||
(run-logic? (_type_) symbol 12)
|
||||
(dummy-13 () none 13)
|
||||
(process-tree-method-13 () none 13)
|
||||
)
|
||||
:size-assert #x24
|
||||
:method-count-assert 14
|
||||
@@ -698,4 +698,4 @@
|
||||
`(rlet ((pp :reg r13 :reset-here #t :type process))
|
||||
(deactivate pp)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1,390 +0,0 @@
|
||||
# This does a (currently) 3 pass cleanup on all-types
|
||||
# 1. Cleanup any symbol definitions that are redundant
|
||||
# 2. Reorder symbol definitions based on file build order
|
||||
# 3. Check for any necessary forward declarations
|
||||
|
||||
import os
|
||||
|
||||
# First pass!
|
||||
print("First Pass - Cleaning up File")
|
||||
new_file = []
|
||||
with open("./decompiler/config/all-types.gc") as f:
|
||||
symbols_found = []
|
||||
lines = f.readlines()
|
||||
for line in lines:
|
||||
if line.startswith("(deftype") or line.startswith("(define-extern") or line.startswith(";;(define-extern"):
|
||||
symbol = line.split(" ")[1]
|
||||
if symbol in symbols_found and "unknown type" in line:
|
||||
continue
|
||||
else:
|
||||
symbols_found.append(symbol)
|
||||
new_file.append(line)
|
||||
|
||||
os.remove("./decompiler/config/all-types.gc")
|
||||
with open("./decompiler/config/all-types.gc", "w") as f:
|
||||
f.writelines(new_file)
|
||||
|
||||
# Second Pass!
|
||||
# I try to preserve comments as best I can:
|
||||
# - comments that are file names are discarded, they are now redundant!
|
||||
# - comments prior to a symbol definition are considered part of that symbol definition, comments after are NOT
|
||||
# Build up a mapping of symbol-name -> symbol-definitions and then we can easily drop them into place
|
||||
# Symbols are defined by reading line-by-line until we see the next:
|
||||
# - deftype
|
||||
# - define-extern
|
||||
# - defenum
|
||||
# declare-types can be discarded, they are handled in the third pass
|
||||
# If something is defined more than once, take the longer definition
|
||||
|
||||
from jak1_file_list import file_list
|
||||
import json
|
||||
|
||||
script_comments = [
|
||||
";; ----------------------",
|
||||
";; File -",
|
||||
";; Source Path -",
|
||||
";; Containing DGOs -",
|
||||
";; Version -",
|
||||
";; - Types",
|
||||
";; - Functions",
|
||||
";; - Symbols",
|
||||
";; - Unknowns",
|
||||
";; NO FILE",
|
||||
";; Unknowns / Built-Ins / Non-Original Types",
|
||||
";; - Nothing Defined in This File!",
|
||||
";; Unknowns with No Definition"
|
||||
]
|
||||
|
||||
def is_filename_comment(line):
|
||||
if not line.startswith(";"):
|
||||
return False
|
||||
if "define" in line or "deftype" in line:
|
||||
return False
|
||||
cleaned_line = line.replace(";", "").strip()
|
||||
for item in file_list:
|
||||
file_name = item[0]
|
||||
if file_name == cleaned_line:
|
||||
return True
|
||||
return False
|
||||
|
||||
with open('./scripts/jak1-symbol-mapping.json') as f:
|
||||
data = json.load(f)
|
||||
all_symbols = []
|
||||
for object_file, symbols in data.items():
|
||||
for symbol in symbols:
|
||||
all_symbols.append(symbol)
|
||||
|
||||
def no_runtime_type(definition):
|
||||
for line in definition:
|
||||
if "no-runtime-type" in line:
|
||||
return True
|
||||
return False
|
||||
|
||||
def strip_trailing_new_lines(definition):
|
||||
new_definition = []
|
||||
found_content = False
|
||||
for line in reversed(definition):
|
||||
if found_content:
|
||||
new_definition.insert(0, line)
|
||||
else:
|
||||
cleaned_line = line.strip()
|
||||
if len(cleaned_line) != 0:
|
||||
found_content = True
|
||||
new_definition.insert(0, line)
|
||||
# Check if the last line has a new-line or not, if it doesn't add one
|
||||
if not new_definition[len(new_definition)-1].endswith("\n"):
|
||||
new_definition[len(new_definition)-1].append("\n")
|
||||
return new_definition
|
||||
|
||||
symbol_definitions = {}
|
||||
# Anything that is custom / not part of the game, I'll place at the top of the file because I have no idea where it should go
|
||||
unknown_symbol_definitions = []
|
||||
|
||||
def is_script_comment(line):
|
||||
for comment in script_comments:
|
||||
if line.startswith(comment):
|
||||
return True
|
||||
return False
|
||||
|
||||
print("Second Pass - Re-Organizing File")
|
||||
with open("./decompiler/config/all-types.gc") as f:
|
||||
lines = f.readlines()
|
||||
current_symbol = ""
|
||||
current_symbol_definition = []
|
||||
comment_buffer = []
|
||||
commented_type = False
|
||||
for i, line in enumerate(lines):
|
||||
# Ignore the following lines:
|
||||
# - declare-types
|
||||
# - empty lines
|
||||
# - file name comments
|
||||
# - comments i generate
|
||||
if is_filename_comment(line) or is_script_comment(line) or (commented_type is False and ("declare-type" in line or (line == "\n" and i != len(lines) - 1))):
|
||||
continue
|
||||
|
||||
# Handle the first line of the file properly
|
||||
if len(current_symbol_definition) == 0: # The only time this variable should be empty, is at the beginning, after that it should always be in use
|
||||
if line.startswith("(deftype") or line.startswith("(define-extern") or line.startswith(";;(define-extern") or line.startswith("(defenum"):
|
||||
current_symbol_definition.append(line)
|
||||
current_symbol = line.split(" ")[1].rstrip("\n")
|
||||
continue
|
||||
|
||||
# To support comments being associated with the following symbol def, we have to keep track of them
|
||||
# then either associate them with the new symbol OR realize they are part of the current one!
|
||||
if not commented_type and line.startswith(";") and not (line.startswith("; (deftype") or line.startswith("(define-extern") or line.startswith(";;(define-extern") or line.startswith("(defenum")):
|
||||
current_symbol_definition.append(line)
|
||||
comment_buffer.append(line)
|
||||
continue
|
||||
|
||||
# Check if we've reached a new symbol or reached the end of the file
|
||||
if i == len(lines) - 1 or line.startswith("(deftype") or line.startswith("; (deftype") or line.startswith("(define-extern") or line.startswith(";;(define-extern") or line.startswith("(defenum"):
|
||||
# Remove any comments from the previous symbol def
|
||||
if not commented_type and len(comment_buffer) > 0:
|
||||
current_symbol_definition = current_symbol_definition[:-len(comment_buffer)]
|
||||
# Check if the symbol we found is valid or invalid
|
||||
if current_symbol in all_symbols:
|
||||
if current_symbol in symbol_definitions:
|
||||
# print("Symbol re-defintion found for '{}', choosing the bigger one!".format(current_symbol))
|
||||
if len(current_symbol_definition) > len(symbol_definitions[current_symbol]):
|
||||
if no_runtime_type(current_symbol_definition):
|
||||
current_symbol_definition.insert(0, "(define-extern {} type) ; deftype provided by C Kernel\n".format(current_symbol))
|
||||
symbol_definitions[current_symbol] = strip_trailing_new_lines(current_symbol_definition.copy())
|
||||
else:
|
||||
if no_runtime_type(current_symbol_definition):
|
||||
current_symbol_definition.insert(0, "(define-extern {} type) ; deftype provided by C Kernel\n".format(current_symbol))
|
||||
symbol_definitions[current_symbol] = strip_trailing_new_lines(current_symbol_definition.copy())
|
||||
else:
|
||||
print("Found a symbol '{}' that is not part of Jak 1!".format(current_symbol))
|
||||
unknown_symbol_definitions.append(current_symbol_definition.copy())
|
||||
if i != len(lines) - 1:
|
||||
if line.startswith("; (deftype"):
|
||||
current_symbol = line.split(" ")[2].rstrip("\n")
|
||||
commented_type = True
|
||||
else:
|
||||
current_symbol = line.split(" ")[1].rstrip("\n")
|
||||
commented_type = False
|
||||
current_symbol_definition.clear()
|
||||
current_symbol_definition += comment_buffer
|
||||
comment_buffer.clear()
|
||||
current_symbol_definition.append(line)
|
||||
elif line.startswith("(deftype") or line.startswith("; (deftype") or line.startswith("(define-extern") or line.startswith(";;(define-extern") or line.startswith("(defenum"):
|
||||
if line.startswith("; (deftype"):
|
||||
current_symbol = line.split(" ")[2].rstrip("\n")
|
||||
else:
|
||||
current_symbol = line.split(" ")[1].rstrip("\n")
|
||||
current_symbol_definition.clear()
|
||||
current_symbol_definition += comment_buffer
|
||||
comment_buffer.clear()
|
||||
current_symbol_definition.append(line)
|
||||
symbol_definitions[current_symbol] = strip_trailing_new_lines(current_symbol_definition.copy())
|
||||
else:
|
||||
current_symbol_definition.append(line)
|
||||
if len(comment_buffer) > 0:
|
||||
comment_buffer.clear()
|
||||
|
||||
# Now armed with our complete list of symbols, print them out in a nice organized manner.
|
||||
# Precedence:
|
||||
# - unknown symbols (after kernel definitions - `gstate` is the last)
|
||||
# - file build order (include info about src file path / DGOs it's contained in)
|
||||
# - types
|
||||
# - functions
|
||||
# - unknown types/symbols/functions
|
||||
|
||||
with open('./scripts/jak1-symbol-mapping.json') as f:
|
||||
symbol_mapping = json.load(f)
|
||||
|
||||
def first_relevant_line(definition):
|
||||
if len(definition) == 1:
|
||||
return definition[0]
|
||||
for line in definition:
|
||||
if line.startswith("; (deftype") or not line.startswith(";"):
|
||||
return line
|
||||
|
||||
def was_previous_definition_multi_line(definition):
|
||||
line_count = 0
|
||||
for line in definition:
|
||||
if "; (deftype" in line:
|
||||
return True
|
||||
if not line.strip().startswith(";"):
|
||||
line_count = line_count + 1
|
||||
return line_count > 1
|
||||
|
||||
def print_definition_block(file_lines, header, prev_block_exists, def_list):
|
||||
if len(def_list) == 0:
|
||||
return False
|
||||
if prev_block_exists:
|
||||
file_lines.append("\n")
|
||||
file_lines.append(";; - {}\n\n".format(header))
|
||||
prev_definition = None
|
||||
for definition in def_list:
|
||||
if prev_definition is not None and was_previous_definition_multi_line(prev_definition):
|
||||
file_lines.append("\n")
|
||||
file_lines.append("".join(definition))
|
||||
prev_definition = definition
|
||||
return True
|
||||
|
||||
def print_definition_blocks(file_lines, types, functions, symbols, unknowns):
|
||||
if not types and not functions and not symbols and not unknowns:
|
||||
file_lines.append(";; - Nothing Defined in This File!\n")
|
||||
else:
|
||||
prev_block_exists = False
|
||||
prev_block_exists |= print_definition_block(file_lines, "Types", prev_block_exists, types)
|
||||
prev_block_exists |= print_definition_block(file_lines, "Functions", prev_block_exists, functions)
|
||||
prev_block_exists |= print_definition_block(file_lines, "Symbols", prev_block_exists, symbols)
|
||||
prev_block_exists |= print_definition_block(file_lines, "Unknowns", prev_block_exists, unknowns)
|
||||
file_lines.append("\n")
|
||||
|
||||
new_file = []
|
||||
for item in file_list:
|
||||
file_name = item[0]
|
||||
extension = "gc"
|
||||
if item[2] == 4:
|
||||
extension = "gd"
|
||||
src_path = "{}/{}.{}".format(item[4], file_name, extension)
|
||||
new_file.append("\n;; ----------------------\n;; File - {}\n;; Source Path - {}\n;; Containing DGOs - {}\n;; Version - {}\n\n".format(file_name, src_path, item[3], item[2]))
|
||||
types = []
|
||||
functions = []
|
||||
symbols = []
|
||||
unknowns = []
|
||||
if file_name in symbol_mapping:
|
||||
for symbol in symbol_mapping[file_name]:
|
||||
if symbol not in symbol_definitions:
|
||||
print("Could not find definition for '{}'".format(symbol))
|
||||
else:
|
||||
symbol_definition = symbol_definitions[symbol]
|
||||
if ";;(define-extern" in first_relevant_line(symbol_definition):
|
||||
unknowns.append(symbol_definition)
|
||||
elif "(function" in first_relevant_line(symbol_definition) or "function)" in first_relevant_line(symbol_definition):
|
||||
functions.append(symbol_definition)
|
||||
elif "deftype" in first_relevant_line(symbol_definition):
|
||||
types.append(symbol_definition)
|
||||
elif "define-extern" in first_relevant_line(symbol_definition):
|
||||
symbols.append(symbol_definition)
|
||||
else:
|
||||
print("Could not find associated symbol def for '{}'".format(symbol))
|
||||
|
||||
print_definition_blocks(new_file, types, functions, symbols, unknowns)
|
||||
|
||||
cleaned_unknown_symbol_defs = []
|
||||
if file_name == "gcommon":
|
||||
new_file.append("\n;; ----------------------\n;; NO FILE\n;; Unknowns / Built-Ins / Non-Original Types\n\n")
|
||||
for definition in unknown_symbol_definitions:
|
||||
if not definition[0].startswith(";;(define-extern"):
|
||||
new_file.append("".join(definition) + "\n")
|
||||
else:
|
||||
cleaned_unknown_symbol_defs.append(definition)
|
||||
|
||||
if len(cleaned_unknown_symbol_defs) > 0:
|
||||
new_file.append("\n;; ----------------------\n;; NO FILE\n;; Unknowns with No Definition\n\n")
|
||||
for definition in cleaned_unknown_symbol_defs:
|
||||
if definition[0].startswith(";;(define-extern"):
|
||||
new_file.append("".join(definition).rstrip() + "\n")
|
||||
|
||||
os.remove("./decompiler/config/all-types.gc")
|
||||
with open("./decompiler/config/all-types.gc", "w") as f:
|
||||
f.writelines(new_file)
|
||||
|
||||
|
||||
# Third pass! Add any necessary forward declarations
|
||||
# - First, let's identify the line numbers where types are defined, and used
|
||||
# - Then, repeat the process, adding forward declarations when appropriate
|
||||
|
||||
type_usages = {}
|
||||
|
||||
def get_root_parent_type(t):
|
||||
if t["parent_type"] in ["basic", "structure", "symbol", "object", "integer", "pair", "number", "binteger", "function", "array", "type", "string", "uint8", "int8", "uint16", "int16", "uint32", "int32", "uint64", "int64", "uint128", "int128", "float", "kheap"]:
|
||||
parent_type = t["parent_type"]
|
||||
if parent_type not in ["basic", "structure"]:
|
||||
return "type" # NOTE - this does not work currently!!! but is VERY RARE
|
||||
return t["parent_type"]
|
||||
return get_root_parent_type(type_usages[t["parent_type"]])
|
||||
|
||||
def get_safe_parent_type(current_type, all_types, earliest_usage_line):
|
||||
parent_type_name = current_type["parent_type"]
|
||||
if parent_type_name in ["basic", "structure", "type"]:
|
||||
return parent_type_name
|
||||
parent_type = all_types[parent_type_name]
|
||||
if parent_type["declared_on_line"] < earliest_usage_line:
|
||||
return parent_type["type_name"]
|
||||
return get_root_parent_type(current_type)
|
||||
|
||||
def symbol_usage(line, sym):
|
||||
if line.strip().startswith(";"):
|
||||
return False
|
||||
tokens = line.strip().split(" ")
|
||||
for token in tokens[1:]:
|
||||
sanitized_token = token.replace("(", "").replace(")", "").strip()
|
||||
if sanitized_token == sym:
|
||||
return True
|
||||
return False
|
||||
|
||||
new_file = []
|
||||
print("Third Pass - Adding Forward Type Declarations")
|
||||
with open("./decompiler/config/all-types.gc") as f:
|
||||
lines = f.readlines()
|
||||
# Get the types
|
||||
for i, line in enumerate(lines):
|
||||
clean_line = line.replace(";", "").strip()
|
||||
if clean_line.startswith("(deftype"):
|
||||
symbol = clean_line.split(" ")[1]
|
||||
parent_type = clean_line.split(" ")[2].rstrip("\n").replace("(", "").replace(")", "")
|
||||
if parent_type == "UNKNOWN":
|
||||
continue
|
||||
type_usages[symbol] = {
|
||||
"type_name": symbol,
|
||||
"parent_type": parent_type,
|
||||
"declared_on_line": i,
|
||||
"first_symbol_usage": "",
|
||||
"used_on_lines": [],
|
||||
"commented_out_type": line.startswith(";")
|
||||
}
|
||||
# Identify Usages - heavy loop
|
||||
for symbol, usage_info in type_usages.items():
|
||||
symbol_index = list(type_usages.keys()).index(symbol)
|
||||
if symbol_index % 100 == 0:
|
||||
print("[{}/{}]: Finding Type Usages".format(symbol_index, len(type_usages)))
|
||||
current_symbol = ""
|
||||
for i, line in enumerate(lines):
|
||||
if i > usage_info["declared_on_line"]:
|
||||
break # For speed reasons, we don't care about usages after the declaration
|
||||
if "; deftype provided by C Kernel" in line:
|
||||
continue
|
||||
if line.startswith("(deftype") or line.startswith("(define-extern") or line.startswith(";;(define-extern") or line.startswith("(defenum"):
|
||||
current_symbol = line.split(" ")[1]
|
||||
if i != usage_info["declared_on_line"] and symbol_usage(line, usage_info["type_name"]):
|
||||
if len(usage_info["used_on_lines"]) == 0:
|
||||
usage_info["first_symbol_usage"] = current_symbol
|
||||
usage_info["used_on_lines"].append(i)
|
||||
|
||||
# Identify Necessary Forward Declarations
|
||||
forward_declarations = {}
|
||||
for symbol, usage_info in type_usages.items():
|
||||
declaration_line = usage_info["declared_on_line"]
|
||||
if len(usage_info["used_on_lines"]) == 0:
|
||||
continue
|
||||
earliest_usage = usage_info["used_on_lines"][0]
|
||||
if declaration_line > earliest_usage or usage_info["commented_out_type"]:
|
||||
if usage_info["first_symbol_usage"] not in forward_declarations:
|
||||
forward_declarations[usage_info["first_symbol_usage"]] = ["(declare-type {} {})\n".format(symbol, get_safe_parent_type(usage_info, type_usages, earliest_usage))]
|
||||
else:
|
||||
forward_declarations[usage_info["first_symbol_usage"]].append("(declare-type {} {})\n".format(symbol, get_safe_parent_type(usage_info, type_usages, earliest_usage)))
|
||||
|
||||
# FINALLY - add the forward declarations\
|
||||
skip_next = False
|
||||
for i, line in enumerate(lines):
|
||||
if skip_next:
|
||||
skip_next = False
|
||||
new_file.append(line)
|
||||
continue
|
||||
if "; deftype provided by C Kernel" in line:
|
||||
skip_next = True
|
||||
if line.startswith("(deftype") or line.startswith("(define-extern") or line.startswith(";;(define-extern"):
|
||||
current_symbol = line.split(" ")[1]
|
||||
if current_symbol in forward_declarations:
|
||||
new_file.append("".join(forward_declarations[current_symbol]))
|
||||
new_file.append(line)
|
||||
|
||||
os.remove("./decompiler/config/all-types.gc")
|
||||
with open("./decompiler/config/all-types.gc", "w") as f:
|
||||
f.writelines(new_file)
|
||||
|
||||
@@ -6,6 +6,7 @@ import sys
|
||||
parser = argparse.ArgumentParser("update-env")
|
||||
parser.add_argument("--game", help="The name of the game", type=str)
|
||||
parser.add_argument("--decomp_config", help="The decompiler config file", type=str)
|
||||
parser.add_argument("--info", help="Just print out current settings", action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
# TODO - read from defaults
|
||||
@@ -28,6 +29,10 @@ with open(env_path, 'r') as env_file:
|
||||
if tokens[0] in file:
|
||||
file[tokens[0]] = tokens[1].strip()
|
||||
|
||||
if args.info:
|
||||
print(file)
|
||||
sys.exit(0)
|
||||
|
||||
valid_games = ["jak1", "jak2"]
|
||||
|
||||
decomp_config_map = {
|
||||
|
||||
-4
@@ -1686,7 +1686,3 @@
|
||||
|
||||
;; failed to figure out what this is:
|
||||
(kmemclose)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user