diff --git a/check.sh b/check.sh new file mode 100755 index 0000000000..b9c9c8a27a --- /dev/null +++ b/check.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Directory of this script +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +cd ${DIR}/out +md5sum --check hash.md5 \ No newline at end of file diff --git a/common/link_types.h b/common/link_types.h index 9572fda96b..d393d9066c 100644 --- a/common/link_types.h +++ b/common/link_types.h @@ -36,4 +36,12 @@ struct ObjectHeader { char name[60]; }; +// Header for link data used for V2 linking data +// used in GOAL and OpenGOAL +struct LinkHeaderV2 { + uint32_t type_tag; // always -1 + uint32_t length; // length of link data + uint32_t version; // always 2 +}; + #endif // JAK1_LINK_TYPES_H diff --git a/decompiler/CMakeLists.txt b/decompiler/CMakeLists.txt index 1710e572a9..0ffc11564d 100644 --- a/decompiler/CMakeLists.txt +++ b/decompiler/CMakeLists.txt @@ -19,7 +19,8 @@ add_executable(decompiler IR/IR.cpp IR/IR_TypeAnalysis.cpp Function/TypeInspector.cpp - data/tpage.cpp) + data/tpage.cpp + data/game_text.cpp) target_link_libraries(decompiler goos diff --git a/decompiler/ObjectFile/LinkedObjectFile.cpp b/decompiler/ObjectFile/LinkedObjectFile.cpp index 6f38d0422d..fdf72a8e17 100644 --- a/decompiler/ObjectFile/LinkedObjectFile.cpp +++ b/decompiler/ObjectFile/LinkedObjectFile.cpp @@ -823,6 +823,7 @@ std::string LinkedObjectFile::get_goal_string(int seg, int word_idx, bool with_q char cword[4]; memcpy(cword, &word.data, 4); result += cword[byte_offset]; + assert(result.back() != 0); } if (with_quotes) { result += "\""; diff --git a/decompiler/ObjectFile/LinkedObjectFileCreation.cpp b/decompiler/ObjectFile/LinkedObjectFileCreation.cpp index 936aefeb0b..73d95a59f7 100644 --- a/decompiler/ObjectFile/LinkedObjectFileCreation.cpp +++ b/decompiler/ObjectFile/LinkedObjectFileCreation.cpp @@ -9,6 +9,7 @@ #include "LinkedObjectFileCreation.h" #include "decompiler/config.h" #include "decompiler/util/DecompilerTypeSystem.h" +#include "common/link_types.h" // There are three link versions: // V2 - not really in use anymore, but V4 will resue logic from it (and the game didn't rename the @@ -26,13 +27,6 @@ struct LinkHeaderCommon { uint16_t version; // what version (2, 3, 4) }; -// Header for link data used for V2 linking data -struct LinkHeaderV2 { - uint32_t type_tag; // always -1 - uint32_t length; // length of link data - uint32_t version; // always 2 -}; - // Header for link data used for V4 struct LinkHeaderV4 { uint32_t type_tag; // always -1 @@ -101,12 +95,15 @@ static uint32_t c_symlink2(LinkedObjectFile& f, uint32_t next_reloc = link_ptr_offset + 1; if (seek & 3) { + // 0b01, 0b10 seek = (relocPtr[1] << 8) | table_value; next_reloc = link_ptr_offset + 2; if (seek & 2) { + // 0b10 seek = (relocPtr[2] << 16) | seek; next_reloc = link_ptr_offset + 3; if (seek & 1) { + // 0b11 seek = (relocPtr[3] << 24) | seek; next_reloc = link_ptr_offset + 4; } @@ -216,24 +213,50 @@ static uint32_t align16(uint32_t in) { } /*! - * Process link data for a "V4" object file. + * Process link data for a "V4" or "V2" object file. * In reality a V4 seems to be just a V2 object, but with the link data after the real data. * There's a V4 header at the very beginning, but another V2 header/link data at the end * ----------------------------------------------- * | V4 header | data | V2 header | V2 link data | * ----------------------------------------------- + * + * V2 + * ----------------------------------- + * | V2 header | V2 link data | data | + * ----------------------------------- + * The V4 format avoids having to copy the data to the left once the V2 link data is discarded. + * Presumably once they decided that data could never be relocated after being loaded in, + * it became worth it to throw away the link data, and avoid the memcpy of the data. + * The memcpy is surprisingly expensive, when you consider the linker ran for ~3% of a frame each + * frame and level data is ~10 MB. */ -static void link_v4(LinkedObjectFile& f, - const std::vector& data, - const std::string& name, - DecompilerTypeSystem& dts) { - // read the V4 header to find where the link data really is +static void link_v2_or_v4(LinkedObjectFile& f, + const std::vector& data, + const std::string& name, + DecompilerTypeSystem& dts) { const auto* header = (const LinkHeaderV4*)&data.at(0); - uint32_t link_data_offset = header->code_size + sizeof(LinkHeaderV4); // no basic offset + assert(header->version == 4 || header->version == 2); - // code starts immediately after the header - uint32_t code_offset = sizeof(LinkHeaderV4); - uint32_t code_size = header->code_size; + // these are different depending on the version. + uint32_t code_offset, link_data_offset, code_size; + + if (header->version == 4) { + // code starts immediately after the V4 header + code_offset = sizeof(LinkHeaderV4); + // link_data_offset points to a V2 header + link_data_offset = header->code_size + sizeof(LinkHeaderV4); + // code size is specified! + code_size = header->code_size; + } else { + // link data starts immediately + link_data_offset = 0; + + // code is after all the link data + code_offset = header->length; + // we have to compute the code size ourself + code_size = data.size() - code_offset; + assert(header->type_tag == 0xffffffff); + } f.stats.total_code_bytes += code_size; f.stats.total_v2_code_bytes += code_size; @@ -241,7 +264,7 @@ static void link_v4(LinkedObjectFile& f, // add all code const uint8_t* code_start = &data.at(code_offset); const uint8_t* code_end = - &data.at(code_offset + code_size); // safe because link data is after code. + &data.at(code_offset + code_size - 1) + 1; // get the pointer to one past the end. assert(((code_end - code_start) % 4) == 0); f.set_segment_count(1); for (auto x = code_start; x < code_end; x += 4) { @@ -250,12 +273,13 @@ static void link_v4(LinkedObjectFile& f, // read v2 header after the code const uint8_t* link_data = &data.at(link_data_offset); - const auto* link_header_v2 = (const LinkHeaderV2*)(link_data); // subtract off type tag + uint32_t link_ptr_offset = link_data_offset; + link_ptr_offset += sizeof(LinkHeaderV2); + auto* link_header_v2 = (const LinkHeaderV2*)(link_data); assert(link_header_v2->type_tag == 0xffffffff); assert(link_header_v2->version == 2); assert(link_header_v2->length == header->length); f.stats.total_v2_link_bytes += link_header_v2->length; - uint32_t link_ptr_offset = link_data_offset + sizeof(LinkHeaderV2); // first "section" of link data is a list of where all the pointer are. if (data.at(link_ptr_offset) == 0) { @@ -369,7 +393,8 @@ static void link_v4(LinkedObjectFile& f, // check length assert(link_header_v2->length == align64(link_ptr_offset - link_data_offset + 1)); - while (link_ptr_offset < data.size()) { + size_t expected_end = header->version == 4 ? data.size() : link_header_v2->length; + while (link_ptr_offset < expected_end) { assert(data.at(link_ptr_offset) == 0); link_ptr_offset++; } @@ -790,12 +815,13 @@ LinkedObjectFile to_linked_object_file(const std::vector& data, if (header->version == 3) { assert(header->type_tag == 0); link_v3(result, data, name, dts); - } else if (header->version == 4) { + } else if (header->version == 4 || header->version == 2) { assert(header->type_tag == 0xffffffff); - link_v4(result, data, name, dts); + link_v2_or_v4(result, data, name, dts); } else if (header->version == 5) { link_v5(result, data, name, dts); } else { + printf("Unsupported version %d\n", header->version); assert(false); } diff --git a/decompiler/ObjectFile/ObjectFileDB.cpp b/decompiler/ObjectFile/ObjectFileDB.cpp index 0b469056a7..0957352133 100644 --- a/decompiler/ObjectFile/ObjectFileDB.cpp +++ b/decompiler/ObjectFile/ObjectFileDB.cpp @@ -11,6 +11,7 @@ #include #include #include "decompiler/data/tpage.h" +#include "decompiler/data/game_text.h" #include "LinkedObjectFileCreation.h" #include "decompiler/config.h" #include "third-party/minilzo/minilzo.h" @@ -32,6 +33,37 @@ std::string strip_dgo_extension(const std::string& x) { } return x; } + +/*! + * Get an object name from a file name. + * Strips off the file extension and anything before the last slash. + */ +std::string obj_filename_to_name(const std::string& x) { + auto end = x.length(); + + // find last dot + auto last_dot = end; + for (; last_dot-- > 0;) { + if (x.at(last_dot) == '.') { + break; + } + } + + if (last_dot == 0) { + last_dot = end; + } + + auto last_slash = end; + for (; last_slash-- > 0;) { + if (x.at(last_slash) == '\\' || x.at(last_slash) == '/') { + break; + } + } + + assert(last_dot > last_slash + 1); + assert(last_slash + 1 < x.length()); + return x.substr(last_slash + 1, last_dot - last_slash - 1); +} } // namespace std::string ObjectFileData::to_unique_name() const { @@ -53,6 +85,7 @@ std::string ObjectFileData::to_unique_name() const { return record.name; } } + ObjectFileData& ObjectFileDB::lookup_record(const ObjectFileRecord& rec) { ObjectFileData* result = nullptr; @@ -72,7 +105,8 @@ ObjectFileData& ObjectFileDB::lookup_record(const ObjectFileRecord& rec) { * Build an object file DB for the given list of DGOs. */ ObjectFileDB::ObjectFileDB(const std::vector& _dgos, - const std::string& obj_file_name_map_file) { + const std::string& obj_file_name_map_file, + const std::vector& object_files) { Timer timer; spdlog::info("-Loading types..."); @@ -93,6 +127,12 @@ ObjectFileDB::ObjectFileDB(const std::vector& _dgos, get_objs_from_dgo(dgo); } + for (auto& obj : object_files) { + auto data = file_util::read_binary_file(obj); + auto name = obj_filename_to_name(obj); + add_obj_from_dgo(name, name, data.data(), data.size(), "NO-XGO"); + } + spdlog::info("ObjectFileDB Initialized:"); spdlog::info("Total DGOs: {}", int(_dgos.size())); spdlog::info("Total data: {} bytes", stats.total_dgo_bytes); @@ -629,6 +669,34 @@ void ObjectFileDB::process_tpages() { 100.f * float(success) / float(total), timer.getMs()); } +std::string ObjectFileDB::process_game_text() { + spdlog::info("- Finding game text..."); + std::string text_string = "COMMON"; + Timer timer; + int file_count = 0; + int string_count = 0; + int char_count = 0; + std::unordered_map> text_by_language_by_id; + + for_each_obj([&](ObjectFileData& data) { + if (data.name_in_dgo.substr(1) == text_string) { + file_count++; + auto statistics = ::process_game_text(data); + string_count += statistics.total_text; + char_count += statistics.total_chars; + if (text_by_language_by_id.find(statistics.language) != text_by_language_by_id.end()) { + assert(false); + } + text_by_language_by_id[statistics.language] = std::move(statistics.text); + } + }); + + spdlog::info("Processed {} text files ({} strings, {} characters) in {:.2f} ms", file_count, + string_count, char_count, timer.getMs()); + + return write_game_text(text_by_language_by_id); +} + void ObjectFileDB::analyze_functions() { spdlog::info("- Analyzing Functions..."); Timer timer; diff --git a/decompiler/ObjectFile/ObjectFileDB.h b/decompiler/ObjectFile/ObjectFileDB.h index 0a6aa15de7..0776cf19c3 100644 --- a/decompiler/ObjectFile/ObjectFileDB.h +++ b/decompiler/ObjectFile/ObjectFileDB.h @@ -45,7 +45,9 @@ struct ObjectFileData { class ObjectFileDB { public: - ObjectFileDB(const std::vector& _dgos, const std::string& obj_file_name_map_file); + ObjectFileDB(const std::vector& _dgos, + const std::string& obj_file_name_map_file, + const std::vector& object_files); std::string generate_dgo_listing(); std::string generate_obj_listing(); void process_link_data(); @@ -57,6 +59,7 @@ class ObjectFileDB { void write_disassembly(const std::string& output_dir, bool disassemble_objects_without_functions); void analyze_functions(); void process_tpages(); + std::string process_game_text(); ObjectFileData& lookup_record(const ObjectFileRecord& rec); DecompilerTypeSystem dts; diff --git a/decompiler/config.cpp b/decompiler/config.cpp index 5715ae4bdf..8642036eda 100644 --- a/decompiler/config.cpp +++ b/decompiler/config.cpp @@ -15,6 +15,7 @@ void set_config(const std::string& path_to_config_file) { gConfig.game_version = cfg.at("game_version").get(); gConfig.dgo_names = cfg.at("dgo_names").get>(); + gConfig.object_file_names = cfg.at("object_file_names").get>(); if (cfg.contains("obj_file_name_map_file")) { gConfig.obj_file_name_map_file = cfg.at("obj_file_name_map_file").get(); } @@ -27,6 +28,7 @@ void set_config(const std::string& path_to_config_file) { gConfig.write_hex_near_instructions = cfg.at("write_hex_near_instructions").get(); gConfig.analyze_functions = cfg.at("analyze_functions").get(); gConfig.process_tpages = cfg.at("process_tpages").get(); + gConfig.process_game_text = cfg.at("process_game_text").get(); std::vector asm_functions_by_name = cfg.at("asm_functions_by_name").get>(); diff --git a/decompiler/config.h b/decompiler/config.h index 3cd31562d5..eea63fa1f6 100644 --- a/decompiler/config.h +++ b/decompiler/config.h @@ -10,6 +10,7 @@ struct Config { int game_version = -1; std::vector dgo_names; + std::vector object_file_names; std::unordered_set bad_inspect_types; std::string obj_file_name_map_file; bool write_disassembly = false; @@ -20,6 +21,7 @@ struct Config { bool write_hex_near_instructions = false; bool analyze_functions = false; bool process_tpages = false; + bool process_game_text = false; std::unordered_set asm_functions_by_name; // ... }; diff --git a/decompiler/config/all-types.gc b/decompiler/config/all-types.gc index 4e0fc4c1bc..3ae62714ff 100644 --- a/decompiler/config/all-types.gc +++ b/decompiler/config/all-types.gc @@ -2570,8 +2570,9 @@ (deftype game-text (structure) ((id uint32 :offset-assert 0) - (text basic :offset-assert 4) + (text string :offset-assert 4) ) + :pack-me :method-count-assert 9 :size-assert #x8 :flag-assert #x900000008 @@ -2580,8 +2581,8 @@ (deftype game-text-info (basic) ((length int32 :offset-assert 4) (language-id int32 :offset-assert 8) - (group-name basic :offset-assert 12) - (data uint8 :dynamic :offset-assert 16) + (group-name string :offset-assert 12) + (data game-text :dynamic :inline :offset-assert 16) ) :method-count-assert 10 :size-assert #x10 diff --git a/decompiler/config/jak1_ntsc_black_label.jsonc b/decompiler/config/jak1_ntsc_black_label.jsonc index 366778002d..793e771bcb 100644 --- a/decompiler/config/jak1_ntsc_black_label.jsonc +++ b/decompiler/config/jak1_ntsc_black_label.jsonc @@ -2,7 +2,7 @@ { "game_version":1, - // the order here matters. KERNEL and GAME should go first + // the order here matters (not sure that this is true any more...). KERNEL and GAME should go first "dgo_names":["CGO/KERNEL.CGO","CGO/GAME.CGO", "CGO/ENGINE.CGO" , "CGO/ART.CGO", "DGO/BEA.DGO", "DGO/CIT.DGO", "CGO/COMMON.CGO", "DGO/DAR.DGO", "DGO/DEM.DGO", @@ -12,6 +12,9 @@ "DGO/VI2.DGO", "DGO/VI3.DGO", "CGO/VILLAGEP.CGO", "CGO/WATER-AN.CGO" ], + "object_file_names":["TEXT/0COMMON.TXT", "TEXT/1COMMON.TXT", "TEXT/2COMMON.TXT", "TEXT/3COMMON.TXT", "TEXT/4COMMON.TXT", + "TEXT/5COMMON.TXT", "TEXT/6COMMON.TXT"], + "analyze_functions":true, "write_disassembly":true, @@ -21,6 +24,7 @@ "disassemble_objects_without_functions":false, "process_tpages":true, + "process_game_text":true, // to write out data of each object file "write_hexdump":false, diff --git a/decompiler/data/game_text.cpp b/decompiler/data/game_text.cpp new file mode 100644 index 0000000000..941ac30296 --- /dev/null +++ b/decompiler/data/game_text.cpp @@ -0,0 +1,162 @@ +#include +#include +#include +#include +#include "third-party/fmt/core.h" +#include "game_text.h" +#include "decompiler/ObjectFile/ObjectFileDB.h" +#include "common/goos/Reader.h" + +namespace { +template +T get_word(const LinkedWord& word) { + T result; + assert(word.kind == LinkedWord::PLAIN_DATA); + static_assert(sizeof(T) == 4, "bad get_word size"); + memcpy(&result, &word.data, 4); + return result; +} + +Label get_label(ObjectFileData& data, const LinkedWord& word) { + assert(word.kind == LinkedWord::PTR); + return data.linked_data.labels.at(word.label_id); +} + +int align16(int in) { + return (in + 15) & ~15; +} + +} // namespace + +/* +(deftype game-text (structure) + ((id uint32 :offset-assert 0) + (text basic :offset-assert 4) + ) + ) + +(deftype game-text-info (basic) + ((length int32 :offset-assert 4) + (language-id int32 :offset-assert 8) + (group-name basic :offset-assert 12) + (data game-text :dynamic :offset-assert 16) + ) + ) + */ + +GameTextResult process_game_text(ObjectFileData& data) { + GameTextResult result; + auto& words = data.linked_data.words_by_seg.at(0); + std::vector read_words(words.size(), false); + + int offset = 0; + + // type tage for game-text-info + if (words.at(offset).kind != LinkedWord::TYPE_PTR || + words.front().symbol_name != "game-text-info") { + assert(false); + } + read_words.at(offset)++; + offset++; + + // length field + read_words.at(offset)++; + u32 text_count = get_word(words.at(offset++)); + + // language-id field + read_words.at(offset)++; + u32 language = get_word(words.at(offset++)); + result.language = language; + + // group-name field + read_words.at(offset)++; + auto group_label = get_label(data, words.at(offset++)); + auto group_name = data.linked_data.get_goal_string_by_label(group_label); + assert(group_name == "common"); + // remember that we read these bytes + auto group_start = (group_label.offset / 4) - 1; + for (int j = 0; j < align16(8 + 1 + group_name.length()) / 4; j++) { + read_words.at(group_start + j)++; + } + + // read each text... + for (u32 i = 0; i < text_count; i++) { + // id number + read_words.at(offset)++; + auto text_id = get_word(words.at(offset++)); + + // label to string + read_words.at(offset)++; + auto text_label = get_label(data, words.at(offset++)); + + // actual string + auto text = data.linked_data.get_goal_string_by_label(text_label); + result.total_text++; + result.total_chars += text.length(); + + // no duplicate ids + if (result.text.find(text_id) != result.text.end()) { + assert(false); + } + + // escape characters + result.text[text_id] = goos::get_readable_string(text.c_str()); + + // remember what we read (-1 for the type tag) + auto string_start = (text_label.offset / 4) - 1; + // 8 for type tag and length fields, 1 for null char. + for (int j = 0; j < align16(8 + 1 + text.length()) / 4; j++) { + read_words.at(string_start + j)++; + } + } + + // alignment to the string section. + while (offset & 3) { + read_words.at(offset)++; + offset++; + } + + // make sure we read each thing at least once. + // reading more than once is ok, some text is duplicated. + for (int i = 0; i < int(words.size()); i++) { + if (read_words[i] < 1) { + std::string debug; + data.linked_data.append_word_to_string(debug, words.at(i)); + printf("[%d] %d 0x%s\n", i, int(read_words[i]), debug.c_str()); + assert(false); + } + } + + return result; +} + +std::string write_game_text( + const std::unordered_map>& data) { + // first sort languages: + std::vector langauges; + for (const auto& lang : data) { + langauges.push_back(lang.first); + } + std::sort(langauges.begin(), langauges.end()); + + // build map + std::map> text_by_id; + for (auto lang : langauges) { + for (auto text : data.at(lang)) { + text_by_id[text.first].push_back(text.second); + } + } + + // write! + std::string result = fmt::format("(language-count {})\n", langauges.size()); + result += "(group-name \"common\")\n"; + for (auto& x : text_by_id) { + result += fmt::format("(#x{:04x}\n ", x.first); + for (auto& y : x.second) { + result += fmt::format("\"{}\"\n ", y); + } + result += ")\n\n"; + } + + return result; +} \ No newline at end of file diff --git a/decompiler/data/game_text.h b/decompiler/data/game_text.h new file mode 100644 index 0000000000..44b6689406 --- /dev/null +++ b/decompiler/data/game_text.h @@ -0,0 +1,16 @@ +#pragma once +#include +#include + +class ObjectFileData; + +struct GameTextResult { + int total_text = 0; + int language = -1; + std::unordered_map text; + int total_chars = 0; +}; + +GameTextResult process_game_text(ObjectFileData& data); +std::string write_game_text( + const std::unordered_map>& data); \ No newline at end of file diff --git a/decompiler/main.cpp b/decompiler/main.cpp index 99ba29d5d9..756f7c022f 100644 --- a/decompiler/main.cpp +++ b/decompiler/main.cpp @@ -11,8 +11,8 @@ int main(int argc, char** argv) { spdlog::info("Beginning disassembly. This may take a few minutes..."); spdlog::set_level(spdlog::level::debug); - auto lu = spdlog::basic_logger_mt("GOAL Decompiler", "logs/decompiler.log"); - spdlog::set_default_logger(lu); + // auto lu = spdlog::basic_logger_mt("GOAL Decompiler", "logs/decompiler.log"); + // spdlog::set_default_logger(lu); spdlog::flush_on(spdlog::level::info); file_util::init_crc(); @@ -27,12 +27,16 @@ int main(int argc, char** argv) { std::string in_folder = argv[2]; std::string out_folder = argv[3]; - std::vector dgos; + std::vector dgos, objs; for (const auto& dgo_name : get_config().dgo_names) { dgos.push_back(file_util::combine_path(in_folder, dgo_name)); } - ObjectFileDB db(dgos, get_config().obj_file_name_map_file); + for (const auto& obj_name : get_config().object_file_names) { + objs.push_back(file_util::combine_path(in_folder, obj_name)); + } + + ObjectFileDB db(dgos, get_config().obj_file_name_map_file, objs); file_util::write_text_file(file_util::combine_path(out_folder, "dgo.txt"), db.generate_dgo_listing()); file_util::write_text_file(file_util::combine_path(out_folder, "obj.txt"), @@ -54,6 +58,11 @@ int main(int argc, char** argv) { db.analyze_functions(); } + if (get_config().process_game_text) { + auto result = db.process_game_text(); + file_util::write_text_file(file_util::get_file_path({"assets", "game_text.txt"}), result); + } + if (get_config().process_tpages) { db.process_tpages(); } diff --git a/game/kernel/fileio.cpp b/game/kernel/fileio.cpp index a5565ee9a5..63b09ba6dd 100644 --- a/game/kernel/fileio.cpp +++ b/game/kernel/fileio.cpp @@ -7,7 +7,7 @@ #include #include #include -#include "game/sce/stubs.h" +#include "game/sce/sif_ee.h" #include "fileio.h" #include "kprint.h" #include "common/versions.h" @@ -255,6 +255,8 @@ char* DecodeFileName(const char* name) { result = MakeFileName(CODE_FILE_TYPE, name + 6, 0); } else if (!strncmp(name, "$RES/", 5)) { result = MakeFileName(RES_FILE_TYPE, name + 5, 0); + } else if (!strncmp(name, "$JAK-PROJECT/", 13)) { + result = MakeFileName(JAK_PROJECT_FILE_TYPE, name + 13, 0); } else { printf("[ERROR] DecodeFileName: UNKNOWN FILE NAME %s\n", name); result = nullptr; @@ -378,6 +380,8 @@ char* MakeFileName(int type, const char* name, int new_string) { // REFPLANT? no idea static char nextDir[] = "/"; sprintf(buf, "%sconfig_data/refplant/%s", nextDir, name); + } else if (type == JAK_PROJECT_FILE_TYPE) { + sprintf(buffer_633, "/%s", name); } else { printf("UNKNOWN FILE TYPE %d\n", type); } diff --git a/game/kernel/fileio.h b/game/kernel/fileio.h index 0770c6a52a..7d094bc5aa 100644 --- a/game/kernel/fileio.h +++ b/game/kernel/fileio.h @@ -42,6 +42,8 @@ enum GoalFileType { CNT_FILE_TYPE = 0x3a, RES_FILE_TYPE = 0x3b, REFPLANT_FILE_TYPE = 0x301, + // added this, allows access directly to jak-project/ from within the game. + JAK_PROJECT_FILE_TYPE = 0x302 }; constexpr char FOLDER_PREFIX[] = ""; diff --git a/game/kernel/klink.cpp b/game/kernel/klink.cpp index 1014f7ed3d..716f4ba81d 100644 --- a/game/kernel/klink.cpp +++ b/game/kernel/klink.cpp @@ -20,6 +20,11 @@ namespace { // turn on printf's for debugging linking issues. constexpr bool link_debug_printfs = false; + +bool is_opengoal_object(const void* data) { + auto* header = (const LinkHeaderV2*)data; + return !(header->type_tag == 0xffffffff && (header->version == 2 || header->version == 4)); +} } // namespace // space to store a single in-progress linking state. @@ -41,105 +46,184 @@ void link_control::begin(Ptr object_file, int32_t size, Ptr heap, uint32_t flags) { - // save data from call to begin - m_object_data = object_file; - kstrcpy(m_object_name, name); - m_object_size = size; - m_heap = heap; - m_flags = flags; + if (is_opengoal_object(object_file.c())) { + // save data from call to begin + m_object_data = object_file; + kstrcpy(m_object_name, name); + m_object_size = size; + m_heap = heap; + m_flags = flags; - // initialize link control - m_entry.offset = 0; - m_heap_top = m_heap->top; - m_keep_debug = false; + // initialize link control + m_entry.offset = 0; + m_heap_top = m_heap->top; + m_keep_debug = false; + m_opengoal = true; - if (link_debug_printfs) { - char* goal_name = object_file.cast().c(); - printf("link %s\n", goal_name); - printf("link_control::begin %c%c%c%c\n", goal_name[0], goal_name[1], goal_name[2], - goal_name[3]); - } - - // points to the beginning of the linking data - m_link_block_ptr = object_file + BASIC_OFFSET; - m_code_size = 0; - m_code_start = object_file; - m_state = 0; - m_segment_process = 0; - - ObjectFileHeader* ofh = m_link_block_ptr.cast().c(); - if (ofh->goal_version_major != versions::GOAL_VERSION_MAJOR) { - fprintf( - stderr, - "VERSION ERROR: C Kernel built from GOAL %d.%d, but object file %s is from GOAL %d.%d\n", - versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR, name, ofh->goal_version_major, - ofh->goal_version_minor); - exit(0); - } - if (link_debug_printfs) { - printf("Object file header:\n"); - printf(" GOAL ver %d.%d obj %d len %d\n", ofh->goal_version_major, ofh->goal_version_minor, - ofh->object_file_version, ofh->link_block_length); - printf(" segment count %d\n", ofh->segment_count); - for (int i = 0; i < N_SEG; i++) { - printf(" seg %d link 0x%04x, 0x%04x data 0x%04x, 0x%04x\n", i, ofh->link_infos[i].offset, - ofh->link_infos[i].size, ofh->code_infos[i].offset, ofh->code_infos[i].size); + if (link_debug_printfs) { + char* goal_name = object_file.cast().c(); + printf("link %s\n", goal_name); + printf("link_control::begin %c%c%c%c\n", goal_name[0], goal_name[1], goal_name[2], + goal_name[3]); } - } - m_version = ofh->object_file_version; - if (ofh->object_file_version < 4) { - // three segment file + // points to the beginning of the linking data + m_link_block_ptr = object_file + BASIC_OFFSET; + m_code_size = 0; + m_code_start = object_file; + m_state = 0; + m_segment_process = 0; - // seek past the header - m_object_data.offset += ofh->link_block_length; - // todo, set m_code_size - - if (m_link_block_ptr.offset < m_heap->base.offset || - m_link_block_ptr.offset >= m_heap->top.offset) { - // the link block is outside our heap, or in the top of our heap. It's somebody else's - // problem. - if (link_debug_printfs) { - printf("Link block somebody else's problem\n"); + ObjectFileHeader* ofh = m_link_block_ptr.cast().c(); + if (ofh->goal_version_major != versions::GOAL_VERSION_MAJOR) { + fprintf( + stderr, + "VERSION ERROR: C Kernel built from GOAL %d.%d, but object file %s is from GOAL %d.%d\n", + versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR, name, ofh->goal_version_major, + ofh->goal_version_minor); + exit(0); + } + if (link_debug_printfs) { + printf("Object file header:\n"); + printf(" GOAL ver %d.%d obj %d len %d\n", ofh->goal_version_major, ofh->goal_version_minor, + ofh->object_file_version, ofh->link_block_length); + printf(" segment count %d\n", ofh->segment_count); + for (int i = 0; i < N_SEG; i++) { + printf(" seg %d link 0x%04x, 0x%04x data 0x%04x, 0x%04x\n", i, ofh->link_infos[i].offset, + ofh->link_infos[i].size, ofh->code_infos[i].offset, ofh->code_infos[i].size); } + } - if (m_heap->base.offset <= m_object_data.offset && // above heap base - m_object_data.offset < m_heap->top.offset && // less than heap top (not needed?) - m_object_data.offset < m_heap->current.offset) { // less than heap current + m_version = ofh->object_file_version; + if (ofh->object_file_version < 4) { + // three segment file + + // seek past the header + m_object_data.offset += ofh->link_block_length; + // todo, set m_code_size + + if (m_link_block_ptr.offset < m_heap->base.offset || + m_link_block_ptr.offset >= m_heap->top.offset) { + // the link block is outside our heap, or in the top of our heap. It's somebody else's + // problem. if (link_debug_printfs) { - printf("Code block in the heap, kicking it out for copy into heap\n"); + printf("Link block somebody else's problem\n"); + } + + if (m_heap->base.offset <= m_object_data.offset && // above heap base + m_object_data.offset < m_heap->top.offset && // less than heap top (not needed?) + m_object_data.offset < m_heap->current.offset) { // less than heap current + if (link_debug_printfs) { + printf("Code block in the heap, kicking it out for copy into heap\n"); + } + m_heap->current = m_object_data; + } + } else { + // in our heap, we need to move it so we can free up its space later on + if (link_debug_printfs) { + printf("Link block needs to be moved!\n"); + } + + // allocate space for a new one + auto new_link_block = kmalloc(m_heap, ofh->link_block_length, KMALLOC_TOP, "link-block"); + auto old_link_block = m_link_block_ptr - BASIC_OFFSET; + + // copy it + ultimate_memcpy(new_link_block.c(), old_link_block.c(), ofh->link_block_length); + m_link_block_ptr = new_link_block + BASIC_OFFSET; + + // if we can save some memory here + if (old_link_block.offset < m_heap->current.offset) { + if (link_debug_printfs) { + printf("Kick out old link block\n"); + } + m_heap->current = old_link_block; } - m_heap->current = m_object_data; } } else { - // in our heap, we need to move it so we can free up its space later on - if (link_debug_printfs) { - printf("Link block needs to be moved!\n"); - } + printf("UNHANDLED OBJECT FILE VERSION\n"); + assert(false); + } - // allocate space for a new one - auto new_link_block = kmalloc(m_heap, ofh->link_block_length, KMALLOC_TOP, "link-block"); - auto old_link_block = m_link_block_ptr - BASIC_OFFSET; - - // copy it - ultimate_memcpy(new_link_block.c(), old_link_block.c(), ofh->link_block_length); - m_link_block_ptr = new_link_block + BASIC_OFFSET; - - // if we can save some memory here - if (old_link_block.offset < m_heap->current.offset) { - if (link_debug_printfs) { - printf("Kick out old link block\n"); - } - m_heap->current = old_link_block; - } + if ((m_flags & LINK_FLAG_FORCE_DEBUG) && MasterDebug && !DiskBoot) { + m_keep_debug = true; } } else { - printf("UNHANDLED OBJECT FILE VERSION\n"); - assert(false); - } + m_opengoal = false; + // not an open goal object. + if (link_debug_printfs) { + printf("Linking GOAL style object\n"); + } - if ((m_flags & LINK_FLAG_FORCE_DEBUG) && MasterDebug && !DiskBoot) { - m_keep_debug = true; + // initialize + m_object_data = object_file; + kstrcpy(m_object_name, name); + m_object_size = size; + m_heap = heap; + m_flags = flags; + m_entry.offset = 0; + m_heap_top = m_heap->top; + m_keep_debug = false; + m_link_block_ptr = object_file + BASIC_OFFSET; + m_code_size = 0; + m_code_start = object_file; + m_state = 0; + m_segment_process = 0; + + const auto* header = (LinkHeaderV2*)(m_link_block_ptr.c() - 4); + + m_version = header->version; + if (header->version < 4) { + // seek past header + m_object_data.offset += header->length; + m_code_size = m_object_size - header->length; + if (m_link_block_ptr.offset < m_heap->base.offset || + m_link_block_ptr.offset >= m_heap->top.offset) { + // the link block is outside our heap, or in the top of our heap. It's somebody else's + // problem. + if (link_debug_printfs) { + printf("Link block somebody else's problem\n"); + } + + if (m_heap->base.offset <= m_object_data.offset && // above heap base + m_object_data.offset < m_heap->top.offset && // less than heap top (not needed?) + m_object_data.offset < m_heap->current.offset) { // less than heap current + if (link_debug_printfs) { + printf("Code block in the heap, kicking it out for copy into heap\n"); + } + m_heap->current = m_object_data; + } + } else { + // in our heap, we need to move it so we can free up its space later on + if (link_debug_printfs) { + printf("Link block needs to be moved!\n"); + } + + // allocate space for a new one + auto new_link_block = kmalloc(m_heap, header->length, KMALLOC_TOP, "link-block"); + auto old_link_block = m_link_block_ptr - BASIC_OFFSET; + + // copy it + ultimate_memcpy(new_link_block.c(), old_link_block.c(), header->length); + m_link_block_ptr = new_link_block + BASIC_OFFSET; + + // if we can save some memory here + if (old_link_block.offset < m_heap->current.offset) { + if (link_debug_printfs) { + printf("Kick out old link block\n"); + } + m_heap->current = old_link_block; + } + } + + } else { + // not yet implemented + assert(false); + } + + if ((m_flags & LINK_FLAG_FORCE_DEBUG) && MasterDebug && !DiskBoot) { + m_keep_debug = true; + } } } @@ -158,9 +242,13 @@ uint32_t link_control::work() { uint32_t rv; if (m_version == 3) { + assert(m_opengoal); rv = work_v3(); + } else if (m_version == 2 || m_version == 4) { + assert(!m_opengoal); + rv = work_v2(); } else { - printf("UNHANDLED OBJECT FILE VERSION IN WORK!\n"); + printf("UNHANDLED OBJECT FILE VERSION %d IN WORK!\n", m_version); assert(false); return 0; } @@ -409,7 +497,247 @@ uint32_t link_control::work_v3() { } } -// TODO - work_v2, once v2 objects are created. +Ptr c_symlink2(Ptr objData, Ptr linkObj, Ptr relocTable) { + u8* relocPtr = relocTable.c(); + Ptr objPtr = objData; + + do { + u8 table_value = *relocPtr; + u32 result = table_value; + u8* next_reloc = relocPtr + 1; + + if (result & 3) { + result = (relocPtr[1] << 8) | table_value; + next_reloc = relocPtr + 2; + if (result & 2) { + result = (relocPtr[2] << 16) | result; + next_reloc = relocPtr + 3; + if (result & 1) { + result = (relocPtr[3] << 24) | result; + next_reloc = relocPtr + 4; + } + } + } + + relocPtr = next_reloc; + objPtr = objPtr + (result & 0xfffffffc); + u32 objValue = *(objPtr.cast()); + if (objValue == 0xffffffff) { + *(objPtr.cast()) = linkObj.offset; + } else { + // I don't think we should hit this ever. + assert(false); + } + } while (*relocPtr); + + return make_ptr(relocPtr + 1); +} + +#define LINK_V2_STATE_INIT_COPY 0 +#define LINK_V2_STATE_OFFSETS 1 +#define LINK_V2_STATE_SYMBOL_TABLE 2 +#define OBJ_V2_CLOSE_ENOUGH 0x90 +#define OBJ_V2_MAX_TRANSFER 0x80000 + +uint32_t link_control::work_v2() { + // u32 startCycle = kernel.read_clock(); todo + + if (m_state == LINK_V2_STATE_INIT_COPY) { // initialization and copying to heap + // we move the data segment to eliminate gaps + // very small gaps can be tolerated, as it is not worth the time penalty to move large objects + // many bytes. if this requires copying a large amount of data, we will do it in smaller chunks, + // allowing the copy to be spread over multiple game frames + + // state initialization + if (m_segment_process == 0) { + m_heap_gap = + m_object_data - m_heap->current; // distance between end of heap and start of object + if (m_object_data.offset < m_heap->current.offset) { + assert(false); + } + } + + if (m_heap_gap < + OBJ_V2_CLOSE_ENOUGH) { // close enough, don't relocate the object, just expand the heap + if (link_debug_printfs) { + printf("[work_v2] close enough, not moving\n"); + } + m_heap->current = m_object_data + m_code_size; + if (m_heap->top.offset <= m_heap->current.offset) { + MsgErr("dkernel: heap overflow\n"); // game has ~% instead of \n :P + return 1; + } + } else { // not close enough, need to move the object + + // on the first run of this state... + if (m_segment_process == 0) { + m_original_object_location = m_object_data; + // allocate on heap, will have no gap + m_object_data = kmalloc(m_heap, m_code_size, 0, "data-segment"); + if (link_debug_printfs) { + printf("[work_v2] moving from 0x%x to 0x%x\n", m_original_object_location.offset, + m_object_data.offset); + } + if (!m_object_data.offset) { + MsgErr("dkernel: unable to malloc %d bytes for data-segment\n", m_code_size); + return 1; + } + } + + // the actual copy + Ptr source = m_original_object_location + m_segment_process; + u32 size = m_code_size - m_segment_process; + + if (size > OBJ_V2_MAX_TRANSFER) { // around .5 MB + ultimate_memcpy((m_object_data + m_segment_process).c(), source.c(), OBJ_V2_MAX_TRANSFER); + m_segment_process += OBJ_V2_MAX_TRANSFER; + return 0; // return, don't want to take too long. + } + + // if we have bytes to copy, but they are less than the max transfer, do it in one shot! + if (size) { + ultimate_memcpy((m_object_data + m_segment_process).c(), source.c(), size); + if (m_segment_process > 0) { // if we did a previous copy, we return now.... + m_state = LINK_V2_STATE_OFFSETS; + m_segment_process = 0; + return 0; + } + } + } + + // otherwise go straight into the next state. + m_state = LINK_V2_STATE_OFFSETS; + m_segment_process = 0; + } + + // init offset phase + if (m_state == LINK_V2_STATE_OFFSETS && m_segment_process == 0) { + m_reloc_ptr = m_link_block_ptr + 8; // seek to link table + if (*m_reloc_ptr == 0) { // do we have pointer links to do? + m_reloc_ptr.offset++; // if not, seek past the \0, and go to next state + m_state = LINK_V2_STATE_SYMBOL_TABLE; + m_segment_process = 0; + } else { + m_base_ptr = m_object_data; // base address for offsetting. + m_loc_ptr = m_object_data; // pointer which seeks thru the code + m_table_toggle = 0; // are we seeking or fixing? + m_segment_process = 1; // we've done first time setup + } + } + + if (m_state == LINK_V2_STATE_OFFSETS) { // pointer fixup + // this state reads through a table. Values alternate between "seek amount" and "number of + // consecutive 4-byte + // words to fix up". The counts are encoded using a variable length encoding scheme. They use + // a very stupid + // method of encoding values which requires O(n) bytes to store the value n. + + // to avoid dropping a frame, we check every 0x400 relocations to see if 0.5 milliseconds have + // elapsed. + u32 relocCounter = 0x400; + while (true) { // loop over entire table + while (true) { // loop over current mode + + // read and seek table + u8 count = *m_reloc_ptr; + m_reloc_ptr.offset++; + + if (!m_table_toggle) { // seek mode + m_loc_ptr.offset += + 4 * + count; // perform seek (MIPS instructions are 4 bytes, so we >> 2 the seek amount) + } else { // offset mode + for (u32 i = 0; i < count; i++) { + if (m_loc_ptr.offset % 4) { + assert(false); + } + u32 code = *(m_loc_ptr.cast()); + code += m_base_ptr.offset; + *(m_loc_ptr.cast()) = code; + m_loc_ptr.offset += 4; + } + } + + if (count != 0xff) { + break; + } + + if (*m_reloc_ptr == 0) { + m_reloc_ptr.offset++; + m_table_toggle = m_table_toggle ^ 1; + } + } + + // reached the end of the tableToggle mode + m_table_toggle = m_table_toggle ^ 1; + if (*m_reloc_ptr == 0) { + break; // end of the state + } + relocCounter--; + if (relocCounter == 0) { + // u32 clock_value = kernel.read_clock(); + // if(clock_value - startCycle > 150000) { // 0.5 milliseconds + // return 0; + // } + relocCounter = 0x400; + } + } + m_reloc_ptr.offset++; + m_state = 2; + m_segment_process = 0; + } + + if (m_state == 2) { // GOAL object fixup + if (*m_reloc_ptr == 0) { + m_state = 3; + m_segment_process = 0; + } else { + while (true) { + u32 relocation = *m_reloc_ptr; + m_reloc_ptr.offset++; + Ptr goalObj; + char* name; + if ((relocation & 0x80) == 0) { + // symbol! + if (relocation > 9) { + m_reloc_ptr.offset--; // no idea what this is. + } + name = m_reloc_ptr.cast().c(); + if (link_debug_printfs) { + printf("[work_v2] symlink: %s\n", name); + } + goalObj = intern_from_c(name).cast(); + } else { + // type! + u8 nMethods = relocation & 0x7f; + if (nMethods == 0) { + nMethods = 1; + } + name = m_reloc_ptr.cast().c(); + if (link_debug_printfs) { + printf("[work_v2] symlink -type: %s\n", name); + } + goalObj = intern_type_from_c(name, nMethods).cast(); + } + m_reloc_ptr.offset += strlen(name) + 1; + // DECOMPILER->hookStartSymlinkV3(_state - 1, _objectData, std::string(name)); + m_reloc_ptr = c_symlink2(m_object_data, goalObj, m_reloc_ptr); + // DECOMPILER->hookFinishSymlinkV3(); + if (*m_reloc_ptr == 0) { + break; // done + } + // u32 currentCycle = kernel.read_clock(); + // if(currentCycle - startCycle > 150000) { + // return 0; + // } + } + m_state = 3; + m_segment_process = 0; + } + } + m_entry = m_object_data + 4; + return 1; +} /*! * Complete linking. This will execute the top-level code for v3 object files, if requested. diff --git a/game/kernel/klink.h b/game/kernel/klink.h index 76f82e485f..47c5b245b4 100644 --- a/game/kernel/klink.h +++ b/game/kernel/klink.h @@ -39,6 +39,14 @@ struct link_control { uint32_t m_state; uint32_t m_segment_process; uint32_t m_version; + int m_heap_gap; + Ptr m_original_object_location; + Ptr m_reloc_ptr; + Ptr m_base_ptr; + Ptr m_loc_ptr; + int m_table_toggle; + + bool m_opengoal; void begin(Ptr object_file, const char* name, int32_t size, @@ -46,6 +54,7 @@ struct link_control { uint32_t flags); uint32_t work(); uint32_t work_v3(); + uint32_t work_v2(); void finish(); void reset() { @@ -66,11 +75,13 @@ struct link_control { } }; +// only used in OpenGOAL struct SegmentInfo { uint32_t offset; uint32_t size; }; +// only used in OpenGOAL struct ObjectFileHeader { uint16_t goal_version_major; uint16_t goal_version_minor; diff --git a/game/sce/sif_ee.cpp b/game/sce/sif_ee.cpp index b37ab58d54..5fbe9de35a 100644 --- a/game/sce/sif_ee.cpp +++ b/game/sce/sif_ee.cpp @@ -1,5 +1,8 @@ #include #include +#include +#include +#include "common/util/FileUtil.h" #include "sif_ee.h" #include "game/system/iop_thread.h" #include "game/runtime.h" @@ -8,7 +11,8 @@ namespace ee { namespace { ::IOP* iop; -} +std::unordered_map sce_fds; +} // namespace void LIBRARY_sceSif_register(::IOP* i) { iop = i; @@ -16,6 +20,10 @@ void LIBRARY_sceSif_register(::IOP* i) { void LIBRARY_INIT_sceSif() { iop = nullptr; + for (auto& kv : sce_fds) { + fclose(kv.second); + } + sce_fds.clear(); } void sceSifInitRpc(unsigned int mode) { (void)mode; @@ -91,4 +99,81 @@ s32 sceSifBindRpc(sceSifClientData* bd, u32 request, u32 mode) { bd->serve = (sceSifServeData*)1; return 0; } + +s32 sceOpen(const char* filename, s32 flag) { + auto name = file_util::get_file_path({filename}); + FILE* fp = nullptr; + switch (flag) { + case SCE_RDONLY: + fp = fopen(name.c_str(), "r"); + break; + default: + assert(false); + } + + if (!fp) { + printf("[SCE] sceOpen(%s) failed.\n", name.c_str()); + return -1; + } + + s32 fp_idx = sce_fds.size() + 1; + sce_fds[fp_idx] = fp; + return fp_idx; +} + +s32 sceClose(s32 fd) { + if (fd < 0) { + // todo, what should we really return? + return 0; + } + + auto kv = sce_fds.find(fd); + if (kv != sce_fds.end()) { + fclose(kv->second); + sce_fds.erase(fd); + return 0; + } else { + printf("[SCE] sceClose called on invalid fd\n"); + return 0; + } +} + +s32 sceRead(s32 fd, void* buf, s32 nbyte) { + auto kv = sce_fds.find(fd); + if (kv == sce_fds.end()) { + return -1; + } else { + return fread(buf, 1, nbyte, kv->second); + } +} + +s32 sceWrite(s32 fd, const void* buf, s32 nbyte) { + (void)fd; + (void)buf; + (void)nbyte; + assert(false); + return 0; +} + +s32 sceLseek(s32 fd, s32 offset, s32 where) { + auto kv = sce_fds.find(fd); + if (kv == sce_fds.end()) { + return -1; + } else { + switch (where) { + case SCE_SEEK_CUR: + fseek(kv->second, offset, SEEK_CUR); + return ftell(kv->second); + case SCE_SEEK_END: + fseek(kv->second, offset, SEEK_END); + return ftell(kv->second); + case SCE_SEEK_SET: + fseek(kv->second, offset, SEEK_SET); + return ftell(kv->second); + default: + assert(false); + } + } +} + } // namespace ee \ No newline at end of file diff --git a/game/sce/sif_ee.h b/game/sce/sif_ee.h index a027035905..51694db95a 100644 --- a/game/sce/sif_ee.h +++ b/game/sce/sif_ee.h @@ -49,5 +49,32 @@ s32 sceSifCallRpc(sceSifClientData* bd, s32 sceSifCheckStatRpc(sceSifRpcData* bd); s32 sceSifBindRpc(sceSifClientData* bd, u32 request, u32 mode); +#ifndef SCE_SEEK_SET +#define SCE_SEEK_SET (0) +#endif +#ifndef SCE_SEEK_CUR +#define SCE_SEEK_CUR (1) +#endif +#ifndef SCE_SEEK_END +#define SCE_SEEK_END (2) +#endif + +#define SCE_RDONLY 0x0001 +#define SCE_WRONLY 0x0002 +#define SCE_RDWR 0x0003 +#define SCE_NBLOCK 0x0010 +#define SCE_APPEND 0x0100 +#define SCE_CREAT 0x0200 +#define SCE_TRUNC 0x0400 +#define SCE_EXCL 0x0800 +#define SCE_NOBUF 0x4000 +#define SCE_NOWAIT 0x8000 + +s32 sceOpen(const char* filename, s32 flag); +s32 sceClose(s32 fd); +s32 sceRead(s32 fd, void* buf, s32 nbyte); +s32 sceWrite(s32 fd, const void* buf, s32 nbyte); +s32 sceLseek(s32 fd, s32 offset, s32 where); + } // namespace ee #endif // JAK1_SIF_EE_H diff --git a/game/sce/stubs.cpp b/game/sce/stubs.cpp index 67cb9bd1b2..6ea0765463 100644 --- a/game/sce/stubs.cpp +++ b/game/sce/stubs.cpp @@ -3,38 +3,6 @@ #include "stubs.h" namespace ee { -s32 sceOpen(const char* filename, s32 flag) { - (void)filename; - (void)flag; - throw std::runtime_error("sceOpen NYI"); -} - -s32 sceClose(s32 fd) { - (void)fd; - throw std::runtime_error("sceClose NYI"); -} - -s32 sceRead(s32 fd, void* buf, s32 nbyte) { - (void)fd; - (void)buf; - (void)nbyte; - throw std::runtime_error("sceRead NYI"); -} - -s32 sceWrite(s32 fd, const void* buf, s32 nbyte) { - (void)fd; - (void)buf; - (void)nbyte; - throw std::runtime_error("sceWrite NYI"); -} - -s32 sceLseek(s32 fd, s32 offset, s32 where) { - (void)fd; - (void)offset; - (void)where; - throw std::runtime_error("sceLseek NYI"); -} - int scePadPortOpen(int port, int slot, void* data) { (void)port; (void)slot; diff --git a/game/sce/stubs.h b/game/sce/stubs.h index 91d1b66b3a..0c4a59b459 100644 --- a/game/sce/stubs.h +++ b/game/sce/stubs.h @@ -5,35 +5,9 @@ #include "common/common_types.h" -#ifndef SCE_SEEK_SET -#define SCE_SEEK_SET (0) -#endif -#ifndef SCE_SEEK_CUR -#define SCE_SEEK_CUR (1) -#endif -#ifndef SCE_SEEK_END -#define SCE_SEEK_END (2) -#endif - -#define SCE_RDONLY 0x0001 -#define SCE_WRONLY 0x0002 -#define SCE_RDWR 0x0003 -#define SCE_NBLOCK 0x0010 -#define SCE_APPEND 0x0100 -#define SCE_CREAT 0x0200 -#define SCE_TRUNC 0x0400 -#define SCE_EXCL 0x0800 -#define SCE_NOBUF 0x4000 -#define SCE_NOWAIT 0x8000 - #define SCE_PAD_DMA_BUFFER_SIZE 0x100 namespace ee { -s32 sceOpen(const char* filename, s32 flag); -s32 sceClose(s32 fd); -s32 sceRead(s32 fd, void* buf, s32 nbyte); -s32 sceWrite(s32 fd, const void* buf, s32 nbyte); -s32 sceLseek(s32 fd, s32 offset, s32 where); void sceGsSyncV(); void sceGsSyncPath(); void sceGsResetPath(); diff --git a/goal_src/build/all_objs.txt b/goal_src/build/all_objs.txt index 2d30c3fa68..c1b4d7d693 100644 --- a/goal_src/build/all_objs.txt +++ b/goal_src/build/all_objs.txt @@ -1084,4 +1084,12 @@ ["vil3-bridge-36-ag", "vil3-bridge-36", 4, ["VI3"], "levels/village3"], ["water-anim-village3-ag", "water-anim-village3", 4, ["VI3"], "levels/village3"], ["village3-vis", "village3-vis", 4, ["VI3"], "levels/village3"], -["lava", "lava", 3, ["WATER-AN"], "old/lava"]] \ No newline at end of file +["lava", "lava", 3, ["WATER-AN"], "old/lava"], +["0COMMON", "0COMMON", 4, ["NO-XGO"], ""], +["1COMMON", "1COMMON", 4, ["NO-XGO"], ""], +["2COMMON", "2COMMON", 4, ["NO-XGO"], ""], +["3COMMON", "3COMMON", 4, ["NO-XGO"], ""], +["4COMMON", "4COMMON", 4, ["NO-XGO"], ""], +["5COMMON", "5COMMON", 4, ["NO-XGO"], ""], +["6COMMON", "6COMMON", 4, ["NO-XGO"], ""] +] \ No newline at end of file diff --git a/goal_src/engine/ui/text-h.gc b/goal_src/engine/ui/text-h.gc index f6ff5caaab..d49a68ff3a 100644 --- a/goal_src/engine/ui/text-h.gc +++ b/goal_src/engine/ui/text-h.gc @@ -5,3 +5,39 @@ ;; name in dgo: text-h ;; dgos: GAME, ENGINE +;; This file contains types related to game text. +;; Each game string is assigned an ID number. +;; This ID is used to lookup the string for the currently selected language. + +;; an individual string. +(deftype game-text (structure) + ((id uint32 :offset-assert 0) + (text string :offset-assert 4) + ) + :pack-me + :method-count-assert 9 + :size-assert #x8 + :flag-assert #x900000008 + ) + +;; A table of all strings. +(deftype game-text-info (basic) + ((length int32 :offset-assert 4) + (language-id int32 :offset-assert 8) + (group-name string :offset-assert 12) + (data game-text :dynamic :inline :offset-assert 16) + ) + :method-count-assert 10 + :size-assert #x10 + :flag-assert #xa00000010 + (:methods + (dummy-9 () none 9) + ) + ) + +;; todo, need support for array +;(define *text-group-names* #("common")) + +(define *common-text-heap* (new 'global 'kheap)) +;; probably some other type. +(define *common-text* #f) \ No newline at end of file diff --git a/goal_src/engine/ui/text.gc b/goal_src/engine/ui/text.gc index e6ebb40056..0f8e65b8f2 100644 --- a/goal_src/engine/ui/text.gc +++ b/goal_src/engine/ui/text.gc @@ -5,3 +5,51 @@ ;; name in dgo: text ;; dgos: GAME, ENGINE +(define *game-text-word* (new 'global 'string 256 (the string '#f))) +(define *game-text-line* (new 'global 'string 256 (the string '#f))) +(define *level-text-file-load-flag* '#t) + +;; allocate the game text heap if it isn't already allocated. +(when (= 0 (-> *common-text-heap* base)) + (let ((heap *common-text-heap*)) + (set! (-> heap base) (malloc 'global 34816)) + (set! (-> heap current) (-> heap base)) + (set! (-> heap top-base) (&+ (-> heap base) 34816)) + (set! (-> heap top) (-> heap top-base)) + ) + ) + +(defmethod length game-text-info ((obj game-text-info)) + "Get the length (number of strings) in a game-text-info." + (-> obj length) + ) + +(defmethod asize-of game-text-info ((obj game-text-info)) + (the int (+ (-> obj type size) (* 8 (-> obj length)))) + ) + +(defmethod inspect game-text-info ((obj game-text-info)) + (format '#t "[~8x] ~A~%" obj (-> obj type)) + (format '#t "~Tlength: ~D~%" (-> obj length)) + (format '#t "~Tdata[~D]: @ #x~X~%" (-> obj length) (-> obj data)) + + (let ((i 0)) + (while (< i (-> obj length)) + (format '#t "~T [~D] #x~X ~A~%" i (-> obj data i id) (-> obj data i text)) + (+! i 1) + ) + ) + obj + ) + +;; todo method 8 +;; todo method 9 +;; todo text-is-loading +;; todo load-game-text-info +;; todo load-level-text-files +;; todo draw-debug-text-box +;; todo set-font-color-alpha +;; todo print-game-text-scaled +;; todo print-game-text +;; todo disable-level-text-file-loading +;; todo enable-level-text-file-loading diff --git a/goal_src/goal-lib.gc b/goal_src/goal-lib.gc index c932cb985c..bd6aa02549 100644 --- a/goal_src/goal-lib.gc +++ b/goal_src/goal-lib.gc @@ -33,6 +33,12 @@ ) ) +(defmacro build-data () + `(begin + (asm-data-file game-text "assets/game_text.txt") + ) + ) + (defmacro blg () `(begin (build-game) diff --git a/goalc/CMakeLists.txt b/goalc/CMakeLists.txt index 4c7e6b4f31..bd3d4471e6 100644 --- a/goalc/CMakeLists.txt +++ b/goalc/CMakeLists.txt @@ -24,6 +24,8 @@ add_library(compiler compiler/compilation/Type.cpp compiler/compilation/Static.cpp compiler/Util.cpp + data_compiler/game_text.cpp + data_compiler/DataObjectGenerator.cpp debugger/Debugger.cpp debugger/DebugInfo.cpp logger/Logger.cpp @@ -36,12 +38,13 @@ add_library(compiler compiler/Compiler.cpp) add_executable(goalc main.cpp) +add_executable(data_compiler data_compiler.cpp) IF (WIN32) target_link_libraries(compiler goos type_system mman common_util spdlog cross_os_debug cross_sockets Zydis) - ELSE () target_link_libraries(compiler goos type_system common_util spdlog cross_os_debug cross_sockets Zydis) ENDIF () target_link_libraries(goalc goos compiler type_system) +target_link_libraries(data_compiler goos compiler type_system) \ No newline at end of file diff --git a/goalc/compiler/Compiler.cpp b/goalc/compiler/Compiler.cpp index 31f70707c1..029fb5cec2 100644 --- a/goalc/compiler/Compiler.cpp +++ b/goalc/compiler/Compiler.cpp @@ -251,6 +251,11 @@ bool Compiler::connect_to_target() { return true; } +void Compiler::run_front_end_on_string(const std::string& src) { + auto code = m_goos.reader.read_from_string({src}); + compile_object_file("run-on-string", code, true); +} + std::vector Compiler::run_test_no_load(const std::string& source_code) { auto code = m_goos.reader.read_from_file({source_code}); compile_object_file("test-code", code, true); diff --git a/goalc/compiler/Compiler.h b/goalc/compiler/Compiler.h index 0cd88098a1..f285d12be5 100644 --- a/goalc/compiler/Compiler.h +++ b/goalc/compiler/Compiler.h @@ -34,6 +34,7 @@ class Compiler { std::vector run_test_from_string(const std::string& src, const std::string& obj_name = "*listener*"); std::vector run_test_no_load(const std::string& source_code); + void run_front_end_on_string(const std::string& src); void shutdown_target(); void enable_throw_on_redefines() { m_throw_on_define_extern_redefinition = true; } Debugger& get_debugger() { return m_debugger; } @@ -54,6 +55,7 @@ class Compiler { Val* compile_pair(const goos::Object& code, Env* env); Val* compile_integer(const goos::Object& code, Env* env); Val* compile_integer(s64 value, Env* env); + Val* compile_char(const goos::Object& code, Env* env); Val* compile_float(const goos::Object& code, Env* env); Val* compile_float(float value, Env* env, int seg); Val* compile_symbol(const goos::Object& form, Env* env); @@ -173,6 +175,7 @@ class Compiler { Val* compile_seval(const goos::Object& form, const goos::Object& rest, Env* env); Val* compile_exit(const goos::Object& form, const goos::Object& rest, Env* env); Val* compile_asm_file(const goos::Object& form, const goos::Object& rest, Env* env); + Val* compile_asm_data_file(const goos::Object& form, const goos::Object& rest, Env* env); Val* compile_listen_to_target(const goos::Object& form, const goos::Object& rest, Env* env); Val* compile_reset_target(const goos::Object& form, const goos::Object& rest, Env* env); Val* compile_poke(const goos::Object& form, const goos::Object& rest, Env* env); diff --git a/goalc/compiler/compilation/Atoms.cpp b/goalc/compiler/compilation/Atoms.cpp index 9da4da6fc4..56b0543002 100644 --- a/goalc/compiler/compilation/Atoms.cpp +++ b/goalc/compiler/compilation/Atoms.cpp @@ -33,6 +33,7 @@ static const std::unordered_map< {"gs", &Compiler::compile_gs}, {":exit", &Compiler::compile_exit}, {"asm-file", &Compiler::compile_asm_file}, + {"asm-data-file", &Compiler::compile_asm_data_file}, {"listen-to-target", &Compiler::compile_listen_to_target}, {"reset-target", &Compiler::compile_reset_target}, {":status", &Compiler::compile_poke}, @@ -139,6 +140,8 @@ Val* Compiler::compile(const goos::Object& code, Env* env) { return compile_pair(code, env); case goos::ObjectType::INTEGER: return compile_integer(code, env); + case goos::ObjectType::CHAR: + return compile_char(code, env); case goos::ObjectType::SYMBOL: return compile_symbol(code, env); case goos::ObjectType::STRING: @@ -194,6 +197,11 @@ Val* Compiler::compile_integer(const goos::Object& code, Env* env) { return compile_integer(code.integer_obj.value, env); } +Val* Compiler::compile_char(const goos::Object& code, Env* env) { + assert(code.is_char()); + return compile_integer(uint8_t(code.char_obj.value), env); +} + /*! * Compile an integer constant. Returns an IntegerConstantVal and emits no code. * These integer constants do not generate static data and are stored directly in the code diff --git a/goalc/compiler/compilation/CompilerControl.cpp b/goalc/compiler/compilation/CompilerControl.cpp index 4f1b91a884..dcfe992f08 100644 --- a/goalc/compiler/compilation/CompilerControl.cpp +++ b/goalc/compiler/compilation/CompilerControl.cpp @@ -8,6 +8,7 @@ #include "common/util/Timer.h" #include "common/util/DgoWriter.h" #include "common/util/FileUtil.h" +#include "goalc/data_compiler/game_text.h" /*! * Exit the compiler. Disconnects the listener and tells the target to reset itself. @@ -46,6 +47,22 @@ Val* Compiler::compile_seval(const goos::Object& form, const goos::Object& rest, return get_none(); } +/*! + * Compile a "data file" + */ +Val* Compiler::compile_asm_data_file(const goos::Object& form, const goos::Object& rest, Env* env) { + (void)env; + auto args = get_va(form, rest); + va_check(form, args, {goos::ObjectType::SYMBOL, goos::ObjectType::STRING}, {}); + auto kind = symbol_string(args.unnamed.at(0)); + if (kind == "game-text") { + compile_game_text(as_string(args.unnamed.at(1))); + } else { + throw_compile_error(form, "Unknown asm data file mode"); + } + return get_none(); +} + /*! * Compile a file, and optionally color, save, or load. * This should only be used for v3 "code object" files. @@ -143,7 +160,7 @@ Val* Compiler::compile_asm_file(const goos::Object& form, const goos::Object& re printf("F: %36s ", obj_file_name.c_str()); timing.emplace_back("total", total_timer.getMs()); for (auto& e : timing) { - printf(" %12s %4.2f", e.first.c_str(), e.second / 1000.f); + printf(" %12s %4.0f", e.first.c_str(), e.second); } printf("\n"); } diff --git a/goalc/compiler/compilation/Type.cpp b/goalc/compiler/compilation/Type.cpp index 0d4a24215f..d79dec483c 100644 --- a/goalc/compiler/compilation/Type.cpp +++ b/goalc/compiler/compilation/Type.cpp @@ -473,7 +473,13 @@ Val* Compiler::compile_deref(const goos::Object& form, const goos::Object& _rest } if (result->type().base_type() == "inline-array") { - assert(false); + auto di = m_ts.get_deref_info(result->type()); + auto base_type = di.result_type; + assert(di.can_deref); + auto offset = compile_integer(di.stride, env)->to_gpr(env); + // todo, check for integer and avoid runtime multiply + env->emit(std::make_unique(IntegerMathKind::IMUL_32, offset, index_value)); + result = fe->alloc_val(di.result_type, result, offset); } else if (result->type().base_type() == "pointer") { auto di = m_ts.get_deref_info(result->type()); auto base_type = di.result_type; diff --git a/goalc/data_compiler.cpp b/goalc/data_compiler.cpp new file mode 100644 index 0000000000..ffd7632730 --- /dev/null +++ b/goalc/data_compiler.cpp @@ -0,0 +1,46 @@ +#include +#include "goalc/compiler/Compiler.h" +#include "common/versions.h" +#include "third-party/spdlog/include/spdlog/spdlog.h" +#include "third-party/spdlog/include/spdlog/sinks/basic_file_sink.h" +#include "third-party/spdlog/include/spdlog/sinks/stdout_color_sinks.h" + +void setup_logging(bool verbose) { + spdlog::set_level(spdlog::level::debug); + if (verbose) { + auto game_logger = spdlog::stdout_color_mt("GOAL Compiler: Data Mode"); + spdlog::set_default_logger(game_logger); + spdlog::flush_on(spdlog::level::info); + spdlog::set_pattern("%v"); + spdlog::info("Verbose logging enabled"); + } else { + auto game_logger = spdlog::basic_logger_mt("GOAL Compiler", "logs/data_compiler.log"); + spdlog::set_default_logger(game_logger); + spdlog::flush_on(spdlog::level::debug); + printf("OpenGOAL Compiler %d.%d: Data Mode\n", versions::GOAL_VERSION_MAJOR, + versions::GOAL_VERSION_MINOR); + } +} + +int main(int argc, char** argv) { + (void)argc; + (void)argv; + + bool verbose = false; + for (int i = 1; i < argc; i++) { + if (std::string("-v") == argv[i]) { + verbose = true; + break; + } + } + setup_logging(verbose); + + spdlog::info("OpenGOAL Compiler {}.{}: Data Mode", versions::GOAL_VERSION_MAJOR, + versions::GOAL_VERSION_MINOR); + + Compiler compiler; + compiler.run_front_end_on_string("(build-data)"); + + printf("Done!\n"); + return 0; +} diff --git a/goalc/data_compiler/DataObjectGenerator.cpp b/goalc/data_compiler/DataObjectGenerator.cpp new file mode 100644 index 0000000000..3c1412a81f --- /dev/null +++ b/goalc/data_compiler/DataObjectGenerator.cpp @@ -0,0 +1,222 @@ +#include +#include +#include +#include "DataObjectGenerator.h" +#include "common/link_types.h" + +namespace { +template +void add_data_to_vector(const T& data, std::vector* vec) { + auto loc = vec->size(); + vec->resize(loc + sizeof(T)); + memcpy(vec->data() + loc, &data, sizeof(T)); +} + +void push_variable_length_integer(u32 value, std::vector* vec) { + while (value > UINT8_MAX) { + vec->push_back(UINT8_MAX); + value -= UINT8_MAX; + } + + if (value == UINT8_MAX) { + vec->push_back(UINT8_MAX); + vec->push_back(0); + } else { + vec->push_back(value); + } +} + +void push_better_variable_length_integer(u32 value, std::vector* vec) { + if (value > 0xffffff) { + vec->push_back((value & 0xff) | 3); + vec->push_back((value >> 8) & 0xff); + vec->push_back((value >> 16) & 0xff); + vec->push_back((value >> 24) & 0xff); + } else if (value > 0xffff) { + vec->push_back((value & 0xff) | 2); + vec->push_back((value >> 8) & 0xff); + vec->push_back((value >> 16) & 0xff); + } else if (value > 0xff) { + vec->push_back((value & 0xff) | 1); + vec->push_back((value >> 8) & 0xff); + } else { + vec->push_back(value & 0xff); + } +} +} // namespace + +int DataObjectGenerator::add_word(u32 word) { + auto result = int(m_words.size()); + m_words.push_back(word); + return result; +} + +void DataObjectGenerator::link_word_to_word(int source, int target, int offset) { + link_word_to_byte(source, target * 4 + offset); +} + +void DataObjectGenerator::link_word_to_byte(int source_word, int target_byte) { + PointerLinkRecord rec; + rec.source_word = source_word; + rec.target_byte = target_byte; + m_ptr_links.push_back(rec); +} + +int DataObjectGenerator::add_ref_to_string_in_pool(const std::string& str) { + auto result = int(m_words.size()); + m_words.push_back(0); + m_string_pool[str].push_back(result); + return result; +} + +int DataObjectGenerator::add_type_tag(const std::string& str) { + auto result = int(m_words.size()); + m_words.push_back(0); + m_type_links[str].push_back(result); + return result; +} + +int DataObjectGenerator::add_symbol_link(const std::string& str) { + auto result = int(m_words.size()); + m_words.push_back(0); + m_symbol_links[str].push_back(result); + return result; +} + +void DataObjectGenerator::align(int alignment_words) { + while (m_words.size() % alignment_words) { + m_words.push_back(0); + } +} + +int DataObjectGenerator::words() const { + return int(m_words.size()); +} + +std::vector DataObjectGenerator::generate_v2() { + // add string data at the end. + add_strings(); + + // Generate the link table. + std::vector link = generate_link_table(); + + // add words + + // header + LinkHeaderV2 header; + header.type_tag = 0xffffffff; + header.version = 2; + header.length = sizeof(LinkHeaderV2) + link.size(); + + // build + std::vector result; + add_data_to_vector(header, &result); + result.insert(result.end(), link.begin(), link.end()); + + auto start = result.size(); + result.resize(result.size() + m_words.size() * 4); + memcpy(result.data() + start, m_words.data(), m_words.size() * 4); + + while (result.size() % 16) { + result.push_back(0); + } + + return result; +} + +std::vector DataObjectGenerator::generate_link_table() { + std::vector link; + + // pointer links are in source order. + std::sort(m_ptr_links.begin(), m_ptr_links.end(), + [](const PointerLinkRecord& a, const PointerLinkRecord& b) { + return a.source_word < b.source_word; + }); + + int i = 0; + + u32 last_word = 0; + while (i < int(m_ptr_links.size())) { + // seeking + auto& entry = m_ptr_links.at(i); + int diff = int(entry.source_word) - int(last_word); + last_word = entry.source_word + 1; + assert(diff >= 0); + push_variable_length_integer(diff, &link); + m_words.at(entry.source_word) = entry.target_byte; + + // count. + int consecutive = 1; + for (;;) { + if (i + 1 < int(m_ptr_links.size()) && + m_ptr_links.at(i + 1).source_word == m_ptr_links.at(i).source_word + 1) { + m_words.at(m_ptr_links.at(i + 1).source_word) = m_ptr_links.at(i + 1).target_byte; + last_word = m_ptr_links.at(i + 1).source_word + 1; + consecutive++; + i++; + } else { + break; + } + } + + push_variable_length_integer(consecutive, &link); + i++; + } + push_variable_length_integer(0, &link); + + // todo symbols + assert(m_symbol_links.empty()); + + // types + for (auto& tl : m_type_links) { + link.push_back(0x80); + for (auto c : tl.first) { + link.push_back(c); + } + link.push_back(0); + + std::sort(tl.second.begin(), tl.second.end()); + int prev = 0; + + for (auto& x : tl.second) { + int diff = x - prev; + assert(diff >= 0); + push_better_variable_length_integer(diff * 4, &link); + m_words.at(x) = 0xffffffff; + prev = x; + } + link.push_back(0); + } + push_variable_length_integer(0, &link); + + // align to 16 bytes for data start! + while ((link.size() + sizeof(LinkHeaderV2)) % 64) { + link.push_back(0); + } + return link; +} + +void DataObjectGenerator::add_strings() { + for (auto& entry : m_string_pool) { + // add the string + align(4); + add_type_tag("string"); + auto target_word = add_word(entry.first.length()); + std::vector string_buff; + for (auto c : entry.first) { + string_buff.push_back(c); + } + string_buff.push_back(0); + while (string_buff.size() & 3) { + string_buff.push_back(0); + } + + for (int i = 0; i < int(string_buff.size()) / 4; i++) { + add_word(*(u32*)(string_buff.data() + i * 4)); + } + + for (auto& source : entry.second) { + link_word_to_word(source, target_word); + } + } +} \ No newline at end of file diff --git a/goalc/data_compiler/DataObjectGenerator.h b/goalc/data_compiler/DataObjectGenerator.h new file mode 100644 index 0000000000..74017ecb6a --- /dev/null +++ b/goalc/data_compiler/DataObjectGenerator.h @@ -0,0 +1,36 @@ +#pragma once + +#include "common/common_types.h" +#include +#include +#include + +class DataObjectGenerator { + public: + int add_word(u32 word); + void link_word_to_word(int source, int target, int offset = 0); + void link_word_to_byte(int source_word, int target_byte); + int add_ref_to_string_in_pool(const std::string& str); + int add_type_tag(const std::string& str); + int add_symbol_link(const std::string& str); + std::vector generate_v2(); + void align(int alignment_words); + int words() const; + + private: + void add_strings(); + std::vector generate_link_table(); + + struct PointerLinkRecord { + int source_word; + int target_byte; + }; + + std::map> m_string_pool; + std::vector m_words; + std::vector m_ptr_links; + + // both alphabetical. + // symbols before types. + std::map> m_type_links, m_symbol_links; +}; diff --git a/goalc/data_compiler/game_text.cpp b/goalc/data_compiler/game_text.cpp new file mode 100644 index 0000000000..857731d07a --- /dev/null +++ b/goalc/data_compiler/game_text.cpp @@ -0,0 +1,220 @@ +/*! + * @file game_text.cpp + * Builds the XCOMMON.TXT text files. Each file contains all the strings that appear in the game + * translated into a language. + * + * The decompiler/data/game_text.cpp file extracts text from the game and creates a file that + * can be read with these functions. + */ + +#include +#include "game_text.h" +#include "common/goos/Reader.h" +#include "DataObjectGenerator.h" +#include "common/util/FileUtil.h" +#include "third-party/fmt/core.h" + +namespace { +template +void for_each_in_list(const goos::Object& list, const T& f) { + const goos::Object* iter = &list; + while (iter->is_pair()) { + auto lap = iter->as_pair(); + f(lap->car); + iter = &lap->cdr; + } + + if (!iter->is_empty_list()) { + throw std::runtime_error("Invalid list"); + } +} + +int64_t get_int(const goos::Object& obj) { + if (obj.is_int()) { + return obj.integer_obj.value; + } + throw std::runtime_error(obj.print() + " was supposed to be an integer, but isn't"); +} + +const goos::Object& car(const goos::Object& x) { + if (!x.is_pair()) { + throw std::runtime_error("invalid pair"); + } + + return x.as_pair()->car; +} + +const goos::Object& cdr(const goos::Object& x) { + if (!x.is_pair()) { + throw std::runtime_error("invalid pair"); + } + + return x.as_pair()->cdr; +} + +std::string get_string(const goos::Object& x) { + if (x.is_string()) { + return x.as_string()->data; + } + throw std::runtime_error(x.print() + " was supposed to be an string, but isn't"); +} + +std::string uppercase(const std::string& in) { + std::string result; + result.reserve(in.size()); + for (auto c : in) { + if (c >= 'a' && c <= 'z') { + c -= ('a' - 'A'); + } + result.push_back(c); + } + return result; +} + +/*! + * Parse a game text file for all languages. + * The result is a vector> + * so result[lang_id][text_id] gets you the text in the given language. + * + * The file should begin with (language-count x) with the given number of languages. + * Each entry should be (text-id "text-in-lang-0" "text-in-lang-1" ... ) + * The text id's can be out of order or missing entries. + */ +std::vector> parse(const goos::Object& data, + std::string* group_name) { + std::vector> text; + bool languages_set = false; + bool group_name_set = false; + std::string possible_group_name; + + for_each_in_list(data.as_pair()->cdr, [&](const goos::Object& obj) { + if (obj.is_pair()) { + auto head = obj.as_pair()->car; + if (head.is_symbol() && head.as_symbol()->name == "language-count") { + if (languages_set) { + throw std::runtime_error("Languages has been set multiple times."); + } + + text.resize(get_int(car(cdr(obj)))); + if (!cdr(cdr(obj)).is_empty_list()) { + throw std::runtime_error("language-count has too many arguments"); + } + } else if (head.is_symbol() && head.as_symbol()->name == "group-name") { + if (group_name_set) { + throw std::runtime_error("group-name has been set multiple times."); + } + group_name_set = true; + + possible_group_name = get_string(car(cdr(obj))); + if (!cdr(cdr(obj)).is_empty_list()) { + throw std::runtime_error("group-name has too many arguments"); + } + } + + else if (head.is_int()) { + int i = 0; + int id = head.as_int(); + for_each_in_list(cdr(obj), [&](const goos::Object& entry) { + if (i >= int(text.size())) { + throw std::runtime_error( + "String has too many entries. There should be one per language"); + } + + if (entry.is_string()) { + auto& map = text.at(i); + if (map.find(id) != map.end()) { + throw std::runtime_error("Entry appears more than once"); + } + + map[id] = entry.as_string()->data; + } else { + throw std::runtime_error("Each entry must be a string"); + } + + i++; + }); + if (i != int(text.size())) { + throw std::runtime_error("String did not have an entry for each language"); + } + } else { + throw std::runtime_error("Invalid game text file entry: " + head.print()); + } + } else { + throw std::runtime_error("Invalid game text file"); + } + }); + + if (!group_name_set) { + throw std::runtime_error("group-name not set."); + } + *group_name = possible_group_name; + return text; +} + +/* +(deftype game-text (structure) + ((id uint32 :offset-assert 0) + (text basic :offset-assert 4) + ) + ) + +(deftype game-text-info (basic) + ((length int32 :offset-assert 4) + (language-id int32 :offset-assert 8) + (group-name basic :offset-assert 12) + (data game-text :dynamic :offset-assert 16) + ) + ) + */ + +/*! + * Write game text data to a file. Uses the V2 object format which is identical between GOAL and + * OpenGOAL, so this should produce exactly identical files to what is found in the game. + */ +void compile(const std::vector>& text, + const std::string& group_name) { + // get all text ID's we know + std::vector add_order; + add_order.reserve(text.front().size()); + for (auto& x : text.front()) { + add_order.push_back(x.first); + } + // and sort them to be added in order. This matches the game. + std::sort(add_order.begin(), add_order.end()); + + for (int lang = 0; lang < int(text.size()); lang++) { + DataObjectGenerator gen; + gen.add_type_tag("game-text-info"); // type + gen.add_word(text.front().size()); // length + gen.add_word(lang); // language-id + // this string is found in the string pool. + gen.add_ref_to_string_in_pool(group_name); // group-name + + // now add all the datas: + for (auto id : add_order) { + gen.add_word(id); // id + // these strings must be in the string pool, as sometimes there are duplicate + // strings in a single language, and these strings should be stored once and have multiple + // references to them. + gen.add_ref_to_string_in_pool(text.at(lang).at(id)); // text + } + auto data = gen.generate_v2(); + + file_util::write_binary_file( + file_util::get_file_path({"out", fmt::format("{}{}.TXT", lang, uppercase(group_name))}), + data.data(), data.size()); + } +} +} // namespace + +/*! + * Read a game text description file and generate GOAL objects. + */ +void compile_game_text(const std::string& filename) { + goos::Reader reader; + auto code = reader.read_from_file({filename}); + printf("[Build Game Text] %s\n", filename.c_str()); + std::string group_name; + auto text_map = parse(code, &group_name); + compile(text_map, group_name); +} diff --git a/goalc/data_compiler/game_text.h b/goalc/data_compiler/game_text.h new file mode 100644 index 0000000000..d2aa968f64 --- /dev/null +++ b/goalc/data_compiler/game_text.h @@ -0,0 +1,4 @@ +#pragma once +#include + +void compile_game_text(const std::string& filename); \ No newline at end of file diff --git a/goalc/emitter/ObjectGenerator.h b/goalc/emitter/ObjectGenerator.h index 5eaf9475e7..fb53be0676 100644 --- a/goalc/emitter/ObjectGenerator.h +++ b/goalc/emitter/ObjectGenerator.h @@ -15,7 +15,7 @@ #include "Instruction.h" #include "goalc/debugger/DebugInfo.h" -class FunctionDebugInfo; +struct FunctionDebugInfo; namespace emitter { diff --git a/offline_test_git_branch.sh b/offline_test_git_branch.sh new file mode 100755 index 0000000000..d5b84290eb --- /dev/null +++ b/offline_test_git_branch.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +set -e + +echo "=======================================" +echo "= Jak Project Offline Test =" +echo "=======================================" +echo "" +echo " ================= Cloning..." + +ISO_DATA_PATH=${1} +BRANCH_NAME=${2:-master} + +if [ -z "$1" ] + then + echo "Must supply path to iso data folder!" + exit 1 +fi + +echo " Branch: ${BRANCH_NAME}" +mkdir project +cd project +git clone https://github.com/water111/jak-project.git +cd jak-project +git checkout $BRANCH_NAME +git submodule update --init --recursive + +# create symlink to the iso_data folder. +rm -r iso_data +ln -s $ISO_DATA_PATH + + +mkdir build +cd build +echo " =============== Building..." +cmake .. +make -j + +echo " ================ Running unit tests..." +../test.sh + +echo " ================ Decompiling..." +../decomp.sh + +echo " ================ Building assets..." +./goalc/data_compiler + +echo " ================ Checking assets..." +../check.sh + +echo "Offline test has completed successfully!" diff --git a/out/.gitignore b/out/.gitignore index d6b7ef32c8..30bd40c3f4 100644 --- a/out/.gitignore +++ b/out/.gitignore @@ -1,2 +1,3 @@ * !.gitignore +!hash.md5 \ No newline at end of file diff --git a/out/hash.md5 b/out/hash.md5 new file mode 100644 index 0000000000..ed43d87721 --- /dev/null +++ b/out/hash.md5 @@ -0,0 +1,7 @@ +b06369c2dd9197cb17aae6286246caf9 0COMMON.TXT +da0d6012181f13803e8b3267a4099b97 1COMMON.TXT +4a9a27beb4ce7e75fa4c70811d2c0013 2COMMON.TXT +abcc25e5d7469dd6a572dc53dbb9671c 3COMMON.TXT +82eabdb7159f2059fbdbd18bb6fc06aa 4COMMON.TXT +5d62de2c78b4cf102b9a78f3aa96c8c9 5COMMON.TXT +9495f80955e6782513fe12f6539fc8e7 6COMMON.TXT diff --git a/test/goalc/source_templates/with_game/test-game-text.gc b/test/goalc/source_templates/with_game/test-game-text.gc new file mode 100644 index 0000000000..5c6c9a3dd4 --- /dev/null +++ b/test/goalc/source_templates/with_game/test-game-text.gc @@ -0,0 +1,13 @@ + +(start-test "game-text") + +(let ((text (the game-text-info (load "$JAK-PROJECT/out/0TEST.TXT" *common-text-heap*)))) + (format 0 "~I~%" text) + (expect-true (= #x123 (-> text data 0 id))) + (expect-true (= #x456 (-> text data 1 id))) + (expect-true (= #\e (-> text data 1 text data 1))) + (expect-true (= 5 (-> text data 1 text allocated-length))) + (expect-true (= 5 (length (-> text data 1 text)))) + ) + +(finish-test) \ No newline at end of file diff --git a/test/goalc/test_with_game.cpp b/test/goalc/test_with_game.cpp index 445e64b071..622baeb599 100644 --- a/test/goalc/test_with_game.cpp +++ b/test/goalc/test_with_game.cpp @@ -27,18 +27,16 @@ struct WithGameParam { class WithGameTests : public testing::TestWithParam { public: static void SetUpTestSuite() { - compiler.run_test_no_load("test/goalc/source_templates/with_game/test-build-game.gc"); - runtime_thread = std::thread((GoalTest::runtime_with_kernel)); - runner.c = &compiler; - - compiler.run_test_from_file("test/goalc/source_templates/with_game/test-load-game.gc"); - try { - compiler.run_test_from_file("test/goalc/source_templates/with_game/test-build-game.gc"); + compiler.run_test_no_load("test/goalc/source_templates/with_game/test-build-game.gc"); } catch (std::exception& e) { fprintf(stderr, "caught exception %s\n", e.what()); EXPECT_TRUE(false); } + runtime_thread = std::thread((GoalTest::runtime_with_kernel)); + runner.c = &compiler; + + compiler.run_test_from_file("test/goalc/source_templates/with_game/test-load-game.gc"); } static void TearDownTestSuite() { @@ -72,72 +70,199 @@ std::vector get_test_pass_string(const std::string& name, int count } } // namespace -// TODO - havn't done anything with these really yet -TEST_F(WithGameTests, All) { +TEST_F(WithGameTests, ReturnConstant) { runner.run_static_test(env, testCategory, "defun-return-constant.static.gc", {"12\n"}); +} + +TEST_F(WithGameTests, ReturnSymbol) { runner.run_static_test(env, testCategory, "defun-return-symbol.static.gc", {"42\n"}); +} + +TEST_F(WithGameTests, MinMax) { runner.run_static_test(env, testCategory, "test-min-max.gc", {"10\n"}); +} + +TEST_F(WithGameTests, BoxedFloat) { runner.run_static_test(env, testCategory, "test-bfloat.gc", {"data 1.2330 print 1.2330 type bfloat\n0\n"}); +} + +TEST_F(WithGameTests, BasicTypeCheck) { runner.run_static_test(env, testCategory, "test-basic-type-check.gc", {"#f#t#t#f#t#f#t#t\n0\n"}); +} + +TEST_F(WithGameTests, ConditionBoolean) { runner.run_static_test(env, testCategory, "test-condition-boolean.gc", {"4\n"}); +} + +TEST_F(WithGameTests, TypeType) { runner.run_static_test(env, testCategory, "test-type-type.gc", {"#t#f\n0\n"}); +} + +TEST_F(WithGameTests, AccessInlineArray) { runner.run_static_test(env, testCategory, "test-access-inline-array.gc", {"1.2345\n0\n"}); +} + +TEST_F(WithGameTests, FindParentMethod) { runner.run_static_test(env, testCategory, "test-find-parent-method.gc", {"\"test pass!\"\n0\n"}); +} + +TEST_F(WithGameTests, Ref) { runner.run_static_test(env, testCategory, "test-ref.gc", {"83\n"}); +} + +TEST_F(WithGameTests, PairASzie) { runner.run_static_test(env, testCategory, "test-pair-asize.gc", {"8\n"}); +} + +TEST_F(WithGameTests, Last) { runner.run_static_test(env, testCategory, "test-last.gc", {"d\n0\n"}); +} + +TEST_F(WithGameTests, Sort) { runner.run_static_test( env, testCategory, "test-sort.gc", {"(24 16 32 56 72 1234 -34 25 654)\n(1234 654 72 56 32 25 24 16 -34)\n0\n"}); +} + +TEST_F(WithGameTests, Sort2) { runner.run_static_test( env, testCategory, "test-sort-2.gc", {"(24 16 32 56 72 1234 -34 25 654)\n(-34 16 24 25 32 56 72 654 1234)\n0\n"}); +} + +TEST_F(WithGameTests, Sort3) { runner.run_static_test( env, testCategory, "test-sort-3.gc", {"(24 16 32 56 72 1234 -34 25 654)\n(-34 16 24 25 32 56 72 654 1234)\n0\n"}); +} + +TEST_F(WithGameTests, PairLength) { runner.run_static_test(env, testCategory, "test-pair-length.gc", {"6\n"}); +} + +TEST_F(WithGameTests, Member1) { runner.run_static_test(env, testCategory, "test-member-1.gc", {"(c d)\n0\n"}); +} + +TEST_F(WithGameTests, Member2) { runner.run_static_test(env, testCategory, "test-member-2.gc", {"#f\n0\n"}); +} + +TEST_F(WithGameTests, Assoc1) { runner.run_static_test(env, testCategory, "test-assoc-1.gc", {"w\n0\n"}); +} + +TEST_F(WithGameTests, Assoc2) { runner.run_static_test(env, testCategory, "test-assoc-2.gc", {"#f\n0\n"}); +} + +TEST_F(WithGameTests, Assoce1) { runner.run_static_test(env, testCategory, "test-assoce-1.gc", {"x\n0\n"}); +} + +TEST_F(WithGameTests, Assoce2) { runner.run_static_test(env, testCategory, "test-assoce-2.gc", {"x\n0\n"}); +} + +TEST_F(WithGameTests, Append) { runner.run_static_test(env, testCategory, "test-append.gc", {"(a b c d e)\n0\n"}); +} + +TEST_F(WithGameTests, DeleteList) { runner.run_static_test(env, testCategory, "test-delete-list.gc", {"(a b d e)\n0\n"}); +} + +TEST_F(WithGameTests, DeleteCar) { runner.run_static_test(env, testCategory, "test-delete-car.gc", {"((a . b) (e . f))\n#f\n0\n"}); +} + +TEST_F(WithGameTests, InsertCar) { runner.run_static_test(env, testCategory, "test-insert-cons.gc", {"((c . w) (a . b) (e . f))\n0\n"}); +} + +TEST_F(WithGameTests, InlineArrayClass) { runner.run_static_test(env, testCategory, "test-new-inline-array-class.gc", {"2824\n"}); +} + +TEST_F(WithGameTests, Memcpy) { runner.run_static_test(env, testCategory, "test-memcpy.gc", {"13\n"}); +} + +TEST_F(WithGameTests, Memset) { runner.run_static_test(env, testCategory, "test-memset.gc", {"11\n"}); +} + +TEST_F(WithGameTests, BintegerPrint) { runner.run_static_test(env, testCategory, "test-binteger-print.gc", {"-17\n0\n"}); +} + +TEST_F(WithGameTests, TestTests) { runner.run_static_test(env, testCategory, "test-tests.gc", {"Test Failed On Test 0: \"unknown\"\nTest Failed On Test 0: " "\"test\"\nTest \"test-of-test\": 1 Passes\n0\n"}); +} + +TEST_F(WithGameTests, TypeArrays) { runner.run_static_test(env, testCategory, "test-type-arrays.gc", {"Test \"test-type-arrays\": 3 Passes\n0\n"}); +} + +TEST_F(WithGameTests, NumberComparison) { runner.run_static_test(env, testCategory, "test-number-comparison.gc", {"Test \"number-comparison\": 14 Passes\n0\n"}); +} + +TEST_F(WithGameTests, ApproxPi) { runner.run_static_test(env, testCategory, "test-approx-pi.gc", get_test_pass_string("approx-pi", 4)); +} + +TEST_F(WithGameTests, DynamicType) { runner.run_static_test(env, testCategory, "test-dynamic-type.gc", get_test_pass_string("dynamic-type", 4)); +} + +TEST_F(WithGameTests, StringType) { runner.run_static_test(env, testCategory, "test-string-type.gc", get_test_pass_string("string-type", 4)); +} + +TEST_F(WithGameTests, NewString) { runner.run_static_test(env, testCategory, "test-new-string.gc", get_test_pass_string("new-string", 5)); +} + +TEST_F(WithGameTests, AddrOf) { runner.run_static_test(env, testCategory, "test-addr-of.gc", get_test_pass_string("addr-of", 2)); +} + +TEST_F(WithGameTests, SetSelf) { runner.run_static_test(env, testCategory, "test-set-self.gc", {"#t\n0\n"}); +} + +TEST_F(WithGameTests, NewArray) { runner.run_static_test(env, testCategory, "test-new-array.gc", get_test_pass_string("new-array", 8)); +} + +TEST_F(WithGameTests, NewStaticStructureIntegers) { runner.run_static_test(env, testCategory, "test-new-static-structure-integers.gc", get_test_pass_string("new-static-structure-integers", 7)); +} + +TEST_F(WithGameTests, NewStaticBasic) { runner.run_static_test(env, testCategory, "test-new-static-basic.gc", get_test_pass_string("new-static-basic", 9)); +} + +TEST_F(WithGameTests, VectorDot) { runner.run_static_test(env, testCategory, "test-vector-dot.gc", get_test_pass_string("vector-dot", 1)); +} +TEST_F(WithGameTests, DebuggerMemoryMap) { auto mem_map = compiler.listener().build_memory_map(); // we should have gkernel main segment @@ -147,7 +272,9 @@ TEST_F(WithGameTests, All) { EXPECT_TRUE(lookup_2.obj_name == "gkernel"); EXPECT_FALSE(lookup_2.empty); EXPECT_EQ(lookup_2.seg_id, MAIN_SEGMENT); +} +TEST_F(WithGameTests, DebuggerDisassemble) { auto di = compiler.get_debugger().get_debug_info_for_object("gcommon"); bool fail = false; auto result = di.disassemble_debug_functions(&fail); @@ -155,6 +282,12 @@ TEST_F(WithGameTests, All) { EXPECT_FALSE(fail); } +TEST_F(WithGameTests, GameText) { + compiler.run_test_from_string("(asm-data-file game-text \"test/test_data/test_game_text.txt\")"); + runner.run_static_test(env, testCategory, "test-game-text.gc", + get_test_pass_string("game-text", 5)); +} + TEST(TypeConsistency, TypeConsistency) { Compiler compiler; compiler.enable_throw_on_redefines(); diff --git a/test/test_data/test_game_text.txt b/test/test_data/test_game_text.txt new file mode 100644 index 0000000000..2d22520fa1 --- /dev/null +++ b/test/test_data/test_game_text.txt @@ -0,0 +1,10 @@ +(language-count 3) +(group-name "test") + +(#x123 "language 0" + "language 1" + "language 2") + +(#x456 "hello" + "goodbye" + "aaaaaaaa") \ No newline at end of file diff --git a/test/test_goos_file0.gs b/test/test_data/test_goos_file0.gs similarity index 100% rename from test/test_goos_file0.gs rename to test/test_data/test_goos_file0.gs diff --git a/test/test_reader_file0.gc b/test/test_data/test_reader_file0.gc similarity index 100% rename from test/test_reader_file0.gc rename to test/test_data/test_reader_file0.gc diff --git a/test/test_goos.cpp b/test/test_goos.cpp index 7b5216179a..a0346bfa10 100644 --- a/test/test_goos.cpp +++ b/test/test_goos.cpp @@ -48,7 +48,7 @@ TEST(GoosBuiltins, Read) { TEST(GoosBuiltins, ReadFile) { Interpreter i; // check that we can read a file. - EXPECT_EQ(e(i, "(read-file \"test/test_reader_file0.gc\")"), "(top-level (1 2 3 4))"); + EXPECT_EQ(e(i, "(read-file \"test/test_data/test_reader_file0.gc\")"), "(top-level (1 2 3 4))"); i.disable_printfs(); for (auto x : {"(read-file 1)", "(read-file)", "(read-file \"goal/test/not_a_real_file.gc\")"}) { @@ -59,7 +59,7 @@ TEST(GoosBuiltins, ReadFile) { TEST(GoosBuiltins, LoadFile) { Interpreter i; // check that we can read and execute a file. - e(i, "(load-file \"test/test_goos_file0.gs\")"); + e(i, "(load-file \"test/test_data/test_goos_file0.gs\")"); EXPECT_EQ(e(i, "x"), "23"); i.disable_printfs(); for (auto x : {"(load-file 1)", "(load-file)", "(load-file \"goal/test/not_a_real_file.gc\")"}) { diff --git a/test/test_reader.cpp b/test/test_reader.cpp index 06fcf9a82e..58fa4c113f 100644 --- a/test/test_reader.cpp +++ b/test/test_reader.cpp @@ -338,16 +338,19 @@ TEST(GoosReader, TopLevel) { TEST(GoosReader, FromFile) { Reader reader; - auto result = reader.read_from_file({"test", "test_reader_file0.gc"}).print(); + auto result = reader.read_from_file({"test", "test_data", "test_reader_file0.gc"}).print(); EXPECT_TRUE(result == "(top-level (1 2 3 4))"); } TEST(GoosReader, TextDb) { // very specific to this particular test file, but whatever. Reader reader; - auto result = - reader.read_from_file({"test", "test_reader_file0.gc"}).as_pair()->cdr.as_pair()->car; - std::string expected = "text from " + file_util::get_file_path({"test", "test_reader_file0.gc"}) + + auto result = reader.read_from_file({"test", "test_data", "test_reader_file0.gc"}) + .as_pair() + ->cdr.as_pair() + ->car; + std::string expected = "text from " + + file_util::get_file_path({"test", "test_data", "test_reader_file0.gc"}) + ", line: 5\n(1 2 3 4)\n"; EXPECT_EQ(expected, reader.db.get_info_for(result)); }