From 50230e05fae5d3583bc53d46fc7e267e48e86073 Mon Sep 17 00:00:00 2001 From: water111 <48171810+water111@users.noreply.github.com> Date: Sun, 23 Jul 2023 12:35:59 -0400 Subject: [PATCH] [jak2] Add static textures for the progress menu (#2838) The progress menu loads its icon textures from a .STR file that we were previously ignoring. This change: - updates the decompiler so it can process a .STR file containing a texture - adds a feature to force an entire page to always be loaded in the PC renderer by putting all textures in the GAME.FR3 file. - regenerates the texture offset map file for jak 2 with these new textures For now, I've just put the icon textures in GAME.FR3. The downside is that these will always stay on the GPU, using up VRAM even when they aren't needed. But the entire GAME.FR3 file is under 3 MB so I think it's ok. ![image](https://github.com/open-goal/jak-project/assets/48171810/39f075b5-7cc5-4168-872a-33026342afab) --- decompiler/ObjectFile/ObjectFileDB.cpp | 14 ++++++++++++ decompiler/ObjectFile/ObjectFileDB.h | 1 + decompiler/config.cpp | 10 +++++++++ decompiler/config.h | 2 ++ decompiler/config/jak2/ntsc_v1/inputs.jsonc | 11 ++++++++++ decompiler/data/StrFileReader.cpp | 23 +++++++++++++++++++- decompiler/data/StrFileReader.h | 13 +++++++++-- decompiler/extractor/main.cpp | 8 +++++-- decompiler/level_extractor/extract_level.cpp | 12 +++++++++- decompiler/main.cpp | 8 +++++-- game/graphics/jak2_texture_remap.cpp | 12 ++++++++++ test/offline/framework/orchestration.cpp | 6 ++--- 12 files changed, 109 insertions(+), 11 deletions(-) diff --git a/decompiler/ObjectFile/ObjectFileDB.cpp b/decompiler/ObjectFile/ObjectFileDB.cpp index b35dc7edeb..66deffca63 100644 --- a/decompiler/ObjectFile/ObjectFileDB.cpp +++ b/decompiler/ObjectFile/ObjectFileDB.cpp @@ -119,6 +119,7 @@ ObjectFileDB::ObjectFileDB(const std::vector& _dgos, const fs::path& obj_file_name_map_file, const std::vector& object_files, const std::vector& str_files, + const std::vector& str_tex_files, const Config& config) : dts(config.game_version), m_version(config.game_version) { Timer timer; @@ -223,6 +224,19 @@ ObjectFileDB::ObjectFileDB(const std::vector& _dgos, } } + if (!str_tex_files.empty()) { + lg::info("-Loading {} streaming texture files...", str_tex_files.size()); + for (auto& obj : str_tex_files) { + StrFileReader reader(obj, version()); + // name from the file name + std::string base_name = obj_filename_to_name(obj.string()); + ASSERT(reader.chunk_count() == 1); + auto name = reader.get_texture_name(); + add_obj_from_dgo(name, name, reader.get_chunk(0).data(), reader.get_chunk(0).size(), + "TEXSPOOL", config, name); + } + } + lg::info("ObjectFileDB Initialized"); if (obj_files_by_name.empty()) { lg::error( diff --git a/decompiler/ObjectFile/ObjectFileDB.h b/decompiler/ObjectFile/ObjectFileDB.h index 977070b2cb..fde699a2e1 100644 --- a/decompiler/ObjectFile/ObjectFileDB.h +++ b/decompiler/ObjectFile/ObjectFileDB.h @@ -163,6 +163,7 @@ class ObjectFileDB { const fs::path& obj_file_name_map_file, const std::vector& object_files, const std::vector& str_files, + const std::vector& str_tex_files, const Config& config); std::string generate_dgo_listing(); std::string generate_obj_listing(const std::unordered_set& merged_objs); diff --git a/decompiler/config.cpp b/decompiler/config.cpp index 8c1618f0f9..8145f69bbe 100644 --- a/decompiler/config.cpp +++ b/decompiler/config.cpp @@ -38,6 +38,12 @@ Config make_config_via_json(nlohmann::json& json) { config.dgo_names = inputs_json.at("dgo_names").get>(); config.object_file_names = inputs_json.at("object_file_names").get>(); config.str_file_names = inputs_json.at("str_file_names").get>(); + + if (inputs_json.contains("str_texture_file_names")) { + config.str_texture_file_names = + inputs_json.at("str_texture_file_names").get>(); + } + config.audio_dir_file_name = inputs_json.at("audio_dir_file_name").get(); config.streamed_audio_file_names = inputs_json.at("streamed_audio_file_names").get>(); @@ -272,6 +278,10 @@ Config make_config_via_json(nlohmann::json& json) { inputs_json.at("animated_textures").get>(); } + if (inputs_json.contains("common_tpages")) { + config.common_tpages = inputs_json.at("common_tpages").get>(); + } + auto art_info_json = read_json_file_from_config(json, "art_info_file"); config.art_groups_by_file = art_info_json.at("files").get>(); diff --git a/decompiler/config.h b/decompiler/config.h index f67303edeb..50531367a7 100644 --- a/decompiler/config.h +++ b/decompiler/config.h @@ -97,6 +97,7 @@ struct Config { std::vector dgo_names; std::vector object_file_names; std::vector str_file_names; + std::vector str_texture_file_names; std::string audio_dir_file_name; std::vector streamed_audio_file_names; @@ -160,6 +161,7 @@ struct Config { std::unordered_map bad_format_strings; std::unordered_set animated_textures; + std::unordered_set common_tpages; std::vector levels_to_extract; bool levels_extract; diff --git a/decompiler/config/jak2/ntsc_v1/inputs.jsonc b/decompiler/config/jak2/ntsc_v1/inputs.jsonc index 9fb6460658..d58b66d2f3 100644 --- a/decompiler/config/jak2/ntsc_v1/inputs.jsonc +++ b/decompiler/config/jak2/ntsc_v1/inputs.jsonc @@ -182,6 +182,12 @@ // "DGO/WASALL.DGO" ], + // STR files containing a texture that should be used by + // FR3 creation. + "str_texture_file_names" : [ + "STR/PRMINIMA.STR" + ], + // some objects are part of STR files (streaming data). "str_file_names": [ "STR/AT1INT.STR", @@ -443,6 +449,11 @@ "streamed_audio_file_names": [], + // tpages that should always be possible to access. + "common_tpages": [ + 3219 // PRMINIMA progress-minimap + ], + // Textures that should be extracted in a special way for use in animated textures. // these will be present in the common FR3, and stored as index textures. "animated_textures": [ diff --git a/decompiler/data/StrFileReader.cpp b/decompiler/data/StrFileReader.cpp index 4bca7ec12d..ab3d7c9b03 100644 --- a/decompiler/data/StrFileReader.cpp +++ b/decompiler/data/StrFileReader.cpp @@ -196,7 +196,7 @@ std::string StrFileReader::get_full_name(const std::string& short_name) const { bool done_first = false; // this string is part of the file info struct and the stuff after it is the file name. - const auto& file_info_string = get_file_info_string(); + const auto& file_info_string = get_art_group_file_info_string(); // it should occur in each chunk. int chunk_id = 0; @@ -241,4 +241,25 @@ std::string StrFileReader::get_full_name(const std::string& short_name) const { return result; } + +std::string StrFileReader::get_texture_name() const { + ASSERT(m_chunks.size() == 1); + const auto& chunk = m_chunks[0]; + auto find_string = get_texture_page_file_info_string(); + int offset; + if (find_string_in_data(chunk.data(), int(chunk.size()), find_string, &offset)) { + offset += find_string.length(); + } else { + ASSERT_MSG(false, fmt::format("did not find string '{}'", find_string)); + } + + for (int i = 0; i < 128; i++) { + if (chunk[offset + i] == '.') { + std::string result; + result.assign((const char*)&chunk[offset], i); + return result; + } + } + ASSERT_NOT_REACHED(); +} } // namespace decompiler diff --git a/decompiler/data/StrFileReader.h b/decompiler/data/StrFileReader.h index 2db7f612cd..714e34c8a4 100644 --- a/decompiler/data/StrFileReader.h +++ b/decompiler/data/StrFileReader.h @@ -20,20 +20,29 @@ class StrFileReader { int chunk_count() const; const std::vector& get_chunk(int idx) const; std::string get_full_name(const std::string& short_name) const; + std::string get_texture_name() const; private: void init_jak1(const fs::path& file_path); void init_jak2(const fs::path& file_path); GameVersion m_version; - const std::string get_file_info_string() const { + std::string get_art_group_file_info_string() const { switch (m_version) { case GameVersion::Jak1: return "/src/next/data/art-group6/"; - break; case GameVersion::Jak2: return "/src/jak2/final/art-group7/"; + default: + ASSERT_MSG(false, "NYI get_file_info_string version"); break; + } + } + + std::string get_texture_page_file_info_string() const { + switch (m_version) { + case GameVersion::Jak2: + return "/src/jak2/final/texture-page8/"; default: ASSERT_MSG(false, "NYI get_file_info_string version"); break; diff --git a/decompiler/extractor/main.cpp b/decompiler/extractor/main.cpp index 278828418f..d839c620dc 100644 --- a/decompiler/extractor/main.cpp +++ b/decompiler/extractor/main.cpp @@ -109,7 +109,7 @@ void decompile(const fs::path& iso_data_path, const std::string& data_subfolder) fmt::format("{}_config.jsonc", version_info.game_name), version_info.decomp_config_version); - std::vector dgos, objs; + std::vector dgos, objs, tex_strs; // grab all DGOS we need (level + common) // TODO - Jak 2 - jak 1 specific code? @@ -133,8 +133,12 @@ void decompile(const fs::path& iso_data_path, const std::string& data_subfolder) } } + for (const auto& str_name : config.str_texture_file_names) { + tex_strs.push_back(iso_data_path / str_name); + } + // set up objects - ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, {}, config); + ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, {}, tex_strs, config); // save object files auto out_folder = file_util::get_jak_project_dir() / "decompiler_out" / data_subfolder; diff --git a/decompiler/level_extractor/extract_level.cpp b/decompiler/level_extractor/extract_level.cpp index 94c8a4af4e..f405324a47 100644 --- a/decompiler/level_extractor/extract_level.cpp +++ b/decompiler/level_extractor/extract_level.cpp @@ -260,11 +260,21 @@ void extract_common(const ObjectFileDB& db, textures_we_have.insert(t.debug_name); } + for (const auto& [id, normal_texture] : tex_db.textures) { + if (config.common_tpages.count(normal_texture.page) && + !textures_we_have.count(normal_texture.name)) { + lg::info("Adding common texture {}", normal_texture.name); + textures_we_have.insert(normal_texture.name); + tfrag_level.textures.push_back( + make_texture(id, normal_texture, tex_db.tpage_names.at(normal_texture.page), true)); + } + } + // add animated textures that are missing. for (const auto& [id, normal_texture] : tex_db.textures) { if (config.animated_textures.count(normal_texture.name) && !textures_we_have.count(normal_texture.name)) { - lg::warn("adding anim normal texture {} ", normal_texture.name); + lg::info("Adding anim texture {}", normal_texture.name); textures_we_have.insert(normal_texture.name); tfrag_level.textures.push_back( make_texture(id, normal_texture, tex_db.tpage_names.at(normal_texture.page), false)); diff --git a/decompiler/main.cpp b/decompiler/main.cpp index 2f60492bd2..6abd6713c3 100644 --- a/decompiler/main.cpp +++ b/decompiler/main.cpp @@ -128,7 +128,7 @@ int main(int argc, char** argv) { mem_log("After init: {} MB\n", get_peak_rss() / (1024 * 1024)); - std::vector dgos, objs, strs; + std::vector dgos, objs, strs, tex_strs; for (const auto& dgo_name : config.dgo_names) { dgos.push_back(in_folder / dgo_name); } @@ -141,11 +141,15 @@ int main(int argc, char** argv) { strs.push_back(in_folder / str_name); } + for (const auto& str_name : config.str_texture_file_names) { + tex_strs.push_back(in_folder / str_name); + } + mem_log("After config read: {} MB", get_peak_rss() / (1024 * 1024)); // build file database lg::info("Setting up object file DB..."); - ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, strs, config); + ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, strs, tex_strs, config); // Explicitly fail if a file in the 'allowed_objects' list wasn't found in the DB // as this is another silent error that can be confusing diff --git a/game/graphics/jak2_texture_remap.cpp b/game/graphics/jak2_texture_remap.cpp index 192b891a1e..65d2113d27 100644 --- a/game/graphics/jak2_texture_remap.cpp +++ b/game/graphics/jak2_texture_remap.cpp @@ -1355,6 +1355,18 @@ const std::unordered_map>> data = { {{64, 1}, {67, 2}, {31, 1}, {32, 2}, {27, 1}, {69, 1}, {61, 1}, {62, 2}, {65, 3}, {94, 1}, {90, 1}, {91, 2}, {93, 3}, {86, 1}, {88, 2}, {83, 1}, {84, 2}, {92, 3}, {97, 1}, {99, 2}}}, {3214, {{8, 1}, {9, 2}, {13, 1}, {6, 1}}}, + {3219, + {{12, 1}, + {16, 1}, + {17, 2}, + {19, 3}, + {8, 2}, + {15, 3}, + {5, 1}, + {6, 2}, + {10, 1}, + {13, 2}, + {23, 3}}}, {3221, {{26, 2}, {72, 3}, {79, 4}, {99, 5}, {49, 1}, {55, 1}, {62, 1}, {63, 2}, {91, 1}, {16, 1}, {15, 2}, {17, 3}, {18, 4}, {105, 4}, {56, 1}, {106, 2}, {19, 1}, {27, 2}, diff --git a/test/offline/framework/orchestration.cpp b/test/offline/framework/orchestration.cpp index e615269a6e..a6ac895990 100644 --- a/test/offline/framework/orchestration.cpp +++ b/test/offline/framework/orchestration.cpp @@ -48,9 +48,9 @@ OfflineTestDecompiler setup_decompiler(const OfflineTestWorkGroup& work, dgo_paths.push_back(iso_data_path / x); } - dc.db = std::make_unique(dgo_paths, dc.config->obj_file_name_map_file, - std::vector{}, - std::vector{}, *dc.config); + dc.db = std::make_unique( + dgo_paths, dc.config->obj_file_name_map_file, std::vector{}, + std::vector{}, std::vector{}, *dc.config); dc.db->dts.art_group_info = dc.config->art_group_info_dump; std::unordered_set db_files;