From 1979f94e4563db879361e3003d631011c74e76cc Mon Sep 17 00:00:00 2001 From: Matt Dallmeyer Date: Mon, 29 Jan 2024 13:15:42 -0800 Subject: [PATCH] Rip collision based on config flag (#3348) Added a new `rip_collision` flag to the decompiler config and wired it up. Cleaned up the comments around these fields in the JSON too for clarity Also refactored a bunch of the related extract functions to read from `config` object rather than pass down a bunch of booleans --- Taskfile.yml | 11 +++++++- decompiler/config.cpp | 4 +++ decompiler/config.h | 2 ++ decompiler/config/jak1/jak1_config.jsonc | 14 ++++++----- decompiler/config/jak2/jak2_config.jsonc | 12 +++++---- decompiler/config/jak3/jak3_config.jsonc | 12 +++++---- decompiler/extractor/main.cpp | 3 +-- .../level_extractor/extract_collide_frags.cpp | 10 +++----- .../level_extractor/extract_collide_frags.h | 6 ++--- decompiler/level_extractor/extract_level.cpp | 25 +++++++------------ decompiler/level_extractor/extract_level.h | 2 -- decompiler/main.cpp | 3 +-- 12 files changed, 55 insertions(+), 49 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 62075cc64e..a977c9a795 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -90,7 +90,6 @@ tasks: - cmd: python ./scripts/cpp/format-includes.py - cmd: python ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test tools lsp -i - task: format-json - # DECOMPILING decomp: cmds: @@ -115,6 +114,16 @@ tasks: - task: decomp-file - python ./scripts/gsrc/update-from-decomp.py --game "{{.GAME}}" --file {{.FILE}} - task: lint-gsrc-file + # ASSET RIPPING + rip-textures: + cmds: + - "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler \"./decompiler/config/{{.DECOMP_CONFIG}}\" \"./iso_data\" \"./decompiler_out\" --version \"{{.DECOMP_CONFIG_VERSION}}\" --config-override '{\"decompile_code\": false, \"levels_extract\": true, \"save_texture_pngs\": true}'" + rip-levels: + cmds: + - "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler \"./decompiler/config/{{.DECOMP_CONFIG}}\" \"./iso_data\" \"./decompiler_out\" --version \"{{.DECOMP_CONFIG_VERSION}}\" --config-override '{\"decompile_code\": false, \"levels_extract\": true, \"rip_levels\": true}'" + rip-collision: + cmds: + - "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler \"./decompiler/config/{{.DECOMP_CONFIG}}\" \"./iso_data\" \"./decompiler_out\" --version \"{{.DECOMP_CONFIG_VERSION}}\" --config-override '{\"decompile_code\": false, \"levels_extract\": true, \"extract_collision\": true, \"rip_collision\": true}'" # TOOLS analyze-ee-memory: cmds: diff --git a/decompiler/config.cpp b/decompiler/config.cpp index e84df0571b..93560c2995 100644 --- a/decompiler/config.cpp +++ b/decompiler/config.cpp @@ -314,6 +314,10 @@ Config make_config_via_json(nlohmann::json& json) { config.import_deps_by_file = import_deps.get>>(); + if (json.contains("rip_collision")) { + config.rip_collision = json.at("rip_collision").get(); + } + config.write_patches = json.at("write_patches").get(); config.apply_patches = json.at("apply_patches").get(); const auto& object_patches = json.at("object_patches"); diff --git a/decompiler/config.h b/decompiler/config.h index a5f87cb78a..dc22108c2f 100644 --- a/decompiler/config.h +++ b/decompiler/config.h @@ -179,6 +179,8 @@ struct Config { std::unordered_map joint_node_hacks; std::unordered_map> import_deps_by_file; + + bool rip_collision = false; }; Config read_config_file(const fs::path& path_to_config_file, diff --git a/decompiler/config/jak1/jak1_config.jsonc b/decompiler/config/jak1/jak1_config.jsonc index 7c14a9474c..5206995913 100644 --- a/decompiler/config/jak1/jak1_config.jsonc +++ b/decompiler/config/jak1/jak1_config.jsonc @@ -56,7 +56,7 @@ "generate_symbol_definition_map": false, // genreate the all-types file - "generate_all_types" : false, + "generate_all_types": false, // debug option for instruction decoder "write_hex_near_instructions": false, @@ -100,14 +100,16 @@ // LEVEL EXTRACTION //////////////////////////// - // turn this on to extract level background graphics data + // turn this on to extract level background graphics data as .fr3 files in out//fr3 "levels_extract": true, - // turn this on if you want extracted levels to be saved out as .obj files + // turn this on if you want extracted levels to be saved out as .glb files in glb_out/ "rip_levels": false, - // should we extract collision meshes? - // these can be displayed in game, but makes the .fr3 files slightly larger + // should we also extract collision meshes to the .fr3 files? + // these can be displayed in-game with the OpenGOAL collision renderer "extract_collision": true, - // save game textures as .png files. + // turn this on if you want extracted level collision to be saved as .obj files in debug_out/ + "rip_collision": false, + // save game textures as .png files to decompiler_out//textures "save_texture_pngs": false, //////////////////////////// diff --git a/decompiler/config/jak2/jak2_config.jsonc b/decompiler/config/jak2/jak2_config.jsonc index c8a3fc49b5..ad141a5aa8 100644 --- a/decompiler/config/jak2/jak2_config.jsonc +++ b/decompiler/config/jak2/jak2_config.jsonc @@ -110,14 +110,16 @@ // LEVEL EXTRACTION //////////////////////////// - // turn this on to extract level background graphics data + // turn this on to extract level background graphics data as .fr3 files in out//fr3 "levels_extract": true, - // turn this on if you want extracted levels to be saved out as .obj files + // turn this on if you want extracted levels to be saved out as .glb files in glb_out/ "rip_levels": false, - // should we extract collision meshes? - // these can be displayed in game, but makes the .fr3 files slightly larger + // should we also extract collision meshes to the .fr3 files? + // these can be displayed in-game with the OpenGOAL collision renderer "extract_collision": true, - + // turn this on if you want extracted level collision to be saved as .obj files in debug_out/ + "rip_collision": false, + // save game textures as .png files to decompiler_out//textures "save_texture_pngs": false, //////////////////////////// diff --git a/decompiler/config/jak3/jak3_config.jsonc b/decompiler/config/jak3/jak3_config.jsonc index f6ecd99525..5d1212c268 100644 --- a/decompiler/config/jak3/jak3_config.jsonc +++ b/decompiler/config/jak3/jak3_config.jsonc @@ -102,14 +102,16 @@ // LEVEL EXTRACTION //////////////////////////// - // turn this on to extract level background graphics data + // turn this on to extract level background graphics data as .fr3 files in out//fr3 "levels_extract": false, - // turn this on if you want extracted levels to be saved out as .obj files + // turn this on if you want extracted levels to be saved out as .glb files in glb_out/ "rip_levels": false, - // should we extract collision meshes? - // these can be displayed in game, but makes the .fr3 files slightly larger + // should we also extract collision meshes to the .fr3 files? + // these can be displayed in-game with the OpenGOAL collision renderer "extract_collision": false, - + // turn this on if you want extracted level collision to be saved as .obj files in debug_out/ + "rip_collision": false, + // save game textures as .png files to decompiler_out//textures "save_texture_pngs": false, //////////////////////////// diff --git a/decompiler/extractor/main.cpp b/decompiler/extractor/main.cpp index 74a9b594fb..18186b7240 100644 --- a/decompiler/extractor/main.cpp +++ b/decompiler/extractor/main.cpp @@ -200,8 +200,7 @@ void decompile(const fs::path& iso_data_path, const std::string& data_subfolder) auto level_out_path = file_util::get_jak_project_dir() / "out" / game_version_names[config.game_version] / "fr3"; file_util::create_dir_if_needed(level_out_path); - extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, config.rip_levels, - config.extract_collision, level_out_path); + extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, level_out_path); } } diff --git a/decompiler/level_extractor/extract_collide_frags.cpp b/decompiler/level_extractor/extract_collide_frags.cpp index 23eb19e785..24c5ca55f8 100644 --- a/decompiler/level_extractor/extract_collide_frags.cpp +++ b/decompiler/level_extractor/extract_collide_frags.cpp @@ -233,8 +233,7 @@ void extract_collide_frags(const level_tools::DrawableTreeCollideFragment* tree, const std::vector& ties, const Config& config, const std::string& debug_name, - tfrag3::Level& out, - bool dump_level) { + tfrag3::Level& out) { // in-game, the broad-phase collision builds a list of fragments, then unpacks them with: /* *(dotimes (i (-> *collide-list* num-items)) @@ -260,7 +259,7 @@ void extract_collide_frags(const level_tools::DrawableTreeCollideFragment* tree, total_faces += frag.unpacked.faces.size(); } - if (dump_level) { + if (config.rip_collision) { auto debug_out = debug_dump_to_obj(all_frags); auto file_path = file_util::get_file_path( {fmt::format("debug_out/{}", config.game_name), fmt::format("collide-{}.obj", debug_name)}); @@ -442,8 +441,7 @@ void extract_collide_frags(const level_tools::CollideHash& chash, const Config& config, const std::string& debug_name, const decompiler::DecompilerTypeSystem& dts, - tfrag3::Level& out, - bool dump_level) { + tfrag3::Level& out) { // We need to find all collide-hash-fragments, but we can't just scan through the entire file. // for collide-hash-fragments, we need to figure out which TIEs they belong to, to apply the // instance transformation matrix. @@ -507,7 +505,7 @@ void extract_collide_frags(const level_tools::CollideHash& chash, } } - if (dump_level) { + if (config.rip_collision) { // out.collision.vertices every 3 vertices make a face, so it duplicates vertices in many cases // for now debug_dump_to_obj isn't smart and doesn't hash these to save space or anything auto debug_out = debug_dump_to_obj(out.collision.vertices); diff --git a/decompiler/level_extractor/extract_collide_frags.h b/decompiler/level_extractor/extract_collide_frags.h index 7abd90ef18..97c56498a0 100644 --- a/decompiler/level_extractor/extract_collide_frags.h +++ b/decompiler/level_extractor/extract_collide_frags.h @@ -10,15 +10,13 @@ void extract_collide_frags(const level_tools::DrawableTreeCollideFragment* tree, const std::vector& ties, const Config& config, const std::string& debug_name, - tfrag3::Level& out, - bool dump_level); + tfrag3::Level& out); void extract_collide_frags(const level_tools::CollideHash& chash, const std::vector& ties, const Config& config, const std::string& debug_name, const decompiler::DecompilerTypeSystem& dts, - tfrag3::Level& out, - bool dump_level); + tfrag3::Level& out); } // namespace decompiler diff --git a/decompiler/level_extractor/extract_level.cpp b/decompiler/level_extractor/extract_level.cpp index 0104849c6c..9fdfe5faa0 100644 --- a/decompiler/level_extractor/extract_level.cpp +++ b/decompiler/level_extractor/extract_level.cpp @@ -160,7 +160,6 @@ level_tools::BspHeader extract_bsp_from_level(const ObjectFileDB& db, const TextureDB& tex_db, const std::string& dgo_name, const Config& config, - bool extract_collision, tfrag3::Level& level_data) { auto hacks = config.hacks; auto bsp_rec = get_bsp_file(db.obj_files_by_dgo.at(dgo_name), dgo_name); @@ -231,14 +230,15 @@ level_tools::BspHeader extract_bsp_from_level(const ObjectFileDB& db, ASSERT(as_shrub_tree); extract_shrub(as_shrub_tree, fmt::format("{}-{}-shrub", dgo_name, i++), bsp_header.texture_remap_table, tex_db, {}, level_data, false, db.version()); - } else if (draw_tree->my_type() == "drawable-tree-collide-fragment" && extract_collision) { + } else if (draw_tree->my_type() == "drawable-tree-collide-fragment" && + config.extract_collision) { auto as_collide_frags = dynamic_cast(draw_tree.get()); ASSERT(as_collide_frags); ASSERT(!got_collide); got_collide = true; extract_collide_frags(as_collide_frags, all_ties, config, - fmt::format("{}-{}-collide", dgo_name, i++), level_data, false); + fmt::format("{}-{}-collide", dgo_name, i++), level_data); } else { lg::print(" unsupported tree {}\n", draw_tree->my_type()); } @@ -247,7 +247,7 @@ level_tools::BspHeader extract_bsp_from_level(const ObjectFileDB& db, if (bsp_header.collide_hash.num_items) { ASSERT(!got_collide); extract_collide_frags(bsp_header.collide_hash, all_ties, config, - fmt::format("{}-{}-collide", dgo_name, i++), db.dts, level_data, false); + fmt::format("{}-{}-collide", dgo_name, i++), db.dts, level_data); } level_data.level_name = level_name; @@ -262,7 +262,6 @@ level_tools::BspHeader extract_bsp_from_level(const ObjectFileDB& db, void extract_common(const ObjectFileDB& db, const TextureDB& tex_db, const std::string& dgo_name, - bool dump_levels, const fs::path& output_folder, const Config& config) { if (db.obj_files_by_dgo.count(dgo_name) == 0) { @@ -325,7 +324,7 @@ void extract_common(const ObjectFileDB& db, output_folder / fmt::format("{}.fr3", dgo_name.substr(0, dgo_name.length() - 4)), compressed.data(), compressed.size()); - if (dump_levels) { + if (config.rip_levels) { auto file_path = file_util::get_jak_project_dir() / "glb_out" / "common.glb"; file_util::create_dir_if_needed_for_file(file_path); save_level_foreground_as_gltf(tfrag_level, art_group_data, file_path); @@ -336,8 +335,6 @@ void extract_from_level(const ObjectFileDB& db, const TextureDB& tex_db, const std::string& dgo_name, const Config& config, - bool dump_level, - bool extract_collision, const fs::path& output_folder, const fs::path& entities_folder) { if (db.obj_files_by_dgo.count(dgo_name) == 0) { @@ -349,8 +346,7 @@ void extract_from_level(const ObjectFileDB& db, add_all_textures_from_level(level_data, dgo_name, tex_db); // the bsp header file data - auto bsp_header = - extract_bsp_from_level(db, tex_db, dgo_name, config, extract_collision, level_data); + auto bsp_header = extract_bsp_from_level(db, tex_db, dgo_name, config, level_data); extract_art_groups_from_level(db, tex_db, bsp_header.texture_remap_table, dgo_name, level_data, art_group_data); @@ -366,7 +362,7 @@ void extract_from_level(const ObjectFileDB& db, output_folder / fmt::format("{}.fr3", dgo_name.substr(0, dgo_name.length() - 4)), compressed.data(), compressed.size()); - if (dump_level) { + if (config.rip_levels) { auto back_file_path = file_util::get_jak_project_dir() / "glb_out" / fmt::format("{}_background.glb", level_data.level_name); file_util::create_dir_if_needed_for_file(back_file_path); @@ -385,18 +381,15 @@ void extract_all_levels(const ObjectFileDB& db, const std::vector& dgo_names, const std::string& common_name, const Config& config, - bool debug_dump_level, - bool extract_collision, const fs::path& output_path) { - extract_common(db, tex_db, common_name, debug_dump_level, output_path, config); + extract_common(db, tex_db, common_name, output_path, config); auto entities_dir = file_util::get_jak_project_dir() / "decompiler_out" / game_version_names[config.game_version] / "entities"; file_util::create_dir_if_needed(entities_dir); SimpleThreadGroup threads; threads.run( [&](int idx) { - extract_from_level(db, tex_db, dgo_names[idx], config, debug_dump_level, extract_collision, - output_path, entities_dir); + extract_from_level(db, tex_db, dgo_names[idx], config, output_path, entities_dir); }, dgo_names.size()); threads.join(); diff --git a/decompiler/level_extractor/extract_level.h b/decompiler/level_extractor/extract_level.h index a28fbaafa9..48800629a6 100644 --- a/decompiler/level_extractor/extract_level.h +++ b/decompiler/level_extractor/extract_level.h @@ -16,8 +16,6 @@ void extract_all_levels(const ObjectFileDB& db, const std::vector& dgo_names, const std::string& common_name, const Config& config, - bool debug_dump_level, - bool extract_collision, const fs::path& path); void add_all_textures_from_level(tfrag3::Level& lev, const std::string& level_name, diff --git a/decompiler/main.cpp b/decompiler/main.cpp index 7a3150a077..e056a68429 100644 --- a/decompiler/main.cpp +++ b/decompiler/main.cpp @@ -321,8 +321,7 @@ int main(int argc, char** argv) { auto level_out_path = file_util::get_jak_project_dir() / "out" / game_version_names[config.game_version] / "fr3"; file_util::create_dir_if_needed(level_out_path); - extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, config.rip_levels, - config.extract_collision, level_out_path); + extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, level_out_path); } mem_log("After extraction: {} MB", get_peak_rss() / (1024 * 1024));