From 13e7a28c087f72891c55f45e744ba2a4bc783dec Mon Sep 17 00:00:00 2001 From: Tyler Wilding Date: Sat, 15 Jun 2024 15:42:27 -0400 Subject: [PATCH] decomp: eliminate duplicate code in extractor/decompiler This allows the extractor's decompilation routine to do all the things the normal decompiler can (ie. extract audio) --- decompiler/extractor/main.cpp | 136 ++++-------------- decompiler/main.cpp | 260 +--------------------------------- 2 files changed, 31 insertions(+), 365 deletions(-) diff --git a/decompiler/extractor/main.cpp b/decompiler/extractor/main.cpp index 879671b855..2bcfbc7d87 100644 --- a/decompiler/extractor/main.cpp +++ b/decompiler/extractor/main.cpp @@ -5,11 +5,10 @@ #include "common/util/term_util.h" #include "common/util/unicode_util.h" -#include "decompiler/ObjectFile/ObjectFileDB.h" #include "decompiler/config.h" #include "decompiler/extractor/extractor_util.h" -#include "decompiler/level_extractor/extract_level.h" #include "goalc/compiler/Compiler.h" +#include #include "third-party/CLI11.hpp" @@ -102,116 +101,24 @@ std::tuple, ExtractorErrorCode> validate( }; } -// TODO - remove code duplication, most of this is copying what happens in decompiler's `main.cpp` -void decompile(const fs::path& iso_data_path, - const std::string& data_subfolder, - const std::string& config_override) { - using namespace decompiler; - +ExtractorErrorCode decompile(const fs::path& in_folder, + const std::string& data_subfolder, + const std::string& config_override) { // Determine which config to use from the database - const auto version_info = get_version_info_or_default(iso_data_path); + const auto version_info = get_version_info_or_default(in_folder); - Config config = read_config_file(file_util::get_jak_project_dir() / "decompiler" / "config" / - version_info.game_name / - fmt::format("{}_config.jsonc", version_info.game_name), - version_info.decomp_config_version, config_override); + decompiler::Config config = decompiler::read_config_file( + file_util::get_jak_project_dir() / "decompiler" / "config" / version_info.game_name / + fmt::format("{}_config.jsonc", version_info.game_name), + version_info.decomp_config_version, config_override); - std::vector dgos, objs, tex_strs, art_strs; - - // grab all DGOS we need (level + common) - // TODO - Jak 2 - jak 1 specific code? - for (const auto& dgo_name : config.dgo_names) { - std::string common_name = "GAME.CGO"; - if (dgo_name.length() > 3 && dgo_name.substr(dgo_name.length() - 3) == "DGO") { - // ends in DGO, it's a level - dgos.push_back(iso_data_path / dgo_name); - } else if (dgo_name.length() >= common_name.length() && - dgo_name.substr(dgo_name.length() - common_name.length()) == common_name) { - // it's COMMON.CGO, we need that too. - dgos.push_back(iso_data_path / dgo_name); - } - } - - // grab all the object files we need (just text) - for (const auto& obj_name : config.object_file_names) { - if (obj_name.length() > 3 && obj_name.substr(obj_name.length() - 3) == "TXT") { - // ends in TXT - objs.push_back(iso_data_path / obj_name); - } - } - - for (const auto& str_name : config.str_texture_file_names) { - tex_strs.push_back(iso_data_path / str_name); - } - - for (const auto& str_name : config.str_art_file_names) { - art_strs.push_back(iso_data_path / str_name); - } - - // set up objects - ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, {}, tex_strs, art_strs, - config); - - // save object files auto out_folder = file_util::get_jak_project_dir() / "decompiler_out" / data_subfolder; - auto raw_obj_folder = out_folder / "raw_obj"; - file_util::create_dir_if_needed(raw_obj_folder); - db.dump_raw_objects(raw_obj_folder); - // analyze object file link data - db.process_link_data(config); - db.find_code(config); - db.process_labels(); - - // ensure asset dir exists - file_util::create_dir_if_needed(out_folder / "assets"); - - // text files - { - auto result = db.process_game_text_files(config); - if (!result.empty()) { - file_util::write_text_file(out_folder / "assets" / "game_text.txt", result); - } - } - - // textures - decompiler::TextureDB tex_db; - auto textures_out = out_folder / "textures"; - auto dump_out = out_folder / "import"; - file_util::create_dir_if_needed(textures_out); - file_util::write_text_file(textures_out / "tpage-dir.txt", - db.process_tpages(tex_db, textures_out, config, dump_out)); - - // texture merges - // TODO - put all this stuff in somewhere common - auto texture_merge_path = file_util::get_jak_project_dir() / "game" / "assets" / - game_version_names[config.game_version] / "texture_merges"; - if (fs::exists(texture_merge_path)) { - tex_db.merge_textures(texture_merge_path); - } - - // texture replacements - auto replacements_path = file_util::get_jak_project_dir() / "custom_assets" / - game_version_names[config.game_version] / "texture_replacements"; - if (fs::exists(replacements_path)) { - tex_db.replace_textures(replacements_path); - } - - // game count - { - auto result = db.process_game_count_file(); - if (!result.empty()) { - file_util::write_text_file(out_folder / "assets" / "game_count.txt", result); - } - } - - // levels - { - auto level_out_path = - file_util::get_jak_project_dir() / "out" / game_version_names[config.game_version] / "fr3"; - file_util::create_dir_if_needed(level_out_path); - extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, level_out_path); + const auto result = run_decompilation_process(config, in_folder, out_folder, true); + if (result != 0) { + return ExtractorErrorCode::DECOMPILATION_GENERIC_ERROR; } + return ExtractorErrorCode::SUCCESS; } const std::unordered_map game_iso_flag_names = { @@ -315,7 +222,6 @@ int main(int argc, char** argv) { } // - SETUP - decompiler::init_opcode_info(); if (!project_path_override.empty()) { if (!fs::exists(project_path_override)) { lg::error("Error: project path override '{}' does not exist", project_path_override.string()); @@ -426,7 +332,11 @@ int main(int argc, char** argv) { // Get hash and file count const auto [hash, file_count] = calculate_extraction_hash(iso_data_path); // Validate - auto [version_info, code] = validate(iso_data_path, hash, file_count); + auto [version_info, validate_code] = validate(iso_data_path, hash, file_count); + if (validate_code == ExtractorErrorCode::VALIDATION_BAD_EXTRACTION || + (flag_fail_on_validation && validate_code != ExtractorErrorCode::SUCCESS)) { + return static_cast(validate_code); + } } // write out a json file with some metadata for the game @@ -455,7 +365,10 @@ int main(int argc, char** argv) { if (flag_decompile) { try { - decompile(iso_data_path, data_subfolder, decomp_config_override); + const auto status_code = decompile(iso_data_path, data_subfolder, decomp_config_override); + if (status_code != ExtractorErrorCode::SUCCESS) { + return static_cast(status_code); + } } catch (std::exception& e) { lg::error("Error during decompile: {}", e.what()); return static_cast(ExtractorErrorCode::DECOMPILATION_GENERIC_ERROR); @@ -463,7 +376,10 @@ int main(int argc, char** argv) { } if (flag_compile) { - compile(iso_data_path, data_subfolder); + const auto status_code = compile(iso_data_path, data_subfolder); + if (status_code != ExtractorErrorCode::SUCCESS) { + return static_cast(status_code); + } } if (flag_play) { diff --git a/decompiler/main.cpp b/decompiler/main.cpp index 5586970e03..1ce9d89c5f 100644 --- a/decompiler/main.cpp +++ b/decompiler/main.cpp @@ -1,33 +1,15 @@ -#include #include -#include #include "config.h" +#include "decompilation_process.h" #include "common/log/log.h" #include "common/util/FileUtil.h" -#include "common/util/Timer.h" -#include "common/util/diff.h" -#include "common/util/os.h" #include "common/util/set_util.h" #include "common/util/term_util.h" #include "common/util/unicode_util.h" -#include "common/versions/versions.h" - -#include "ObjectFile/ObjectFileDB.h" -#include "decompiler/data/TextureDB.h" -#include "decompiler/data/streamed_audio.h" -#include "decompiler/level_extractor/extract_level.h" #include "third-party/CLI11.hpp" -#include "third-party/json.hpp" - -template -static void mem_log(const std::string& format, Args&&... args) { -#ifndef _WIN32 - lg::info("[Mem] " + format, std::forward(args)...); -#endif -} int main(int argc, char** argv) { ArgumentGuard u8_guard(argc, argv); @@ -81,11 +63,9 @@ int main(int argc, char** argv) { return 1; } - using namespace decompiler; - - Config config; + decompiler::Config config; try { - config = read_config_file(config_path, config_game_version, config_override); + config = decompiler::read_config_file(config_path, config_game_version, config_override); } catch (const std::exception& e) { lg::error("Failed to parse config: {}", e.what()); return 1; @@ -115,7 +95,7 @@ int main(int argc, char** argv) { // Warning message if expected ELF isn't found, user could be using bad assets / didn't extract // the ISO properly - if (!config.expected_elf_name.empty() && !exists(in_folder / config.expected_elf_name)) { + if (!config.expected_elf_name.empty() && !fs::exists(in_folder / config.expected_elf_name)) { lg::error( "WARNING - '{}' does not contain the expected ELF file '{}'. Was the ISO extracted " "properly or is there a version mismatch?", @@ -123,235 +103,5 @@ int main(int argc, char** argv) { } // -- Begin the Decompilation! - - Timer decomp_timer; - - mem_log("Top of main: {} MB\n", get_peak_rss() / (1024 * 1024)); - - init_opcode_info(); - - mem_log("After init: {} MB\n", get_peak_rss() / (1024 * 1024)); - - std::vector dgos, objs, strs, tex_strs, art_strs; - for (const auto& dgo_name : config.dgo_names) { - dgos.push_back(in_folder / dgo_name); - } - - for (const auto& obj_name : config.object_file_names) { - objs.push_back(in_folder / obj_name); - } - - for (const auto& str_name : config.str_file_names) { - strs.push_back(in_folder / str_name); - } - - for (const auto& str_name : config.str_texture_file_names) { - tex_strs.push_back(in_folder / str_name); - } - - for (const auto& str_name : config.str_art_file_names) { - art_strs.push_back(in_folder / str_name); - } - - mem_log("After config read: {} MB", get_peak_rss() / (1024 * 1024)); - - // build file database - lg::info("Setting up object file DB..."); - ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, strs, tex_strs, art_strs, - config); - - // Explicitly fail if a file in the 'allowed_objects' list wasn't found in the DB - // as this is another silent error that can be confusing - if (!config.allowed_objects.empty()) { - for (const auto& expected_obj : config.allowed_objects) { - if (db.obj_files_by_name.count(expected_obj) == 0) { - // TODO - this is wrong for jak1, fix eventually as this is now done in 3 places - lg::error( - "Expected to find '{}' in the ObjectFileDB but did not. Check " - "./decompiler/config/{}/inputs.jsonc", - expected_obj, config.game_name); - return 1; - } - } - } - - mem_log("After DB setup: {} MB", get_peak_rss() / (1024 * 1024)); - - // write out DGO file info - file_util::write_text_file(out_folder / "dgo.txt", db.generate_dgo_listing()); - // write out object file map (used for future decompilations, if desired) - file_util::write_text_file(out_folder / "obj.txt", - db.generate_obj_listing(config.merged_objects)); - - // dump raw objs - if (config.dump_objs) { - auto path = out_folder / "raw_obj"; - file_util::create_dir_if_needed(path); - db.dump_raw_objects(path); - } - - // process files (required for all analysis) - db.process_link_data(config); - mem_log("After link data: {} MB", get_peak_rss() / (1024 * 1024)); - db.find_code(config); - db.process_labels(); - mem_log("After code: {} MB", get_peak_rss() / (1024 * 1024)); - - // top level decompile (do this before printing asm so we get function names) - if (config.find_functions) { - db.ir2_top_level_pass(config); - } - - // print disassembly - if (config.disassemble_code || config.disassemble_data) { - db.write_disassembly(out_folder, config.disassemble_data, config.disassemble_code, - config.write_hex_near_instructions); - } - - if (config.process_art_groups) { - db.extract_art_info(); - // dump art info to json if requested - if (config.dump_art_group_info) { - auto ag_file_name = out_folder / "dump" / "art-group-info.min.json"; - nlohmann::json ag_json = db.dts.art_group_info; - file_util::create_dir_if_needed_for_file(ag_file_name); - file_util::write_text_file(ag_file_name, ag_json.dump(-1)); - lg::info("[DUMP] Dumped art group info to {}", ag_file_name.string()); - } - if (config.dump_joint_geo_info) { - auto jg_file_name = out_folder / "dump" / "joint-node-info.min.json"; - nlohmann::json jg_json = db.dts.jg_info; - file_util::create_dir_if_needed_for_file(jg_file_name); - file_util::write_text_file(jg_file_name, jg_json.dump(-1)); - lg::info("[DUMP] Dumped joint node info to {}", jg_file_name.string()); - } - } else if (!config.art_group_info_dump.empty() || !config.jg_info_dump.empty()) { - // process art groups (used in decompilation) - // - if the config has a path to the art info dump, just use that - // - otherwise (or if we want to dump it fresh) extract it - if (!config.art_group_info_dump.empty()) { - db.dts.art_group_info = config.art_group_info_dump; - } - if (!config.jg_info_dump.empty()) { - db.dts.jg_info = config.jg_info_dump; - } - } else { - lg::error("`process_art_groups` was false and no art-group-info dump was provided!"); - return 1; - } - - if (config.process_tpages && !config.texture_info_dump.empty()) { - db.dts.textures = config.texture_info_dump; - } - - // main decompile. - if (config.decompile_code) { - db.analyze_functions_ir2(out_folder, config, {}, {}, {}); - } - - if (config.generate_all_types) { - ASSERT_MSG(config.decompile_code, "Must decompile code to generate all-types"); - db.ir2_analyze_all_types(out_folder / "new-all-types.gc", config.old_all_types_file, - config.hacks.types_with_bad_inspect_methods); - } - - mem_log("After decomp: {} MB", get_peak_rss() / (1024 * 1024)); - - // write out all symbols - file_util::write_text_file(out_folder / "all-syms.gc", db.dts.dump_symbol_types()); - - // write art groups - if (config.process_art_groups) { - db.dump_art_info(out_folder); - } - - if (config.hexdump_code || config.hexdump_data) { - db.write_object_file_words(out_folder, config.hexdump_data, config.hexdump_code); - } - - // data stuff - if (config.write_scripts) { - db.find_and_write_scripts(out_folder); - } - - if (config.process_game_text) { - auto result = db.process_game_text_files(config); - if (!result.empty()) { - file_util::write_text_file(out_folder / "assets" / "game_text.txt", result); - } - } - - mem_log("After text: {} MB", get_peak_rss() / (1024 * 1024)); - - if (config.process_subtitle_text || config.process_subtitle_images) { - auto result = db.process_all_spool_subtitles( - config, config.process_subtitle_images ? out_folder / "assets" / "subtitle-images" : ""); - if (!result.empty()) { - file_util::write_text_file(out_folder / "assets" / "game_subs.txt", result); - } - } - - mem_log("After spool handling: {} MB", get_peak_rss() / (1024 * 1024)); - - TextureDB tex_db; - if (config.process_tpages || config.levels_extract) { - auto textures_out = out_folder / "textures"; - auto dump_out = out_folder / "import"; - file_util::create_dir_if_needed(textures_out); - auto result = db.process_tpages(tex_db, textures_out, config, dump_out); - if (!result.empty() && config.process_tpages) { - file_util::write_text_file(textures_out / "tpage-dir.txt", result); - file_util::write_text_file(textures_out / "tex-remap.txt", - tex_db.generate_texture_dest_adjustment_table()); - } - if (config.dump_tex_info) { - auto texture_file_name = out_folder / "dump" / "tex-info.min.json"; - nlohmann::json texture_json = db.dts.textures; - file_util::create_dir_if_needed_for_file(texture_file_name); - file_util::write_text_file(texture_file_name, texture_json.dump(-1)); - lg::info("[DUMP] Dumped texture info to {}", texture_file_name.string()); - } - } - - mem_log("After textures: {} MB", get_peak_rss() / (1024 * 1024)); - - // Merge textures before replacing them, in other words, replacements take priority - auto texture_merge_path = file_util::get_jak_project_dir() / "game" / "assets" / - game_version_names[config.game_version] / "texture_merges"; - if (fs::exists(texture_merge_path)) { - tex_db.merge_textures(texture_merge_path); - } - - auto replacements_path = file_util::get_jak_project_dir() / "custom_assets" / - game_version_names[config.game_version] / "texture_replacements"; - if (fs::exists(replacements_path)) { - tex_db.replace_textures(replacements_path); - } - - if (config.process_game_count) { - auto result = db.process_game_count_file(); - if (!result.empty()) { - file_util::write_text_file(out_folder / "assets" / "game_count.txt", result); - } - } - - if (config.levels_extract) { - auto level_out_path = - file_util::get_jak_project_dir() / "out" / game_version_names[config.game_version] / "fr3"; - file_util::create_dir_if_needed(level_out_path); - extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config, level_out_path); - } - - mem_log("After extraction: {} MB", get_peak_rss() / (1024 * 1024)); - - if (!config.audio_dir_file_name.empty()) { - auto streaming_audio_in = in_folder / "VAG"; - auto streaming_audio_out = out_folder / "assets" / "streaming_audio"; - file_util::create_dir_if_needed(streaming_audio_out); - process_streamed_audio(config, streaming_audio_out, in_folder, - config.streamed_audio_file_names); - } - - lg::info("Decompiler has finished successfully in {:.2f} seconds.", decomp_timer.getSeconds()); - return 0; + return run_decompilation_process(config, in_folder, out_folder, false); }