diff --git a/assets/.gitignore b/assets/.gitignore deleted file mode 100644 index c96a04f008..0000000000 --- a/assets/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore \ No newline at end of file diff --git a/common/audio/audio_formats.cpp b/common/audio/audio_formats.cpp index b6f3216eb1..e540d6681e 100644 --- a/common/audio/audio_formats.cpp +++ b/common/audio/audio_formats.cpp @@ -9,7 +9,7 @@ */ void write_wave_file_mono(const std::vector& samples, s32 sample_rate, - const std::string& name) { + const std::filesystem::path& name) { WaveFileHeader header; memcpy(header.chunk_id, "RIFF", 4); header.chunk_size = 36 + samples.size() * sizeof(s16); diff --git a/common/audio/audio_formats.h b/common/audio/audio_formats.h index 0b5d2b2487..ed98889763 100644 --- a/common/audio/audio_formats.h +++ b/common/audio/audio_formats.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -30,7 +31,7 @@ struct WaveFileHeader { void write_wave_file_mono(const std::vector& samples, s32 sample_rate, - const std::string& name); + const std::filesystem::path& name); std::vector decode_adpcm(BinaryReader& reader); diff --git a/common/util/BinaryWriter.h b/common/util/BinaryWriter.h index a4a7b22c61..739492d035 100644 --- a/common/util/BinaryWriter.h +++ b/common/util/BinaryWriter.h @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -63,12 +64,12 @@ class BinaryWriter { void* get_data() { return data.data(); } - void write_to_file(const std::string& filename) { - auto fp = fopen(filename.c_str(), "wb"); + void write_to_file(const std::filesystem::path& filename) { + auto fp = fopen(filename.string().c_str(), "wb"); if (!fp) - throw std::runtime_error("failed to open " + filename); + throw std::runtime_error("failed to open " + filename.string()); if (fwrite(get_data(), get_size(), 1, fp) != 1) - throw std::runtime_error("failed to write " + filename); + throw std::runtime_error("failed to write " + filename.string()); fclose(fp); } diff --git a/common/util/DgoWriter.cpp b/common/util/DgoWriter.cpp index 3cd1842413..d0b0b07bcb 100644 --- a/common/util/DgoWriter.cpp +++ b/common/util/DgoWriter.cpp @@ -15,6 +15,7 @@ void build_dgo(const DgoDescription& description) { writer.add_cstr_len(description.dgo_name.c_str(), 60); for (auto& obj : description.entries) { + // todo: hardcoded out auto obj_data = file_util::read_binary_file(file_util::get_file_path({"out", "obj", obj.file_name})); // size diff --git a/common/util/FileUtil.cpp b/common/util/FileUtil.cpp index d553575408..c81694d3aa 100644 --- a/common/util/FileUtil.cpp +++ b/common/util/FileUtil.cpp @@ -169,7 +169,7 @@ std::string get_file_path(const std::vector& input) { return current_path.string(); } -bool create_dir_if_needed(const std::string& path) { +bool create_dir_if_needed(const std::filesystem::path& path) { if (!std::filesystem::is_directory(path)) { std::filesystem::create_directories(path); return true; @@ -181,59 +181,70 @@ bool create_dir_if_needed_for_file(const std::string& path) { return std::filesystem::create_directories(std::filesystem::path(path).parent_path()); } -void write_binary_file(const std::string& name, const void* data, size_t size) { - FILE* fp = fopen(name.c_str(), "wb"); +void write_binary_file(const std::filesystem::path& name, const void* data, size_t size) { + FILE* fp = fopen(name.string().c_str(), "wb"); if (!fp) { - throw std::runtime_error("couldn't open file " + name); + throw std::runtime_error("couldn't open file " + name.string()); } if (fwrite(data, size, 1, fp) != 1) { fclose(fp); - throw std::runtime_error("couldn't write file " + name); + throw std::runtime_error("couldn't write file " + name.string()); } fclose(fp); } -void write_rgba_png(const std::string& name, void* data, int w, int h) { +void write_binary_file(const std::string& name, const void* data, size_t size) { + write_binary_file(std::filesystem::path(name), data, size); +} + +void write_rgba_png(const std::filesystem::path& name, void* data, int w, int h) { auto flags = 0; - auto ok = fpng::fpng_encode_image_to_file(name.c_str(), data, w, h, 4, flags); + auto ok = fpng::fpng_encode_image_to_file(name.string().c_str(), data, w, h, 4, flags); if (!ok) { - throw std::runtime_error("couldn't save png file " + name); + throw std::runtime_error("couldn't save png file " + name.string()); } } void write_text_file(const std::string& file_name, const std::string& text) { - FILE* fp = fopen(file_name.c_str(), "w"); + write_text_file(std::filesystem::path(file_name), text); +} + +void write_text_file(const std::filesystem::path& file_name, const std::string& text) { + FILE* fp = fopen(file_name.string().c_str(), "w"); if (!fp) { - lg::error("Failed to fopen {}\n", file_name); + lg::error("Failed to fopen {}\n", file_name.string()); throw std::runtime_error("Failed to open file"); } fprintf(fp, "%s\n", text.c_str()); fclose(fp); } - std::vector read_binary_file(const std::string& filename) { - // make sure file exists and isn't a directory - std::filesystem::path path(filename); + return read_binary_file(std::filesystem::path(filename)); +} - auto status = std::filesystem::status(std::filesystem::path(filename)); +std::vector read_binary_file(const std::filesystem::path& path) { + // make sure file exists and isn't a directory + + auto status = std::filesystem::status(path); if (!std::filesystem::exists(status)) { - throw std::runtime_error(fmt::format("File {} cannot be opened: does not exist.", filename)); + throw std::runtime_error( + fmt::format("File {} cannot be opened: does not exist.", path.string())); } if (status.type() != std::filesystem::file_type::regular && status.type() != std::filesystem::file_type::symlink) { throw std::runtime_error( - fmt::format("File {} cannot be opened: not a regular file or symlink.", filename)); + fmt::format("File {} cannot be opened: not a regular file or symlink.", path.string())); } - auto fp = fopen(filename.c_str(), "rb"); + auto fp = fopen(path.string().c_str(), "rb"); if (!fp) - throw std::runtime_error("File " + filename + + throw std::runtime_error("File " + path.string() + " cannot be opened: " + std::string(strerror(errno))); fseek(fp, 0, SEEK_END); auto len = ftell(fp); @@ -244,23 +255,27 @@ std::vector read_binary_file(const std::string& filename) { if (fread(data.data(), len, 1, fp) != 1) { fclose(fp); - throw std::runtime_error("File " + filename + " cannot be read"); + throw std::runtime_error("File " + path.string() + " cannot be read"); } fclose(fp); return data; } -std::string read_text_file(const std::string& path) { - std::ifstream file(path); +std::string read_text_file(const std::filesystem::path& path) { + std::ifstream file(path.string()); if (!file.good()) { - throw std::runtime_error("couldn't open " + path); + throw std::runtime_error("couldn't open " + path.string()); } std::stringstream ss; ss << file.rdbuf(); return ss.str(); } +std::string read_text_file(const std::string& path) { + return read_text_file(std::filesystem::path(path)); +} + bool is_printable_char(char c) { return c >= ' ' && c <= '~'; } diff --git a/common/util/FileUtil.h b/common/util/FileUtil.h index 2f5eada17b..49df440b4b 100644 --- a/common/util/FileUtil.h +++ b/common/util/FileUtil.h @@ -21,15 +21,19 @@ std::filesystem::path get_user_settings_dir(); std::filesystem::path get_user_memcard_dir(); std::filesystem::path get_jak_project_dir(); -bool create_dir_if_needed(const std::string& path); +bool create_dir_if_needed(const std::filesystem::path& path); bool create_dir_if_needed_for_file(const std::string& path); bool setup_project_path(std::optional project_path_override); std::string get_file_path(const std::vector& path); void write_binary_file(const std::string& name, const void* data, size_t size); -void write_rgba_png(const std::string& name, void* data, int w, int h); +void write_binary_file(const std::filesystem::path& name, const void* data, size_t size); +void write_rgba_png(const std::filesystem::path& name, void* data, int w, int h); void write_text_file(const std::string& file_name, const std::string& text); +void write_text_file(const std::filesystem::path& file_name, const std::string& text); std::vector read_binary_file(const std::string& filename); +std::vector read_binary_file(const std::filesystem::path& filename); std::string read_text_file(const std::string& path); +std::string read_text_file(const std::filesystem::path& path); bool is_printable_char(char c); std::string combine_path(const std::string& parent, const std::string& child); bool file_exists(const std::string& path); diff --git a/decompiler/ObjectFile/ObjectFileDB.cpp b/decompiler/ObjectFile/ObjectFileDB.cpp index 03dfd53fb6..1d63198205 100644 --- a/decompiler/ObjectFile/ObjectFileDB.cpp +++ b/decompiler/ObjectFile/ObjectFileDB.cpp @@ -113,10 +113,10 @@ const ObjectFileData& ObjectFileDB::lookup_record(const ObjectFileRecord& rec) c /*! * Build an object file DB for the given list of DGOs. */ -ObjectFileDB::ObjectFileDB(const std::vector& _dgos, - const std::string& obj_file_name_map_file, - const std::vector& object_files, - const std::vector& str_files, +ObjectFileDB::ObjectFileDB(const std::vector& _dgos, + const std::filesystem::path& obj_file_name_map_file, + const std::vector& object_files, + const std::vector& str_files, const Config& config) { Timer timer; @@ -125,7 +125,8 @@ ObjectFileDB::ObjectFileDB(const std::vector& _dgos, if (!obj_file_name_map_file.empty()) { lg::info("-Loading obj name map file..."); - load_map_file(file_util::read_text_file(file_util::get_file_path({obj_file_name_map_file}))); + load_map_file( + file_util::read_text_file(file_util::get_jak_project_dir() / obj_file_name_map_file)); } else { lg::warn( "Not using an obj name map file! The decompiler will automatically generate object file " @@ -138,14 +139,14 @@ ObjectFileDB::ObjectFileDB(const std::vector& _dgos, try { get_objs_from_dgo(dgo, config); } catch (std::runtime_error& e) { - lg::warn("Error when reading DGOs: {} on {}", e.what(), dgo); + lg::warn("Error when reading DGOs: {} on {}", e.what(), dgo.string()); } } lg::info("-Loading {} plain object files...", object_files.size()); for (auto& obj : object_files) { auto data = file_util::read_binary_file(obj); - auto name = obj_filename_to_name(obj); + auto name = obj_filename_to_name(obj.string()); if (auto it = config.object_patches.find(name); it != config.object_patches.end()) { // print the file CRC fmt::print("CRC for {} is: 0x{:X}\n", name, crc32(data.data(), data.size())); @@ -206,7 +207,7 @@ ObjectFileDB::ObjectFileDB(const std::vector& _dgos, for (auto& obj : str_files) { StrFileReader reader(obj); // name from the file name - std::string base_name = obj_filename_to_name(obj); + std::string base_name = obj_filename_to_name(obj.string()); // name from inside the file (this does a lot of sanity checking) auto obj_name = reader.get_full_name(base_name + ".STR"); for (int i = 0; i < reader.chunk_count(); i++) { @@ -257,7 +258,7 @@ void ObjectFileDB::load_map_file(const std::string& map_data) { /*! * Load the objects stored in the given DGO into the ObjectFileDB */ -void ObjectFileDB::get_objs_from_dgo(const std::string& filename, const Config& config) { +void ObjectFileDB::get_objs_from_dgo(const std::filesystem::path& filename, const Config& config) { auto dgo_data = file_util::read_binary_file(filename); stats.total_dgo_bytes += dgo_data.size(); @@ -268,7 +269,7 @@ void ObjectFileDB::get_objs_from_dgo(const std::string& filename, const Config& BinaryReader reader(dgo_data); auto header = reader.read(); - auto dgo_base_name = file_util::base_name(filename); + auto dgo_base_name = filename.filename().string(); ASSERT(header.name == dgo_base_name); assert_string_empty_after(header.name, 60); @@ -515,7 +516,7 @@ void ObjectFileDB::process_labels() { /*! * Dump object files and their linking data to text files for debugging */ -void ObjectFileDB::write_object_file_words(const std::string& output_dir, +void ObjectFileDB::write_object_file_words(const std::filesystem::path& output_dir, bool dump_data, bool dump_code) { lg::info("- Writing object file dumps (code? {} data? {})...", dump_code, dump_data); @@ -527,7 +528,7 @@ void ObjectFileDB::write_object_file_words(const std::string& output_dir, if ((obj.linked_data.segments == 3 && dump_code) || (obj.linked_data.segments != 3 && dump_data)) { auto file_text = obj.linked_data.print_words(); - auto file_name = file_util::combine_path(output_dir, obj.to_unique_name() + ".txt"); + auto file_name = output_dir / (obj.to_unique_name() + ".txt"); total_bytes += file_text.size(); file_util::write_text_file(file_name, file_text); total_files++; @@ -544,7 +545,7 @@ void ObjectFileDB::write_object_file_words(const std::string& output_dir, /*! * Dump disassembly for object files containing code. Data zones will also be dumped. */ -void ObjectFileDB::write_disassembly(const std::string& output_dir, +void ObjectFileDB::write_disassembly(const std::filesystem::path& output_dir, bool disassemble_data, bool disassemble_code, bool print_hex) { @@ -558,7 +559,7 @@ void ObjectFileDB::write_disassembly(const std::string& output_dir, if ((obj.obj_version == 3 && disassemble_code) || (obj.obj_version != 3 && disassemble_data)) { auto file_text = obj.linked_data.print_disassembly(print_hex); asm_functions += obj.linked_data.print_asm_function_disassembly(obj.to_unique_name()); - auto file_name = file_util::combine_path(output_dir, obj.to_unique_name() + ".asm"); + auto file_name = output_dir / (obj.to_unique_name() + ".asm"); total_bytes += file_text.size(); file_util::write_text_file(file_name, file_text); @@ -568,8 +569,7 @@ void ObjectFileDB::write_disassembly(const std::string& output_dir, total_bytes += asm_functions.size(); total_files++; - file_util::write_text_file(file_util::combine_path(output_dir, "asm_functions.func"), - asm_functions); + file_util::write_text_file(output_dir / "asm_functions.func", asm_functions); lg::info("Wrote functions dumps:"); lg::info(" Total {} files", total_files); @@ -623,7 +623,7 @@ void ObjectFileDB::find_code(const Config& config) { * Finds and writes all scripts into a file named all_scripts.lisp. * Doesn't change any state in ObjectFileDB. */ -void ObjectFileDB::find_and_write_scripts(const std::string& output_dir) { +void ObjectFileDB::find_and_write_scripts(const std::filesystem::path& output_dir) { lg::info("- Finding scripts in object files..."); Timer timer; std::string all_scripts; @@ -638,14 +638,15 @@ void ObjectFileDB::find_and_write_scripts(const std::string& output_dir) { } }); - auto file_name = file_util::combine_path(output_dir, "all_scripts.lisp"); + auto file_name = output_dir / "all_scripts.lisp"; file_util::write_text_file(file_name, all_scripts); lg::info("Found scripts:"); lg::info(" Total {:.3f} ms\n", timer.getMs()); } -std::string ObjectFileDB::process_tpages(TextureDB& tex_db) { +std::string ObjectFileDB::process_tpages(TextureDB& tex_db, + const std::filesystem::path& output_path) { lg::info("- Finding textures in tpages..."); std::string tpage_string = "tpage-"; int total = 0, success = 0; @@ -656,7 +657,7 @@ std::string ObjectFileDB::process_tpages(TextureDB& tex_db) { std::string result; for_each_obj([&](ObjectFileData& data) { if (data.name_in_dgo.substr(0, tpage_string.length()) == tpage_string) { - auto statistics = process_tpage(data, tex_db); + auto statistics = process_tpage(data, tex_db, output_path); total += statistics.total_textures; success += statistics.successful_textures; total_px += statistics.num_px; @@ -795,16 +796,16 @@ void ObjectFileDB::extract_art_info() { /*! * Write out the art group information. */ -void ObjectFileDB::dump_art_info(const std::string& output_dir) { +void ObjectFileDB::dump_art_info(const std::filesystem::path& output_dir) { lg::info("Writing art group info..."); Timer timer; if (!dts.art_group_info.empty()) { - file_util::create_dir_if_needed(file_util::combine_path(output_dir, "import")); + file_util::create_dir_if_needed(output_dir / "import"); } for (const auto& [ag_name, info] : dts.art_group_info) { auto ag_fname = ag_name + ".gc"; - auto filename = file_util::get_file_path({output_dir, "import", ag_fname}); + auto filename = output_dir / "import" / ag_fname; std::string result = ";;-*-Lisp-*-\n"; result += "(in-package goal)\n\n"; result += fmt::format(";; {} - art group OpenGOAL import file\n", ag_fname); @@ -819,9 +820,9 @@ void ObjectFileDB::dump_art_info(const std::string& output_dir) { lg::info("Written art group info: in {:.2f} ms\n", timer.getMs()); } -void ObjectFileDB::dump_raw_objects(const std::string& output_dir) { +void ObjectFileDB::dump_raw_objects(const std::filesystem::path& output_dir) { for_each_obj([&](ObjectFileData& data) { - auto dest = output_dir + "/" + data.to_unique_name(); + auto dest = output_dir / data.to_unique_name(); if (data.obj_version != 3) { dest += ".go"; } diff --git a/decompiler/ObjectFile/ObjectFileDB.h b/decompiler/ObjectFile/ObjectFileDB.h index 70e5007ca3..e2497b2e0f 100644 --- a/decompiler/ObjectFile/ObjectFileDB.h +++ b/decompiler/ObjectFile/ObjectFileDB.h @@ -144,28 +144,30 @@ struct LetRewriteStats { class ObjectFileDB { public: - ObjectFileDB(const std::vector& _dgos, - const std::string& obj_file_name_map_file, - const std::vector& object_files, - const std::vector& str_files, + ObjectFileDB(const std::vector& _dgos, + const std::filesystem::path& obj_file_name_map_file, + const std::vector& object_files, + const std::vector& str_files, const Config& config); std::string generate_dgo_listing(); std::string generate_obj_listing(const std::unordered_set& merged_objs); void process_link_data(const Config& config); void process_labels(); void find_code(const Config& config); - void find_and_write_scripts(const std::string& output_dir); + void find_and_write_scripts(const std::filesystem::path& output_dir); void extract_art_info(); - void dump_art_info(const std::string& output_dir); - void dump_raw_objects(const std::string& output_dir); - void write_object_file_words(const std::string& output_dir, bool dump_data, bool dump_code); - void write_disassembly(const std::string& output_dir, + void dump_art_info(const std::filesystem::path& output_dir); + void dump_raw_objects(const std::filesystem::path& output_dir); + void write_object_file_words(const std::filesystem::path& output_dir, + bool dump_data, + bool dump_code); + void write_disassembly(const std::filesystem::path& output_dir, bool disassemble_data, bool disassemble_code, bool print_hex); void analyze_functions_ir2( - const std::string& output_dir, + const std::filesystem::path& output_dir, const Config& config, const std::unordered_set& skip_functions, const std::unordered_map>& skip_states = {}); @@ -183,7 +185,7 @@ class ObjectFileDB { void ir2_rewrite_inline_asm_instructions(int seg, ObjectFileData& data); void ir2_insert_anonymous_functions(int seg, ObjectFileData& data); void ir2_symbol_definition_map(ObjectFileData& data); - void ir2_write_results(const std::string& output_dir, + void ir2_write_results(const std::filesystem::path& output_dir, const Config& config, const std::vector& imports, ObjectFileData& data); @@ -191,7 +193,7 @@ class ObjectFileDB { void ir2_do_segment_analysis_phase2(int seg, const Config& config, ObjectFileData& data); void ir2_setup_labels(const Config& config, ObjectFileData& data); void ir2_run_mips2c(const Config& config, ObjectFileData& data); - void ir2_analyze_all_types(const std::string& output_file, + void ir2_analyze_all_types(const std::filesystem::path& output_file, const std::optional& previous_game_types, const std::unordered_set& bad_types); std::string ir2_to_file(ObjectFileData& data, const Config& config); @@ -200,7 +202,7 @@ class ObjectFileDB { const std::vector& imports, const std::unordered_set& skip_functions); - std::string process_tpages(TextureDB& tex_db); + std::string process_tpages(TextureDB& tex_db, const std::filesystem::path& output_path); std::string process_game_count_file(); std::string process_game_text_files(const Config& cfg); @@ -214,7 +216,7 @@ class ObjectFileDB { public: void load_map_file(const std::string& map_data); - void get_objs_from_dgo(const std::string& filename, const Config& config); + void get_objs_from_dgo(const std::filesystem::path& filename, const Config& config); void add_obj_from_dgo(const std::string& obj_name, const std::string& name_in_dgo, const uint8_t* obj_data, diff --git a/decompiler/ObjectFile/ObjectFileDB_IR2.cpp b/decompiler/ObjectFile/ObjectFileDB_IR2.cpp index aa1ba0c558..34d19f7df9 100644 --- a/decompiler/ObjectFile/ObjectFileDB_IR2.cpp +++ b/decompiler/ObjectFile/ObjectFileDB_IR2.cpp @@ -38,7 +38,7 @@ namespace decompiler { * functions, but nothing else. */ void ObjectFileDB::analyze_functions_ir2( - const std::string& output_dir, + const std::filesystem::path& output_dir, const Config& config, const std::unordered_set& skip_functions, const std::unordered_map>& skip_states) { @@ -100,7 +100,7 @@ void ObjectFileDB::analyze_functions_ir2( imports = imports_it->second; } - if (!output_dir.empty()) { + if (!output_dir.string().empty()) { ir2_write_results(output_dir, config, imports, data); } else { if (!skip_functions.empty()) { @@ -135,8 +135,7 @@ void ObjectFileDB::analyze_functions_ir2( lg::info("Generating symbol definition map..."); map_builder.build_map(); std::string result = map_builder.convert_to_json(); - auto file_name = file_util::combine_path(output_dir, "symbol_map.json"); - file_util::write_text_file(file_name, result); + file_util::write_text_file(output_dir / "symbol_map.json", result); } } @@ -301,7 +300,7 @@ void ObjectFileDB::ir2_top_level_pass(const Config& config) { lg::info("{:4d} logins {:.2f}%\n", total_top_levels, 100.f * total_top_levels / total_functions); } -void ObjectFileDB::ir2_analyze_all_types(const std::string& output_file, +void ObjectFileDB::ir2_analyze_all_types(const std::filesystem::path& output_file, const std::optional& previous_game_types, const std::unordered_set& bad_types) { struct PerObject { @@ -689,7 +688,7 @@ void ObjectFileDB::ir2_insert_anonymous_functions(int seg, ObjectFileData& data) }); } -void ObjectFileDB::ir2_write_results(const std::string& output_dir, +void ObjectFileDB::ir2_write_results(const std::filesystem::path& output_dir, const Config& config, const std::vector& imports, ObjectFileData& obj) { @@ -697,11 +696,11 @@ void ObjectFileDB::ir2_write_results(const std::string& output_dir, // todo auto file_text = ir2_to_file(obj, config); - auto file_name = file_util::combine_path(output_dir, obj.to_unique_name() + "_ir2.asm"); + auto file_name = output_dir / (obj.to_unique_name() + "_ir2.asm"); file_util::write_text_file(file_name, file_text); auto final = ir2_final_out(obj, imports, {}); - auto final_name = file_util::combine_path(output_dir, obj.to_unique_name() + "_disasm.gc"); + auto final_name = output_dir / (obj.to_unique_name() + "_disasm.gc"); file_util::write_text_file(final_name, final); } } diff --git a/decompiler/config/jak1_ntsc_black_label.jsonc b/decompiler/config/jak1_ntsc_black_label.jsonc index 93f2942f8b..3cdaa84d6c 100644 --- a/decompiler/config/jak1_ntsc_black_label.jsonc +++ b/decompiler/config/jak1_ntsc_black_label.jsonc @@ -105,10 +105,10 @@ //////////////////////////// // PATCHING OPTIONS //////////////////////////// - + // these are options related to xdelta3 patches on specific objects - // this allows us to get a more consistent input - + // this allows us to get a more consistent input + // set to true to write new patch files "write_patches": false, // set to true to apply patch files @@ -151,5 +151,5 @@ "out": "game/assets/jak1/patches/6common.xd3" } } - + } diff --git a/decompiler/data/StrFileReader.cpp b/decompiler/data/StrFileReader.cpp index 83c94ddc91..5f8874fe91 100644 --- a/decompiler/data/StrFileReader.cpp +++ b/decompiler/data/StrFileReader.cpp @@ -14,7 +14,7 @@ #include "game/common/str_rpc_types.h" namespace decompiler { -StrFileReader::StrFileReader(const std::string& file_path) { +StrFileReader::StrFileReader(const std::filesystem::path& file_path) { auto data = file_util::read_binary_file(file_path); ASSERT(data.size() >= SECTOR_SIZE); // must have at least the header sector ASSERT(data.size() % SECTOR_SIZE == 0); // should be multiple of the sector size. diff --git a/decompiler/data/StrFileReader.h b/decompiler/data/StrFileReader.h index c97685cbb2..03d00d8e26 100644 --- a/decompiler/data/StrFileReader.h +++ b/decompiler/data/StrFileReader.h @@ -5,6 +5,7 @@ * Utility class to read a .STR file and extract the full file name. */ +#include #include #include @@ -13,7 +14,7 @@ namespace decompiler { class StrFileReader { public: - explicit StrFileReader(const std::string& file_path); + explicit StrFileReader(const std::filesystem::path& file_path); int chunk_count() const; const std::vector& get_chunk(int idx) const; std::string get_full_name(const std::string& short_name) const; diff --git a/decompiler/data/TextureDB.cpp b/decompiler/data/TextureDB.cpp index 34a53985c1..97d2a0db67 100644 --- a/decompiler/data/TextureDB.cpp +++ b/decompiler/data/TextureDB.cpp @@ -45,7 +45,7 @@ void TextureDB::add_texture(u32 tpage, } } -void TextureDB::replace_textures(const std::string& path) { +void TextureDB::replace_textures(const std::filesystem::path& path) { std::filesystem::path base_path(path); for (auto& tex : textures) { std::filesystem::path full_path = diff --git a/decompiler/data/TextureDB.h b/decompiler/data/TextureDB.h index 23e8c0ae4c..9e2280ace6 100644 --- a/decompiler/data/TextureDB.h +++ b/decompiler/data/TextureDB.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -29,6 +30,6 @@ struct TextureDB { const std::string& tpage_name, const std::vector& level_names); - void replace_textures(const std::string& path); + void replace_textures(const std::filesystem::path& path); }; } // namespace decompiler diff --git a/decompiler/data/streamed_audio.cpp b/decompiler/data/streamed_audio.cpp index e26aa410e1..59c297d99c 100644 --- a/decompiler/data/streamed_audio.cpp +++ b/decompiler/data/streamed_audio.cpp @@ -89,7 +89,7 @@ struct VagFileHeader { /*! * Read the DIR file into an AudioDir */ -AudioDir read_audio_dir(const std::string& path) { +AudioDir read_audio_dir(const std::filesystem::path& path) { // matches the format in file. struct DirEntry { char name[8]; @@ -145,7 +145,8 @@ struct AudioFileInfo { double length_seconds; }; -AudioFileInfo process_audio_file(const std::vector& data, +AudioFileInfo process_audio_file(const std::filesystem::path& output_folder, + const std::vector& data, const std::string& name, const std::string& suffix) { BinaryReader reader(data); @@ -168,10 +169,9 @@ AudioFileInfo process_audio_file(const std::vector& data, ASSERT(reader.read() == 0); } - file_util::create_dir_if_needed(file_util::get_file_path({"assets", "streaming_audio", suffix})); + file_util::create_dir_if_needed(output_folder / suffix); auto file_name = fmt::format("{}.wav", remove_trailing_spaces(name)); - write_wave_file_mono(decoded_samples, header.sample_rate, - file_util::get_file_path({"assets", "streaming_audio", suffix, file_name})); + write_wave_file_mono(decoded_samples, header.sample_rate, output_folder / suffix / file_name); std::string vag_filename; for (int i = 0; i < 16; i++) { @@ -182,11 +182,10 @@ AudioFileInfo process_audio_file(const std::vector& data, return {vag_filename, (double)decoded_samples.size() / header.sample_rate}; } -void process_streamed_audio(const std::string& dir, const std::vector& audio_files) { - auto dir_file_name = file_util::combine_path(dir, "VAGDIR.AYB"); - lg::info("Streaming audio: {}\n", dir_file_name); - file_util::create_dir_if_needed(file_util::get_file_path({"assets", "streaming_audio"})); - auto dir_data = read_audio_dir(file_util::get_file_path({"iso_data", dir_file_name})); +void process_streamed_audio(const std::filesystem::path& output_path, + const std::filesystem::path& input_dir, + const std::vector& audio_files) { + auto dir_data = read_audio_dir(input_dir / "VAG" / "VAGDIR.AYB"); double audio_len = 0.f; std::vector langs; @@ -201,14 +200,14 @@ void process_streamed_audio(const std::string& dir, const std::vector #include #include namespace decompiler { -void process_streamed_audio(const std::string& dir, const std::vector& audio_files); +void process_streamed_audio(const std::filesystem::path& output_path, + const std::filesystem::path& input_dir, + const std::vector& audio_files); } \ No newline at end of file diff --git a/decompiler/data/tpage.cpp b/decompiler/data/tpage.cpp index 4d020c0973..cf4ec7be6b 100644 --- a/decompiler/data/tpage.cpp +++ b/decompiler/data/tpage.cpp @@ -75,7 +75,9 @@ namespace { */ // texture format names. -std::unordered_map psms = {{0x02, "PSMCT16"}, {0x13, "PSMT8"}, {0x14, "PSMT4"}}; +const std::unordered_map psms = {{0x02, "PSMCT16"}, + {0x13, "PSMT8"}, + {0x14, "PSMT4"}}; /*! * GOAL texture type. Stores info about a single texture in a texture page. @@ -417,7 +419,9 @@ TexturePage read_texture_page(ObjectFileData& data, * Process a texture page. * TODO - document */ -TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) { +TPageResultStats process_tpage(ObjectFileData& data, + TextureDB& texture_db, + const std::filesystem::path& output_path) { TPageResultStats stats; auto& words = data.linked_data.words_by_seg.at(0); const auto& level_names = data.dgo_names; @@ -437,6 +441,8 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) { // Read the texture_page struct TexturePage texture_page = read_texture_page(data, words, 0, end_of_texture_page); + auto texture_dump_dir = output_path / texture_page.name; + file_util::create_dir_if_needed(texture_dump_dir); // Get raw data for textures. std::vector tex_data; @@ -518,12 +524,8 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) { } // write texture to a PNG. - file_util::create_dir_if_needed( - file_util::get_file_path({"assets", "textures", texture_page.name})); - file_util::write_rgba_png( - fmt::format(file_util::get_file_path({"assets", "textures", texture_page.name, "{}.png"}), - tex.name), - out.data(), tex.w, tex.h); + file_util::write_rgba_png(texture_dump_dir / fmt::format("{}.png", tex.name), out.data(), + tex.w, tex.h); texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name, texture_page.name, level_names); stats.successful_textures++; @@ -565,12 +567,8 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) { } // write texture to a PNG. - file_util::create_dir_if_needed( - file_util::get_file_path({"assets", "textures", texture_page.name})); - file_util::write_rgba_png( - fmt::format(file_util::get_file_path({"assets", "textures", texture_page.name, "{}.png"}), - tex.name), - out.data(), tex.w, tex.h); + file_util::write_rgba_png(texture_dump_dir / fmt::format("{}.png", tex.name), out.data(), + tex.w, tex.h); texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name, texture_page.name, level_names); stats.successful_textures++; @@ -594,12 +592,8 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) { } // write texture to a PNG. - file_util::create_dir_if_needed( - file_util::get_file_path({"assets", "textures", texture_page.name})); - file_util::write_rgba_png( - fmt::format(file_util::get_file_path({"assets", "textures", texture_page.name, "{}.png"}), - tex.name), - out.data(), tex.w, tex.h); + file_util::write_rgba_png(texture_dump_dir / fmt::format("{}.png", tex.name), out.data(), + tex.w, tex.h); texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name, texture_page.name, level_names); stats.successful_textures++; @@ -639,12 +633,8 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) { } // write texture to a PNG. - file_util::create_dir_if_needed( - file_util::get_file_path({"assets", "textures", texture_page.name})); - file_util::write_rgba_png( - fmt::format(file_util::get_file_path({"assets", "textures", texture_page.name, "{}.png"}), - tex.name), - out.data(), tex.w, tex.h); + file_util::write_rgba_png(texture_dump_dir / fmt::format("{}.png", tex.name), out.data(), + tex.w, tex.h); texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name, texture_page.name, level_names); stats.successful_textures++; @@ -684,12 +674,8 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) { } // write texture to a PNG. - file_util::create_dir_if_needed( - file_util::get_file_path({"assets", "textures", texture_page.name})); - file_util::write_rgba_png( - fmt::format(file_util::get_file_path({"assets", "textures", texture_page.name, "{}.png"}), - tex.name), - out.data(), tex.w, tex.h); + file_util::write_rgba_png(texture_dump_dir / fmt::format("{}.png", tex.name), out.data(), + tex.w, tex.h); texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name, texture_page.name, level_names); stats.successful_textures++; diff --git a/decompiler/data/tpage.h b/decompiler/data/tpage.h index 601696c174..1643302731 100644 --- a/decompiler/data/tpage.h +++ b/decompiler/data/tpage.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "decompiler/data/TextureDB.h" namespace decompiler { @@ -11,5 +13,7 @@ struct TPageResultStats { int num_px = 0; }; -TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db); +TPageResultStats process_tpage(ObjectFileData& data, + TextureDB& texture_db, + const std::filesystem::path& output_path); } // namespace decompiler \ No newline at end of file diff --git a/decompiler/extractor/main.cpp b/decompiler/extractor/main.cpp index 9e9d6b8a9b..8c49746862 100644 --- a/decompiler/extractor/main.cpp +++ b/decompiler/extractor/main.cpp @@ -366,18 +366,18 @@ void decompile(std::filesystem::path jak1_input_files) { .string(), {}); - std::vector dgos, objs; + std::vector dgos, objs; // grab all DGOS we need (level + common) for (const auto& dgo_name : config.dgo_names) { std::string common_name = "GAME.CGO"; if (dgo_name.length() > 3 && dgo_name.substr(dgo_name.length() - 3) == "DGO") { // ends in DGO, it's a level - dgos.push_back((jak1_input_files / dgo_name).string()); + dgos.push_back(jak1_input_files / dgo_name); } else if (dgo_name.length() >= common_name.length() && dgo_name.substr(dgo_name.length() - common_name.length()) == common_name) { // it's COMMON.CGO, we need that too. - dgos.push_back((jak1_input_files / dgo_name).string()); + dgos.push_back(jak1_input_files / dgo_name); } } @@ -385,16 +385,16 @@ void decompile(std::filesystem::path jak1_input_files) { for (const auto& obj_name : config.object_file_names) { if (obj_name.length() > 3 && obj_name.substr(obj_name.length() - 3) == "TXT") { // ends in TXT - objs.push_back((jak1_input_files / obj_name).string()); + objs.push_back(jak1_input_files / obj_name); } } // set up objects - ObjectFileDB db(dgos, config.obj_file_name_map_file, objs, {}, config); + ObjectFileDB db(dgos, std::filesystem::path(config.obj_file_name_map_file), objs, {}, config); // save object files - auto out_folder = (file_util::get_jak_project_dir() / "decompiler_out" / "jak1").string(); - auto raw_obj_folder = file_util::combine_path(out_folder, "raw_obj"); + auto out_folder = file_util::get_jak_project_dir() / "decompiler_out" / "jak1"; + auto raw_obj_folder = out_folder / "raw_obj"; file_util::create_dir_if_needed(raw_obj_folder); db.dump_raw_objects(raw_obj_folder); @@ -404,22 +404,24 @@ void decompile(std::filesystem::path jak1_input_files) { db.process_labels(); // ensure asset dir exists - file_util::create_dir_if_needed(file_util::get_file_path({"assets"})); + file_util::create_dir_if_needed(out_folder / "assets"); // text files { auto result = db.process_game_text_files(config); if (!result.empty()) { - file_util::write_text_file(file_util::get_file_path({"assets", "game_text.txt"}), result); + file_util::write_text_file(out_folder / "assets" / "game_text.txt", result); } } // textures decompiler::TextureDB tex_db; - file_util::write_text_file(file_util::get_file_path({"assets", "tpage-dir.txt"}), - db.process_tpages(tex_db)); + auto textures_out = out_folder / "textures"; + file_util::create_dir_if_needed(textures_out); + file_util::write_text_file(textures_out / "tpage-dir.txt", + db.process_tpages(tex_db, textures_out)); // texture replacements - auto replacements_path = file_util::get_file_path({"texture_replacements"}); + auto replacements_path = file_util::get_jak_project_dir() / "texture_replacements"; if (std::filesystem::exists(replacements_path)) { tex_db.replace_textures(replacements_path); } @@ -428,14 +430,17 @@ void decompile(std::filesystem::path jak1_input_files) { { auto result = db.process_game_count_file(); if (!result.empty()) { - file_util::write_text_file(file_util::get_file_path({"assets", "game_count.txt"}), result); + file_util::write_text_file(out_folder / "assets" / "game_count.txt", result); } } // levels { + // TODO separate out dirs + auto level_out_path = file_util::get_jak_project_dir() / "out" / "fr3"; + file_util::create_dir_if_needed(level_out_path); extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config.hacks, - config.rip_levels, config.extract_collision); + config.rip_levels, config.extract_collision, level_out_path); } } diff --git a/decompiler/level_extractor/extract_level.cpp b/decompiler/level_extractor/extract_level.cpp index 58922eda34..05917531c7 100644 --- a/decompiler/level_extractor/extract_level.cpp +++ b/decompiler/level_extractor/extract_level.cpp @@ -202,7 +202,8 @@ std::vector extract_bsp_from_level(const ObjectFileDB void extract_common(const ObjectFileDB& db, const TextureDB& tex_db, const std::string& dgo_name, - bool dump_levels) { + bool dump_levels, + const std::filesystem::path& output_folder) { if (db.obj_files_by_dgo.count(dgo_name) == 0) { lg::warn("Skipping common extract for {} because the DGO was not part of the input", dgo_name); return; @@ -227,9 +228,9 @@ void extract_common(const ObjectFileDB& db, print_memory_usage(tfrag_level, ser.get_save_result().second); fmt::print("compressed: {} -> {} ({:.2f}%)\n", ser.get_save_result().second, compressed.size(), 100.f * compressed.size() / ser.get_save_result().second); - file_util::write_binary_file(file_util::get_file_path({fmt::format( - "assets/{}.fr3", dgo_name.substr(0, dgo_name.length() - 4))}), - compressed.data(), compressed.size()); + file_util::write_binary_file( + output_folder / fmt::format("{}.fr3", dgo_name.substr(0, dgo_name.length() - 4)), + compressed.data(), compressed.size()); } void extract_from_level(const ObjectFileDB& db, @@ -237,7 +238,8 @@ void extract_from_level(const ObjectFileDB& db, const std::string& dgo_name, const DecompileHacks& hacks, bool dump_level, - bool extract_collision) { + bool extract_collision, + const std::filesystem::path& output_folder) { if (db.obj_files_by_dgo.count(dgo_name) == 0) { lg::warn("Skipping extract for {} because the DGO was not part of the input", dgo_name); return; @@ -258,9 +260,9 @@ void extract_from_level(const ObjectFileDB& db, print_memory_usage(level_data, ser.get_save_result().second); fmt::print("compressed: {} -> {} ({:.2f}%)\n", ser.get_save_result().second, compressed.size(), 100.f * compressed.size() / ser.get_save_result().second); - file_util::write_binary_file(file_util::get_file_path({fmt::format( - "assets/{}.fr3", dgo_name.substr(0, dgo_name.length() - 4))}), - compressed.data(), compressed.size()); + file_util::write_binary_file( + output_folder / fmt::format("{}.fr3", dgo_name.substr(0, dgo_name.length() - 4)), + compressed.data(), compressed.size()); } void extract_all_levels(const ObjectFileDB& db, @@ -269,12 +271,14 @@ void extract_all_levels(const ObjectFileDB& db, const std::string& common_name, const DecompileHacks& hacks, bool debug_dump_level, - bool extract_collision) { - extract_common(db, tex_db, common_name, debug_dump_level); + bool extract_collision, + const std::filesystem::path& output_path) { + extract_common(db, tex_db, common_name, debug_dump_level, output_path); SimpleThreadGroup threads; threads.run( [&](int idx) { - extract_from_level(db, tex_db, dgo_names[idx], hacks, debug_dump_level, extract_collision); + extract_from_level(db, tex_db, dgo_names[idx], hacks, debug_dump_level, extract_collision, + output_path); }, dgo_names.size()); threads.join(); diff --git a/decompiler/level_extractor/extract_level.h b/decompiler/level_extractor/extract_level.h index f83307e0e8..b36cb2deb7 100644 --- a/decompiler/level_extractor/extract_level.h +++ b/decompiler/level_extractor/extract_level.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include "common/math/Vector.h" @@ -15,5 +16,6 @@ void extract_all_levels(const ObjectFileDB& db, const std::string& common_name, const DecompileHacks& hacks, bool debug_dump_level, - bool extract_collision); + bool extract_collision, + const std::filesystem::path& path); } // namespace decompiler diff --git a/decompiler/main.cpp b/decompiler/main.cpp index cf4e02d69f..5140bb6e0d 100644 --- a/decompiler/main.cpp +++ b/decompiler/main.cpp @@ -86,62 +86,62 @@ int main(int argc, char** argv) { return 1; } - std::string in_folder = file_util::combine_path(argv[2], config.game_name); - std::string out_folder = file_util::combine_path(argv[3], config.game_name); + // std::string in_folder = file_util::combine_path(argv[2], config.game_name); + std::filesystem::path in_folder = std::filesystem::path(argv[2]) / config.game_name; + std::filesystem::path out_folder = std::filesystem::path(argv[3]) / config.game_name; // Verify the in_folder is correct - // TODO - refactor to use ghc::filesystem, cleanup file_util - if (!file_util::file_exists(in_folder)) { - fmt::print("Aborting - 'in_folder' does not exist '{}'\n", in_folder); + if (!exists(in_folder)) { + fmt::print("Aborting - 'in_folder' does not exist '{}'\n", in_folder.string()); return 1; } // Warning message if expected ELF isn't found, user could be using bad assets / didn't extract // the ISO properly - if (!config.expected_elf_name.empty() && - !file_util::file_exists(file_util::combine_path(in_folder, config.expected_elf_name))) { + if (!config.expected_elf_name.empty() && !exists(in_folder / config.expected_elf_name)) { fmt::print( "WARNING - '{}' does not contain the expected ELF file '{}'. Was the ISO extracted " "properly or is there a version mismatch?\n", - in_folder, config.expected_elf_name); + in_folder.string(), config.expected_elf_name); } - std::vector dgos, objs, strs; + std::vector dgos, objs, strs; for (const auto& dgo_name : config.dgo_names) { - dgos.push_back(file_util::combine_path(in_folder, dgo_name)); + dgos.push_back(in_folder / dgo_name); } for (const auto& obj_name : config.object_file_names) { - objs.push_back(file_util::combine_path(in_folder, obj_name)); + objs.push_back(in_folder / obj_name); } for (const auto& str_name : config.str_file_names) { - strs.push_back(file_util::combine_path(in_folder, str_name)); + strs.push_back(in_folder / str_name); } file_util::create_dir_if_needed(out_folder); + file_util::create_dir_if_needed(out_folder / "assets"); + if (config.rip_levels) { - file_util::create_dir_if_needed(file_util::get_file_path({"debug_out"})); + file_util::create_dir_if_needed(file_util::get_jak_project_dir() / "debug_out"); } fmt::print("[Mem] After config read: {} MB\n", get_peak_rss() / (1024 * 1024)); // build file database lg::info("Setting up object file DB..."); - ObjectFileDB db(dgos, config.obj_file_name_map_file, objs, strs, config); + ObjectFileDB db(dgos, std::filesystem::path(config.obj_file_name_map_file), objs, strs, config); fmt::print("[Mem] After DB setup: {} MB\n", get_peak_rss() / (1024 * 1024)); // write out DGO file info - file_util::write_text_file(file_util::combine_path(out_folder, "dgo.txt"), - db.generate_dgo_listing()); + file_util::write_text_file(out_folder / "dgo.txt", db.generate_dgo_listing()); // write out object file map (used for future decompilations, if desired) - file_util::write_text_file(file_util::combine_path(out_folder, "obj.txt"), + file_util::write_text_file(out_folder / "obj.txt", db.generate_obj_listing(config.merged_objects)); // dump raw objs if (config.dump_objs) { - auto path = file_util::combine_path(out_folder, "raw_obj"); + auto path = out_folder / "raw_obj"; file_util::create_dir_if_needed(path); db.dump_raw_objects(path); } @@ -176,16 +176,14 @@ int main(int argc, char** argv) { if (config.generate_all_types) { ASSERT_MSG(config.decompile_code, "Must decompile code to generate all-types"); - db.ir2_analyze_all_types(file_util::combine_path(out_folder, "new-all-types.gc"), - config.old_all_types_file, + db.ir2_analyze_all_types(out_folder / "new-all-types.gc", config.old_all_types_file, config.hacks.types_with_bad_inspect_methods); } fmt::print("[Mem] After decomp: {} MB\n", get_peak_rss() / (1024 * 1024)); // write out all symbols - file_util::write_text_file(file_util::combine_path(out_folder, "all-syms.gc"), - db.dts.dump_symbol_types()); + file_util::write_text_file(out_folder / "all-syms.gc", db.dts.dump_symbol_types()); // write art groups if (config.process_art_groups) { @@ -204,7 +202,7 @@ int main(int argc, char** argv) { if (config.process_game_text) { auto result = db.process_game_text_files(config); if (!result.empty()) { - file_util::write_text_file(file_util::get_file_path({"assets", "game_text.txt"}), result); + file_util::write_text_file(out_folder / "assets" / "game_text.txt", result); } } @@ -212,15 +210,16 @@ int main(int argc, char** argv) { decompiler::TextureDB tex_db; if (config.process_tpages || config.levels_extract) { - auto result = db.process_tpages(tex_db); + auto textures_out = out_folder / "textures"; + file_util::create_dir_if_needed(textures_out); + auto result = db.process_tpages(tex_db, textures_out); if (!result.empty() && config.process_tpages) { - file_util::write_text_file(file_util::get_file_path({"assets", "tpage-dir.txt"}), result); + file_util::write_text_file(textures_out / "tpage-dir.txt", result); } } fmt::print("[Mem] After textures: {} MB\n", get_peak_rss() / (1024 * 1024)); - // todo config - auto replacements_path = file_util::get_file_path({"texture_replacements"}); + auto replacements_path = file_util::get_jak_project_dir() / "texture_replacements"; if (std::filesystem::exists(replacements_path)) { tex_db.replace_textures(replacements_path); } @@ -228,19 +227,25 @@ int main(int argc, char** argv) { if (config.process_game_count) { auto result = db.process_game_count_file(); if (!result.empty()) { - file_util::write_text_file(file_util::get_file_path({"assets", "game_count.txt"}), result); + file_util::write_text_file(out_folder / "assets" / "game_count.txt", result); } } if (config.levels_extract) { + // TODO separate out dirs + auto level_out_path = file_util::get_jak_project_dir() / "out" / "fr3"; + file_util::create_dir_if_needed(level_out_path); extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config.hacks, - config.rip_levels, config.extract_collision); + config.rip_levels, config.extract_collision, level_out_path); } fmt::print("[Mem] After extraction: {} MB\n", get_peak_rss() / (1024 * 1024)); if (!config.audio_dir_file_name.empty()) { - process_streamed_audio(config.audio_dir_file_name, config.streamed_audio_file_names); + auto streaming_audio_in = in_folder / "VAG"; + auto streaming_audio_out = out_folder / "assets" / "streaming_audio"; + file_util::create_dir_if_needed(streaming_audio_out); + process_streamed_audio(streaming_audio_out, in_folder, config.streamed_audio_file_names); } lg::info("Decompiler has finished successfully in {:.2f} seconds.", decomp_timer.getSeconds()); diff --git a/game/assets/game_text.gp b/game/assets/game_text.gp index cc045f1fa1..f450149223 100644 --- a/game/assets/game_text.gp +++ b/game/assets/game_text.gp @@ -5,7 +5,7 @@ (text ;; NOTE : we compile using the fixed v2 encoding because it's what we use. - (file "assets/game_text.txt") ;; this is the decompiler-generated file! + (file "$DECOMP/assets/game_text.txt") ;; this is the decompiler-generated file! ;; "patch" files so we can fix some errors and perhaps maintain consistency (file "game/assets/jak1/text/text_patch_ja.gs") ;; add custom files down here diff --git a/game/graphics/opengl_renderer/loader/Loader.cpp b/game/graphics/opengl_renderer/loader/Loader.cpp index 5121dee7ea..8212c26264 100644 --- a/game/graphics/opengl_renderer/loader/Loader.cpp +++ b/game/graphics/opengl_renderer/loader/Loader.cpp @@ -16,7 +16,7 @@ std::string uppercase_string(const std::string& s) { } } // namespace -Loader::Loader() { +Loader::Loader(const std::filesystem::path& base_path) : m_base_path(base_path) { m_loader_thread = std::thread(&Loader::loader_thread, this); m_loader_stages = make_loader_stages(); } @@ -116,8 +116,8 @@ void Loader::loader_thread() { // load the fr3 file Timer disk_timer; - auto data = file_util::read_binary_file( - file_util::get_file_path({fmt::format("assets/{}.fr3", uppercase_string(lev))})); + auto data = + file_util::read_binary_file(m_base_path / fmt::format("{}.fr3", uppercase_string(lev))); double disk_load_time = disk_timer.getSeconds(); // the FR3 files are compressed @@ -167,8 +167,7 @@ void Loader::loader_thread() { * This should be called during initialization, before any threaded loading goes on. */ void Loader::load_common(TexturePool& tex_pool, const std::string& name) { - auto data = - file_util::read_binary_file(file_util::get_file_path({fmt::format("assets/{}.fr3", name)})); + auto data = file_util::read_binary_file(m_base_path / fmt::format("{}.fr3", name)); auto decomp_data = compression::decompress_zstd(data.data(), data.size()); Serializer ser(decomp_data.data(), decomp_data.size()); diff --git a/game/graphics/opengl_renderer/loader/Loader.h b/game/graphics/opengl_renderer/loader/Loader.h index dccdb683ee..5dd103a7c1 100644 --- a/game/graphics/opengl_renderer/loader/Loader.h +++ b/game/graphics/opengl_renderer/loader/Loader.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -15,7 +16,7 @@ class Loader { public: static constexpr float TIE_LOAD_BUDGET = 1.5f; static constexpr float SHARED_TEXTURE_LOAD_BUDGET = 3.f; - Loader(); + Loader(const std::filesystem::path& base_path); ~Loader(); void update(TexturePool& tex_pool); void update_blocking(TexturePool& tex_pool); @@ -50,4 +51,6 @@ class Loader { std::vector m_desired_levels; std::vector> m_loader_stages; + + std::filesystem::path m_base_path; }; diff --git a/game/graphics/pipelines/opengl.cpp b/game/graphics/pipelines/opengl.cpp index 128eec1080..cf692fadfe 100644 --- a/game/graphics/pipelines/opengl.cpp +++ b/game/graphics/pipelines/opengl.cpp @@ -68,7 +68,8 @@ struct GraphicsData { GraphicsData() : dma_copier(EE_MAIN_MEM_SIZE), texture_pool(std::make_shared()), - loader(std::make_shared()), + // TODO out dir + loader(std::make_shared(file_util::get_jak_project_dir() / "out" / "fr3")), ogl_renderer(texture_pool, loader) {} }; diff --git a/goal_src/jak1/game.gp b/goal_src/jak1/game.gp index ff1e6067b7..f0da9283c7 100644 --- a/goal_src/jak1/game.gp +++ b/goal_src/jak1/game.gp @@ -25,6 +25,50 @@ ;; It is an error to not provide a step to make a required file. ;; It is an error to have a circular dependency and this will crash the compiler due to stack overflow. +;; our input/output directories aren't fixed ahead of time and some users may override them. +;; the map-path! command can be used to define path constants that can be used in in/out/dep and +;; inside certain tool files, like text project files. + +;; The substitution is only used at the beginning of paths, and I hope to keep the number +;; of these to a minimum - it's quite confusing to have all these non-fixed paths everywhere. + +;;;;;;;;;;;;;;;;;;;;;;; +;; Inputs from ISO +;;;;;;;;;;;;;;;;;;;;;;; + +(cond + ;; extractor can override everything by providing *use-iso-data-path* + (*use-iso-data-path* + (map-path! "$ISO" (string-append *iso-data* "/"))) + ;; user-specific places to put $ISO + ((user? dass) + (map-path! "$ISO" "iso_data/jak1_us2/")) + ;; for normal people, just use jak1. + (#t + (map-path! "$ISO" "iso_data/jak1/"))) + +;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Inputs from decompiler +;;;;;;;;;;;;;;;;;;;;;;;;;; + +(cond + ;; user-specific places to put $ISO + ((user? dass) + (map-path! "$DECOMP" "decompiler_out/jak1_us2/")) + (#t + (map-path! "$DECOMP" "decompiler_out/jak1/"))) + +;;;;;;;;;;;;;;;;;;;;;;; +;; Output +;;;;;;;;;;;;;;;;;;;;;;; + +;; NOTE: this isn't perfect yet, some tools still are hardcoded to look in out/ for stuff. +(map-path! "$OUT" "out/") + + +;; use defmacro to define goos macros. +(define defmacro defsmacro) +(define defun desfun) ;;;;;;;;;;;;;;;;;;;;;;; ;; Build Groups ;;;;;;;;;;;;;;;;;;;;;;; @@ -36,18 +80,12 @@ (define *all-vag* '()) (define *all-gc* '()) - ;;;;;;;;;;;;;;;;;;;;;;; ;; Build system macros ;;;;;;;;;;;;;;;;;;;;;;; - -;; use defmacro to define goos macros. -(define defmacro defsmacro) -(define defun desfun) - (defun gc-file->o-file (filename) "Get the name of the object file for the given GOAL (*.gc) source file." - (string-append "out/obj/" (stem filename) ".o") + (string-append "$OUT/obj/" (stem filename) ".o") ) (defmacro goal-src (src-file &rest deps) @@ -105,7 +143,7 @@ (defun custom-level-cgo (output-name desc-file-name) "Add a CGO with the given output name (in out/iso) and input name (in custom_levels/)" - (let ((out-name (string-append "out/iso/" output-name))) + (let ((out-name (string-append "$OUT/iso/" output-name))) (defstep :in (string-append "custom_levels/" desc-file-name) :tool 'dgo :out `(,out-name) @@ -116,7 +154,7 @@ (defun cgo (output-name desc-file-name) "Add a CGO with the given output name (in out/iso) and input name (in goal_src/jak1/dgos)" - (let ((out-name (string-append "out/iso/" output-name))) + (let ((out-name (string-append "$OUT/iso/" output-name))) (defstep :in (string-append "goal_src/jak1/dgos/" desc-file-name) :tool 'dgo :out `(,out-name) @@ -131,16 +169,13 @@ ) -(define *game-directory* (get-environment-variable "OPENGOAL_DECOMP_DIR" :default "jak1/")) -(when (user? dass) - (set! *game-directory* "jak1_us2/")) (defmacro copy-texture (tpage-id) "Copy a texture from the game, using the given tpage ID" - (let* ((path (string-append "decompiler_out/" *game-directory* "raw_obj/" (tpage-name tpage-id)))) + (let* ((path (string-append "$DECOMP/raw_obj/" (tpage-name tpage-id)))) `(defstep :in ,path :tool 'copy - :out '(,(string-append "out/obj/" (tpage-name tpage-id)))))) + :out '(,(string-append "$OUT/obj/" (tpage-name tpage-id)))))) (defmacro copy-textures (&rest ids) `(begin @@ -149,10 +184,10 @@ ) (defmacro copy-go (name) - (let* ((path (string-append "decompiler_out/" *game-directory* "raw_obj/" name ".go"))) + (let* ((path (string-append "$DECOMP/raw_obj/" name ".go"))) `(defstep :in ,path :tool 'copy - :out '(,(string-append "out/obj/" name ".go"))))) + :out '(,(string-append "$OUT/obj/" name ".go"))))) (defmacro copy-gos (&rest gos) `(begin @@ -164,18 +199,12 @@ (let* ((path (string-append "custom_levels/" name "/" name ".jsonc"))) `(defstep :in ,path :tool 'build-level - :out '(,(string-append "out/obj/" name ".go"))))) + :out '(,(string-append "$OUT/obj/" name ".go"))))) -(defun get-iso-data-path () - (if *use-iso-data-path* - (string-append *iso-data* "/") - (string-append "iso_data/" *game-directory* "/") - ) - ) (defun copy-iso-file (name subdir ext) - (let* ((path (string-append (get-iso-data-path) subdir name ext)) - (out-name (string-append "out/iso/" name ext))) + (let* ((path (string-append "$ISO/" subdir name ext)) + (out-name (string-append "$OUT/iso/" name ext))) (defstep :in path :tool 'copy :out `(,out-name)) @@ -248,36 +277,36 @@ ;;;;;;;;;;;;;;;;;;;;;;;; ;; The tpage directory -(defstep :in "assets/tpage-dir.txt" +(defstep :in "$DECOMP/textures/tpage-dir.txt" :tool 'tpage-dir - :out '("out/obj/dir-tpages.go") + :out '("$OUT/obj/dir-tpages.go") ) ;; the count file. -(defstep :in "assets/game_count.txt" +(defstep :in "$DECOMP/assets/game_count.txt" :tool 'game-cnt - :out '("out/obj/game-cnt.go") + :out '("$OUT/obj/game-cnt.go") ) ;; the TWEAKVAL file -(defstep :in (string-append (get-iso-data-path) "MUS/TWEAKVAL.MUS") +(defstep :in "$ISO/MUS/TWEAKVAL.MUS" :tool 'copy - :out '("out/iso/TWEAKVAL.MUS")) + :out '("$OUT/iso/TWEAKVAL.MUS")) ;; the VAGDIR file -(defstep :in (string-append (get-iso-data-path) "VAG/VAGDIR.AYB") +(defstep :in "$ISO/VAG/VAGDIR.AYB" :tool 'copy - :out '("out/iso/VAGDIR.AYB")) + :out '("$OUT/iso/VAGDIR.AYB")) ;; the save icon file -(defstep :in (string-append (get-iso-data-path) "DRIVERS/SAVEGAME.ICO") +(defstep :in "$ISO/DRIVERS/SAVEGAME.ICO" :tool 'copy - :out '("out/iso/SAVEGAME.ICO")) + :out '("$OUT/iso/SAVEGAME.ICO")) ;; the loading screen file -(defstep :in (string-append (get-iso-data-path) "DRIVERS/SCREEN1.USA") +(defstep :in "$ISO/DRIVERS/SCREEN1.USA" :tool 'copy - :out '("out/iso/SCREEN1.USA")) + :out '("$OUT/iso/SCREEN1.USA")) ;;;;;;;;;;;;;;;;;;;;; ;; Textures (Common) @@ -350,20 +379,20 @@ (defstep :in "game/assets/game_text.gp" :tool 'text - :out '("out/iso/0COMMON.TXT" - "out/iso/1COMMON.TXT" - "out/iso/2COMMON.TXT" - "out/iso/3COMMON.TXT" - "out/iso/4COMMON.TXT" - "out/iso/5COMMON.TXT" - "out/iso/6COMMON.TXT") + :out '("$OUT/iso/0COMMON.TXT" + "$OUT/iso/1COMMON.TXT" + "$OUT/iso/2COMMON.TXT" + "$OUT/iso/3COMMON.TXT" + "$OUT/iso/4COMMON.TXT" + "$OUT/iso/5COMMON.TXT" + "$OUT/iso/6COMMON.TXT") ) (defstep :in "game/assets/game_subtitle.gp" :tool 'subtitle - :out '("out/iso/0SUBTIT.TXT" - "out/iso/3SUBTIT.TXT" - "out/iso/6SUBTIT.TXT") + :out '("$OUT/iso/0SUBTIT.TXT" + "$OUT/iso/3SUBTIT.TXT" + "$OUT/iso/6SUBTIT.TXT") ) @@ -374,10 +403,10 @@ ;; the engine group is a group of files required to boot the game engine with no levels (group "engine" - "out/iso/0COMMON.TXT" - "out/iso/0SUBTIT.TXT" - "out/iso/KERNEL.CGO" - "out/iso/GAME.CGO" + "$OUT/iso/0COMMON.TXT" + "$OUT/iso/0SUBTIT.TXT" + "$OUT/iso/KERNEL.CGO" + "$OUT/iso/GAME.CGO" ) ;;;;;;;;;;;;;;;;;;;;;;;; @@ -417,7 +446,7 @@ (goal-src-sequence "levels/" :deps ;; no idea what these depend on, make it depend on the whole engine - ("out/obj/ticky.o") + ("$OUT/obj/ticky.o") "village_common/villagep-obs.gc" "village_common/oracle.gc" @@ -453,7 +482,7 @@ (goal-src-sequence "levels/beach/" - :deps ("out/obj/ticky.o") + :deps ("$OUT/obj/ticky.o") "air-h.gc" "air.gc" "wobbler.gc" @@ -527,7 +556,7 @@ (goal-src-sequence "levels/jungle/" :deps ;; no idea what these depend on, make it depend on the whole engine - ("out/obj/ticky.o") + ("$OUT/obj/ticky.o") "jungle-elevator.gc" "bouncer.gc" @@ -598,7 +627,7 @@ (goal-src-sequence "levels/" :deps ;; no idea what these depend on, make it depend on the whole engine - ("out/obj/ticky.o") + ("$OUT/obj/ticky.o") "village1/farmer.gc" "village1/explorer.gc" @@ -677,7 +706,7 @@ (goal-src-sequence "levels/jungleb/" :deps ;; no idea what these depend on, make it depend on the whole engine - ("out/obj/ticky.o") + ("$OUT/obj/ticky.o") "jungleb-obs.gc" "plat-flip.gc" @@ -710,7 +739,7 @@ (goal-src-sequence "levels/misty/" - :deps ("out/obj/evilbro.o") + :deps ("$OUT/obj/evilbro.o") "mistycannon.gc" "babak-with-cannon.gc" "misty-obs.gc" @@ -778,7 +807,7 @@ (goal-src-sequence "levels/swamp/" - :deps ("out/obj/ticky.o") + :deps ("$OUT/obj/ticky.o") "swamp-obs.gc" "swamp-bat.gc" "swamp-rat.gc" @@ -822,7 +851,7 @@ (goal-src-sequence "levels/sunken/" - :deps ("out/obj/ticky.o") + :deps ("$OUT/obj/ticky.o") "sunken-part.gc" "sunken-part2.gc" "sunken-part3.gc" @@ -915,7 +944,7 @@ (goal-src-sequence "levels/snow/" - :deps ("out/obj/ticky.o") + :deps ("$OUT/obj/ticky.o") "target-snowball.gc" "target-ice.gc" "ice-cube.gc" @@ -979,7 +1008,7 @@ (goal-src-sequence "levels/firecanyon/" :deps ;; no idea what these depend on, make it depend on the whole engine - ("out/obj/ticky.o") + ("$OUT/obj/ticky.o") "firecanyon-part.gc" "assistant-firecanyon.gc" @@ -1011,7 +1040,7 @@ (goal-src-sequence "levels/ogre/" - :deps ("out/obj/ticky.o") + :deps ("$OUT/obj/ticky.o") "ogre-part.gc" "ogreboss.gc" "ogre-obs.gc" @@ -1052,7 +1081,7 @@ (goal-src-sequence "levels/village2/" - :deps ("out/obj/ticky.o") + :deps ("$OUT/obj/ticky.o") "village2-part.gc" "village2-obs.gc" "village2-part2.gc" @@ -1122,7 +1151,7 @@ (goal-src-sequence "levels/rolling/" - :deps ("out/obj/ticky.o") + :deps ("$OUT/obj/ticky.o") "rolling-obs.gc" "rolling-lightning-mole.gc" "rolling-robber.gc" @@ -1164,7 +1193,7 @@ (goal-src-sequence "levels/" :deps ;; no idea what these depend on, make it depend on the whole engine - ("out/obj/ticky.o") + ("$OUT/obj/ticky.o") "village3/village3-part.gc" "village3/village3-obs.gc" "village3/minecart.gc" @@ -1222,7 +1251,7 @@ ;; The code (goal-src-sequence "levels/training/" - :deps ("out/obj/ticky.o") ;; makes us depend on the whole engine + :deps ("$OUT/obj/ticky.o") ;; makes us depend on the whole engine "training-obs.gc" "training-part.gc" @@ -1251,7 +1280,7 @@ (goal-src-sequence "levels/" :deps ;; no idea what these depend on, make it depend on the whole engine - ("out/obj/ticky.o" + ("$OUT/obj/ticky.o" ) "maincave/cavecrystal-light.gc" "darkcave/darkcave-obs.gc" @@ -1318,7 +1347,7 @@ (goal-src-sequence "levels/robocave/" :deps ;; no idea what these depend on, make it depend on the whole engine - ("out/obj/ticky.o") + ("$OUT/obj/ticky.o") "cave-trap.gc" "spider-egg.gc" "robocave-part.gc" @@ -1346,7 +1375,7 @@ (goal-src-sequence "levels/lavatube/" - :deps ("out/obj/ticky.o") + :deps ("$OUT/obj/ticky.o") "lavatube-obs.gc" "lavatube-energy.gc" @@ -1391,7 +1420,7 @@ (goal-src-sequence "levels/citadel/" - :deps ("out/obj/battlecontroller.o" "out/obj/snow-bunny.o") + :deps ("$OUT/obj/battlecontroller.o" "out/obj/snow-bunny.o") "citadel-part.gc" "citadel-obs.gc" @@ -1455,7 +1484,7 @@ (goal-src-sequence "levels/finalboss/" - :deps ("out/obj/assistant-citadel.o") + :deps ("$OUT/obj/assistant-citadel.o") "robotboss-h.gc" "robotboss-part.gc" @@ -1510,7 +1539,7 @@ (goal-src-sequence "levels/intro/" :deps ;; no idea what these depend on, make it depend on the whole engine - ("out/obj/ticky.o") + ("$OUT/obj/ticky.o") "evilbro.gc" ) @@ -1537,7 +1566,7 @@ (goal-src-sequence "levels/demo/" :deps ;; no idea what these depend on, make it depend on the whole engine - ("out/obj/ticky.o") + ("$OUT/obj/ticky.o") "static-screen.gc" "demo-obs.gc" @@ -1559,7 +1588,7 @@ (goal-src-sequence "levels/title/" :deps ;; no idea what these depend on, make it depend on the whole engine - ("out/obj/ticky.o") + ("$OUT/obj/ticky.o") "title-obs.gc" ) @@ -1600,10 +1629,10 @@ "engine/" :deps - ("out/obj/gcommon.o" - "out/obj/gstate.o" - "out/obj/gstring.o" - "out/obj/gkernel.o" + ("$OUT/obj/gcommon.o" + "$OUT/obj/gstate.o" + "$OUT/obj/gstring.o" + "$OUT/obj/gkernel.o" ) ;; sources @@ -1648,7 +1677,7 @@ "engine/" :deps - ("out/ps2/pad.o") + ("$OUT/ps2/pad.o") "gfx/hw/gs.gc" "gfx/hw/display-h.gc" "math/vector.gc" @@ -1875,7 +1904,7 @@ "engine/" :deps - ("out/main.o") + ("$OUT/main.o") "collide/collide-cache.gc" "entity/relocate.gc" @@ -1919,7 +1948,7 @@ (goal-src-sequence "levels/common/" - :deps ("out/obj/default-menu.o") + :deps ("$OUT/obj/default-menu.o") "texture-upload.gc" "rigid-body-h.gc" "water-anim.gc" @@ -1951,10 +1980,10 @@ ;; the iso group is a group of files built by the "(mi)" command. (group-list "iso" - `("out/iso/0COMMON.TXT" - "out/iso/0SUBTIT.TXT" - "out/iso/TWEAKVAL.MUS" - "out/iso/VAGDIR.AYB" + `("$OUT/iso/0COMMON.TXT" + "$OUT/iso/0SUBTIT.TXT" + "$OUT/iso/TWEAKVAL.MUS" + "$OUT/iso/VAGDIR.AYB" ,@(reverse *all-vis*) ,@(reverse *all-str*) ,@(reverse *all-sbk*) @@ -1969,8 +1998,8 @@ (group-list "text" - `("out/iso/0COMMON.TXT" - "out/iso/0SUBTIT.TXT" + `("$OUT/iso/0COMMON.TXT" + "$OUT/iso/0SUBTIT.TXT" ) ) diff --git a/goalc/build_level/build_level.cpp b/goalc/build_level/build_level.cpp index a8da5a1842..3f6e04a9d1 100644 --- a/goalc/build_level/build_level.cpp +++ b/goalc/build_level/build_level.cpp @@ -14,7 +14,9 @@ #include "third-party/fmt/core.h" -void save_pc_data(const std::string& nickname, tfrag3::Level& data) { +void save_pc_data(const std::string& nickname, + tfrag3::Level& data, + const std::filesystem::path& output_dir) { Serializer ser; data.serialize(ser); auto compressed = @@ -23,8 +25,8 @@ void save_pc_data(const std::string& nickname, tfrag3::Level& data) { print_memory_usage(data, ser.get_save_result().second); fmt::print("compressed: {} -> {} ({:.2f}%)\n", ser.get_save_result().second, compressed.size(), 100.f * compressed.size() / ser.get_save_result().second); - file_util::write_binary_file(file_util::get_file_path({fmt::format("assets/{}.fr3", nickname)}), - compressed.data(), compressed.size()); + file_util::write_binary_file(output_dir / fmt::format("{}.fr3", nickname), compressed.data(), + compressed.size()); } std::vector get_build_level_deps(const std::string& input_file) { @@ -106,7 +108,8 @@ bool run_build_level(const std::string& input_file, const std::string& output_fi file_util::write_binary_file(save_path, result.data(), result.size()); // Save the PC level - save_pc_data(file.nickname, pc_level); + // TODO out dir + save_pc_data(file.nickname, pc_level, file_util::get_jak_project_dir() / "out" / "fr3"); return true; } diff --git a/goalc/make/MakeSystem.cpp b/goalc/make/MakeSystem.cpp index 171c0ff99e..416ef8c875 100644 --- a/goalc/make/MakeSystem.cpp +++ b/goalc/make/MakeSystem.cpp @@ -54,6 +54,11 @@ MakeSystem::MakeSystem(const std::string& username) : m_goos(username) { return handle_stem(obj, args, env); }); + m_goos.register_form("map-path!", [=](const goos::Object& obj, goos::Arguments& args, + const std::shared_ptr& env) { + return handle_map_path(obj, args, env); + }); + m_goos.set_global_variable_to_symbol("ASSETS", "#t"); set_constant("*iso-data*", file_util::get_file_path({"iso_data"})); @@ -98,7 +103,7 @@ goos::Object MakeSystem::handle_defstep(const goos::Object& form, auto step = std::make_shared(); goos::for_each_in_list(args.get_named("out"), [&](const goos::Object& obj) { - step->outputs.push_back(obj.as_string()->data); + step->outputs.push_back(m_path_map.apply_remaps(obj.as_string()->data)); }); step->tool = args.get_named("tool").as_symbol()->name; @@ -111,16 +116,17 @@ goos::Object MakeSystem::handle_defstep(const goos::Object& form, const auto& in = args.get_named("in"); if (in.is_pair()) { step->input.clear(); - goos::for_each_in_list( - in, [&](const goos::Object& o) { step->input.push_back(o.as_string()->data); }); + goos::for_each_in_list(in, [&](const goos::Object& o) { + step->input.push_back(m_path_map.apply_remaps(o.as_string()->data)); + }); } else { - step->input = {in.as_string()->data}; + step->input = {m_path_map.apply_remaps(in.as_string()->data)}; } } if (args.has_named("dep")) { goos::for_each_in_list(args.get_named("dep"), [&](const goos::Object& obj) { - step->deps.push_back(obj.as_string()->data); + step->deps.push_back(m_path_map.apply_remaps(obj.as_string()->data)); }); } @@ -182,6 +188,20 @@ goos::Object MakeSystem::handle_stem(const goos::Object& form, return goos::StringObject::make_new(input.stem().u8string()); } +goos::Object MakeSystem::handle_map_path(const goos::Object& form, + goos::Arguments& args, + const std::shared_ptr& env) { + m_goos.eval_args(&args, env); + va_check(form, args, {goos::ObjectType::STRING, goos::ObjectType::STRING}, {}); + auto old_path = args.unnamed.at(0).as_string()->data; + if (old_path.empty() || old_path[0] != '$') { + throw std::runtime_error(fmt::format("Invalid path remap {}, must start with $", old_path)); + } + auto new_path = args.unnamed.at(1).as_string()->data; + m_path_map.path_remap[old_path] = new_path; + return goos::Object::make_empty_list(); +} + void MakeSystem::get_dependencies(const std::string& master_target, const std::string& output, std::vector* result, @@ -202,9 +222,9 @@ void MakeSystem::get_dependencies(const std::string& master_target, } const auto& rule = rule_it->second; - for (auto& dep : - m_tools.at(rule->tool) - ->get_additional_dependencies({rule->input, rule->deps, rule->outputs, rule->arg})) { + for (auto& dep : m_tools.at(rule->tool) + ->get_additional_dependencies( + {rule->input, rule->deps, rule->outputs, rule->arg}, m_path_map)) { get_dependencies(master_target, dep, result, result_set); } @@ -247,7 +267,7 @@ std::vector MakeSystem::filter_dependencies(const std::vectorneeds_run(task)) { + if (tool->needs_run(task, m_path_map)) { result.push_back(to_make); stale_deps.insert(to_make); added = true; @@ -267,7 +287,7 @@ std::vector MakeSystem::filter_dependencies(const std::vectorget_additional_dependencies(task)) { + for (auto& dep : tool->get_additional_dependencies(task, m_path_map)) { if (stale_deps.find(dep) != stale_deps.end()) { result.push_back(to_make); stale_deps.insert(to_make); @@ -334,7 +354,7 @@ bool MakeSystem::make(const std::string& target, bool force, bool verbose) { bool success = false; try { - success = tool->run({rule->input, rule->deps, rule->outputs, rule->arg}); + success = tool->run({rule->input, rule->deps, rule->outputs, rule->arg}, m_path_map); } catch (std::exception& e) { fmt::print("\n"); fmt::print("Error: {}\n", e.what()); diff --git a/goalc/make/MakeSystem.h b/goalc/make/MakeSystem.h index 247e1ee056..63a0db6297 100644 --- a/goalc/make/MakeSystem.h +++ b/goalc/make/MakeSystem.h @@ -30,6 +30,10 @@ class MakeSystem { goos::Arguments&, const std::shared_ptr& env); + goos::Object handle_map_path(const goos::Object& obj, + goos::Arguments& args, + const std::shared_ptr& env); + std::vector get_dependencies(const std::string& target) const; std::vector filter_dependencies(const std::vector& all_deps); @@ -62,4 +66,5 @@ class MakeSystem { std::unordered_map> m_output_to_step; std::unordered_map> m_tools; + PathMap m_path_map; }; diff --git a/goalc/make/Tool.cpp b/goalc/make/Tool.cpp index e2e7fa2b98..69f1f2a2b8 100644 --- a/goalc/make/Tool.cpp +++ b/goalc/make/Tool.cpp @@ -10,7 +10,7 @@ Tool::Tool(const std::string& name) : m_name(name) {} -bool Tool::needs_run(const ToolInput& task) { +bool Tool::needs_run(const ToolInput& task, const PathMap& path_map) { // for this to return false, all outputs need to be newer than all inputs. for (auto& in : task.input) { @@ -33,7 +33,7 @@ bool Tool::needs_run(const ToolInput& task) { } } - for (auto& dep : get_additional_dependencies(task)) { + for (auto& dep : get_additional_dependencies(task, path_map)) { auto dep_path = std::filesystem::path(file_util::get_file_path({dep})); if (std::filesystem::exists(dep_path)) { auto dep_time = std::filesystem::last_write_time(dep_path); @@ -60,3 +60,31 @@ bool Tool::needs_run(const ToolInput& task) { return false; } + +std::string PathMap::apply_remaps(const std::string& input) const { + if (!input.empty() && input[0] == '$') { + std::string prefix = "$"; + size_t i = 1; + for (; i < input.size(); i++) { + char c = input[i]; + if (c == '/') { + break; + } else { + prefix.push_back(c); + } + } + const auto& it = path_remap.find(prefix); + if (it == path_remap.end()) { + return input; + } else { + std::string result = it->second; + while (!result.empty() && result.back() == '/') { + result.pop_back(); + } + result.insert(result.end(), input.begin() + i, input.end()); + return result; + } + } else { + return input; + } +} \ No newline at end of file diff --git a/goalc/make/Tool.h b/goalc/make/Tool.h index d1d2168a94..d594d3db74 100644 --- a/goalc/make/Tool.h +++ b/goalc/make/Tool.h @@ -6,6 +6,11 @@ #include "common/goos/Object.h" +struct PathMap { + std::unordered_map path_remap; + std::string apply_remaps(const std::string& input) const; +}; + struct ToolInput { std::vector& input; // the input file std::vector& deps; // explicit dependencies @@ -16,9 +21,12 @@ struct ToolInput { class Tool { public: Tool(const std::string& name); - virtual bool run(const ToolInput& task) = 0; - virtual std::vector get_additional_dependencies(const ToolInput&) { return {}; } - virtual bool needs_run(const ToolInput& task); + virtual bool run(const ToolInput& task, const PathMap& path_map) = 0; + virtual std::vector get_additional_dependencies(const ToolInput&, + const PathMap& /*path_map*/) { + return {}; + } + virtual bool needs_run(const ToolInput& task, const PathMap& path_map); virtual ~Tool() = default; const std::string& name() const { return m_name; } diff --git a/goalc/make/Tools.cpp b/goalc/make/Tools.cpp index 99a3e8e5ec..a267d6c2ef 100644 --- a/goalc/make/Tools.cpp +++ b/goalc/make/Tools.cpp @@ -16,7 +16,7 @@ CompilerTool::CompilerTool(Compiler* compiler) : Tool("goalc"), m_compiler(compiler) {} -bool CompilerTool::needs_run(const ToolInput& task) { +bool CompilerTool::needs_run(const ToolInput& task, const PathMap& path_map) { if (task.input.size() != 1) { throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name())); } @@ -24,10 +24,10 @@ bool CompilerTool::needs_run(const ToolInput& task) { if (!m_compiler->knows_object_file(std::filesystem::path(task.input.at(0)).stem().u8string())) { return true; } - return Tool::needs_run(task); + return Tool::needs_run(task, path_map); } -bool CompilerTool::run(const ToolInput& task) { +bool CompilerTool::run(const ToolInput& task, const PathMap& /*path_map*/) { // todo check inputs try { CompilationOptions options; @@ -76,7 +76,7 @@ DgoDescription parse_desc_file(const std::string& filename, goos::Reader& reader DgoTool::DgoTool() : Tool("dgo") {} -bool DgoTool::run(const ToolInput& task) { +bool DgoTool::run(const ToolInput& task, const PathMap& /*path_map*/) { if (task.input.size() != 1) { throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name())); } @@ -85,10 +85,12 @@ bool DgoTool::run(const ToolInput& task) { return true; } -std::vector DgoTool::get_additional_dependencies(const ToolInput& task) { +std::vector DgoTool::get_additional_dependencies(const ToolInput& task, + const PathMap& /*path_map*/) { std::vector result; auto desc = parse_desc_file(task.input.at(0), m_reader); for (auto& x : desc.entries) { + // todo out result.push_back(fmt::format("out/obj/{}", x.file_name)); } return result; @@ -96,7 +98,7 @@ std::vector DgoTool::get_additional_dependencies(const ToolInput& t TpageDirTool::TpageDirTool() : Tool("tpage-dir") {} -bool TpageDirTool::run(const ToolInput& task) { +bool TpageDirTool::run(const ToolInput& task, const PathMap& /*path_map*/) { if (task.input.size() != 1) { throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name())); } @@ -106,7 +108,7 @@ bool TpageDirTool::run(const ToolInput& task) { CopyTool::CopyTool() : Tool("copy") {} -bool CopyTool::run(const ToolInput& task) { +bool CopyTool::run(const ToolInput& task, const PathMap& /*path_map*/) { if (task.input.size() != 1) { throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name())); } @@ -120,7 +122,7 @@ bool CopyTool::run(const ToolInput& task) { GameCntTool::GameCntTool() : Tool("game-cnt") {} -bool GameCntTool::run(const ToolInput& task) { +bool GameCntTool::run(const ToolInput& task, const PathMap& /*path_map*/) { if (task.input.size() != 1) { throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name())); } @@ -130,63 +132,75 @@ bool GameCntTool::run(const ToolInput& task) { TextTool::TextTool() : Tool("text") {} -bool TextTool::needs_run(const ToolInput& task) { +bool TextTool::needs_run(const ToolInput& task, const PathMap& path_map) { if (task.input.size() != 1) { throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name())); } std::vector deps; open_text_project("text", task.input.at(0), deps); - return Tool::needs_run({task.input, deps, task.output, task.arg}); + for (auto& dep : deps) { + dep = path_map.apply_remaps(dep); + } + return Tool::needs_run({task.input, deps, task.output, task.arg}, path_map); } -bool TextTool::run(const ToolInput& task) { +bool TextTool::run(const ToolInput& task, const PathMap& path_map) { GameTextDB db; std::vector inputs; open_text_project("text", task.input.at(0), inputs); + for (auto& in : inputs) { + in = path_map.apply_remaps(in); + } compile_game_text(inputs, db); return true; } GroupTool::GroupTool() : Tool("group") {} -bool GroupTool::run(const ToolInput&) { +bool GroupTool::run(const ToolInput&, const PathMap& /*path_map*/) { return true; } SubtitleTool::SubtitleTool() : Tool("subtitle") {} -bool SubtitleTool::needs_run(const ToolInput& task) { +bool SubtitleTool::needs_run(const ToolInput& task, const PathMap& path_map) { if (task.input.size() != 1) { throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name())); } std::vector deps; open_text_project("subtitle", task.input.at(0), deps); - return Tool::needs_run({task.input, deps, task.output, task.arg}); + for (auto& dep : deps) { + dep = path_map.apply_remaps(dep); + } + return Tool::needs_run({task.input, deps, task.output, task.arg}, path_map); } -bool SubtitleTool::run(const ToolInput& task) { +bool SubtitleTool::run(const ToolInput& task, const PathMap& path_map) { GameSubtitleDB db; db.m_subtitle_groups = std::make_unique(); db.m_subtitle_groups->hydrate_from_asset_file(); std::vector inputs; open_text_project("subtitle", task.input.at(0), inputs); + for (auto& in : inputs) { + in = path_map.apply_remaps(in); + } compile_game_subtitle(inputs, db); return true; } BuildLevelTool::BuildLevelTool() : Tool("build-level") {} -bool BuildLevelTool::needs_run(const ToolInput& task) { +bool BuildLevelTool::needs_run(const ToolInput& task, const PathMap& path_map) { if (task.input.size() != 1) { throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name())); } auto deps = get_build_level_deps(task.input.at(0)); - return Tool::needs_run({task.input, deps, task.output, task.arg}); + return Tool::needs_run({task.input, deps, task.output, task.arg}, path_map); } -bool BuildLevelTool::run(const ToolInput& task) { +bool BuildLevelTool::run(const ToolInput& task, const PathMap& /*path_map*/) { if (task.input.size() != 1) { throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name())); } diff --git a/goalc/make/Tools.h b/goalc/make/Tools.h index 59a0390d08..c3cadd253e 100644 --- a/goalc/make/Tools.h +++ b/goalc/make/Tools.h @@ -9,8 +9,8 @@ class Compiler; class CompilerTool : public Tool { public: CompilerTool(Compiler* compiler); - bool run(const ToolInput& task) override; - bool needs_run(const ToolInput& task) override; + bool run(const ToolInput& task, const PathMap& path_map) override; + bool needs_run(const ToolInput& task, const PathMap& path_map) override; private: Compiler* m_compiler = nullptr; @@ -19,8 +19,9 @@ class CompilerTool : public Tool { class DgoTool : public Tool { public: DgoTool(); - bool run(const ToolInput& task) override; - std::vector get_additional_dependencies(const ToolInput&) override; + bool run(const ToolInput& task, const PathMap& path_map) override; + std::vector get_additional_dependencies(const ToolInput&, + const PathMap& path_map) override; private: goos::Reader m_reader; @@ -29,44 +30,44 @@ class DgoTool : public Tool { class TpageDirTool : public Tool { public: TpageDirTool(); - bool run(const ToolInput& task) override; + bool run(const ToolInput& task, const PathMap& path_map) override; }; class CopyTool : public Tool { public: CopyTool(); - bool run(const ToolInput& task) override; + bool run(const ToolInput& task, const PathMap& path_map) override; }; class GameCntTool : public Tool { public: GameCntTool(); - bool run(const ToolInput& task) override; + bool run(const ToolInput& task, const PathMap& path_map) override; }; class TextTool : public Tool { public: TextTool(); - bool run(const ToolInput& task) override; - bool needs_run(const ToolInput& task) override; + bool run(const ToolInput& task, const PathMap& path_map) override; + bool needs_run(const ToolInput& task, const PathMap& path_map) override; }; class GroupTool : public Tool { public: GroupTool(); - bool run(const ToolInput& task) override; + bool run(const ToolInput& task, const PathMap& path_map) override; }; class SubtitleTool : public Tool { public: SubtitleTool(); - bool run(const ToolInput& task) override; - bool needs_run(const ToolInput& task) override; + bool run(const ToolInput& task, const PathMap& path_map) override; + bool needs_run(const ToolInput& task, const PathMap& path_map) override; }; class BuildLevelTool : public Tool { public: BuildLevelTool(); - bool run(const ToolInput& task) override; - bool needs_run(const ToolInput& task) override; + bool run(const ToolInput& task, const PathMap& path_map) override; + bool needs_run(const ToolInput& task, const PathMap& path_map) override; }; diff --git a/test/offline/offline_test_main.cpp b/test/offline/offline_test_main.cpp index 838b7753d4..1372675184 100644 --- a/test/offline/offline_test_main.cpp +++ b/test/offline/offline_test_main.cpp @@ -264,20 +264,20 @@ Decompiler setup_decompiler(const std::vector& files, // don't try to do this because we can't write the file dc.config->generate_symbol_definition_map = false; - std::vector dgo_paths; + std::vector dgo_paths; if (args.iso_data_path.empty()) { for (auto& x : offline_config.dgos) { - dgo_paths.push_back((file_util::get_jak_project_dir() / "iso_data" / "jak1" / x).string()); + dgo_paths.push_back(file_util::get_jak_project_dir() / "iso_data" / "jak1" / x); } } else { for (auto& x : offline_config.dgos) { - dgo_paths.push_back(file_util::combine_path(args.iso_data_path, x)); + dgo_paths.push_back(std::filesystem::path(args.iso_data_path) / x); } } - dc.db = std::make_unique(dgo_paths, dc.config->obj_file_name_map_file, - std::vector{}, - std::vector{}, *dc.config); + dc.db = std::make_unique( + dgo_paths, dc.config->obj_file_name_map_file, std::vector{}, + std::vector{}, *dc.config); std::unordered_set db_files; for (auto& files_by_name : dc.db->obj_files_by_name) {