mirror/jak-project
1
0
Fork 0
mirror of https://github.com/open-goal/jak-project synced 2026-07-09 14:55:51 -04:00
Code Issues Packages Projects Releases Wiki Activity

Remove assets folder, use more std::filesystem (#1575)

Browse Source 
* Remove assets folder, use more std::filesystem

* windows fix

* another one for windows

* another one

* better system for different folders

* rm debugging stuff

* let extractor override everything

* dont revert jak1 change
This commit is contained in:
water111
2022-06-29 23:32:46 -04:00
committed by GitHub
parent 36dc015d62
commit 1e33dcda4b
36 changed files with 486 additions and 344 deletions
Download Patch File Download Diff File Expand all files Collapse all files
assets/.gitignore
-2
View File
@@ -1,2 +0,0 @@
*
!.gitignore
common/audio/audio_formats.cpp
+1 -1
View File
@@ -9,7 +9,7 @@
*/
void write_wave_file_mono(const std::vector<s16>& samples,
s32 sample_rate,
const std::string& name) {
const std::filesystem::path& name) {
WaveFileHeader header;
memcpy(header.chunk_id, "RIFF", 4);
header.chunk_size = 36 + samples.size() * sizeof(s16);
common/audio/audio_formats.h
+2 -1
View File
@@ -1,5 +1,6 @@
#pragma once
#include <filesystem>
#include <string>
#include <vector>
@@ -30,7 +31,7 @@ struct WaveFileHeader {
void write_wave_file_mono(const std::vector<s16>& samples,
s32 sample_rate,
const std::string& name);
const std::filesystem::path& name);
std::vector<s16> decode_adpcm(BinaryReader& reader);
common/util/BinaryWriter.h
+5 -4
View File
@@ -7,6 +7,7 @@
#include <cstdint>
#include <cstring>
#include <filesystem>
#include <stdexcept>
#include <vector>
@@ -63,12 +64,12 @@ class BinaryWriter {
void* get_data() { return data.data(); }
void write_to_file(const std::string& filename) {
auto fp = fopen(filename.c_str(), "wb");
void write_to_file(const std::filesystem::path& filename) {
auto fp = fopen(filename.string().c_str(), "wb");
if (!fp)
throw std::runtime_error("failed to open " + filename);
throw std::runtime_error("failed to open " + filename.string());
if (fwrite(get_data(), get_size(), 1, fp) != 1)
throw std::runtime_error("failed to write " + filename);
throw std::runtime_error("failed to write " + filename.string());
fclose(fp);
}
common/util/DgoWriter.cpp
+1
View File
@@ -15,6 +15,7 @@ void build_dgo(const DgoDescription& description) {
writer.add_cstr_len(description.dgo_name.c_str(), 60);
for (auto& obj : description.entries) {
// todo: hardcoded out
auto obj_data =
file_util::read_binary_file(file_util::get_file_path({"out", "obj", obj.file_name}));
// size
common/util/FileUtil.cpp
+37 -22
View File
@@ -169,7 +169,7 @@ std::string get_file_path(const std::vector<std::string>& input) {
return current_path.string();
}
bool create_dir_if_needed(const std::string& path) {
bool create_dir_if_needed(const std::filesystem::path& path) {
if (!std::filesystem::is_directory(path)) {
std::filesystem::create_directories(path);
return true;
@@ -181,59 +181,70 @@ bool create_dir_if_needed_for_file(const std::string& path) {
return std::filesystem::create_directories(std::filesystem::path(path).parent_path());
}
void write_binary_file(const std::string& name, const void* data, size_t size) {
FILE* fp = fopen(name.c_str(), "wb");
void write_binary_file(const std::filesystem::path& name, const void* data, size_t size) {
FILE* fp = fopen(name.string().c_str(), "wb");
if (!fp) {
throw std::runtime_error("couldn't open file " + name);
throw std::runtime_error("couldn't open file " + name.string());
}
if (fwrite(data, size, 1, fp) != 1) {
fclose(fp);
throw std::runtime_error("couldn't write file " + name);
throw std::runtime_error("couldn't write file " + name.string());
}
fclose(fp);
}
void write_rgba_png(const std::string& name, void* data, int w, int h) {
void write_binary_file(const std::string& name, const void* data, size_t size) {
write_binary_file(std::filesystem::path(name), data, size);
}
void write_rgba_png(const std::filesystem::path& name, void* data, int w, int h) {
auto flags = 0;
auto ok = fpng::fpng_encode_image_to_file(name.c_str(), data, w, h, 4, flags);
auto ok = fpng::fpng_encode_image_to_file(name.string().c_str(), data, w, h, 4, flags);
if (!ok) {
throw std::runtime_error("couldn't save png file " + name);
throw std::runtime_error("couldn't save png file " + name.string());
}
}
void write_text_file(const std::string& file_name, const std::string& text) {
FILE* fp = fopen(file_name.c_str(), "w");
write_text_file(std::filesystem::path(file_name), text);
}
void write_text_file(const std::filesystem::path& file_name, const std::string& text) {
FILE* fp = fopen(file_name.string().c_str(), "w");
if (!fp) {
lg::error("Failed to fopen {}\n", file_name);
lg::error("Failed to fopen {}\n", file_name.string());
throw std::runtime_error("Failed to open file");
}
fprintf(fp, "%s\n", text.c_str());
fclose(fp);
}
std::vector<uint8_t> read_binary_file(const std::string& filename) {
// make sure file exists and isn't a directory
std::filesystem::path path(filename);
return read_binary_file(std::filesystem::path(filename));
}
auto status = std::filesystem::status(std::filesystem::path(filename));
std::vector<uint8_t> read_binary_file(const std::filesystem::path& path) {
// make sure file exists and isn't a directory
auto status = std::filesystem::status(path);
if (!std::filesystem::exists(status)) {
throw std::runtime_error(fmt::format("File {} cannot be opened: does not exist.", filename));
throw std::runtime_error(
fmt::format("File {} cannot be opened: does not exist.", path.string()));
}
if (status.type() != std::filesystem::file_type::regular &&
status.type() != std::filesystem::file_type::symlink) {
throw std::runtime_error(
fmt::format("File {} cannot be opened: not a regular file or symlink.", filename));
fmt::format("File {} cannot be opened: not a regular file or symlink.", path.string()));
}
auto fp = fopen(filename.c_str(), "rb");
auto fp = fopen(path.string().c_str(), "rb");
if (!fp)
throw std::runtime_error("File " + filename +
throw std::runtime_error("File " + path.string() +
" cannot be opened: " + std::string(strerror(errno)));
fseek(fp, 0, SEEK_END);
auto len = ftell(fp);
@@ -244,23 +255,27 @@ std::vector<uint8_t> read_binary_file(const std::string& filename) {
if (fread(data.data(), len, 1, fp) != 1) {
fclose(fp);
throw std::runtime_error("File " + filename + " cannot be read");
throw std::runtime_error("File " + path.string() + " cannot be read");
}
fclose(fp);
return data;
}
std::string read_text_file(const std::string& path) {
std::ifstream file(path);
std::string read_text_file(const std::filesystem::path& path) {
std::ifstream file(path.string());
if (!file.good()) {
throw std::runtime_error("couldn't open " + path);
throw std::runtime_error("couldn't open " + path.string());
}
std::stringstream ss;
ss << file.rdbuf();
return ss.str();
}
std::string read_text_file(const std::string& path) {
return read_text_file(std::filesystem::path(path));
}
bool is_printable_char(char c) {
return c >= ' ' && c <= '~';
}
common/util/FileUtil.h
+6 -2
View File
@@ -21,15 +21,19 @@ std::filesystem::path get_user_settings_dir();
std::filesystem::path get_user_memcard_dir();
std::filesystem::path get_jak_project_dir();
bool create_dir_if_needed(const std::string& path);
bool create_dir_if_needed(const std::filesystem::path& path);
bool create_dir_if_needed_for_file(const std::string& path);
bool setup_project_path(std::optional<std::filesystem::path> project_path_override);
std::string get_file_path(const std::vector<std::string>& path);
void write_binary_file(const std::string& name, const void* data, size_t size);
void write_rgba_png(const std::string& name, void* data, int w, int h);
void write_binary_file(const std::filesystem::path& name, const void* data, size_t size);
void write_rgba_png(const std::filesystem::path& name, void* data, int w, int h);
void write_text_file(const std::string& file_name, const std::string& text);
void write_text_file(const std::filesystem::path& file_name, const std::string& text);
std::vector<uint8_t> read_binary_file(const std::string& filename);
std::vector<uint8_t> read_binary_file(const std::filesystem::path& filename);
std::string read_text_file(const std::string& path);
std::string read_text_file(const std::filesystem::path& path);
bool is_printable_char(char c);
std::string combine_path(const std::string& parent, const std::string& child);
bool file_exists(const std::string& path);
decompiler/ObjectFile/ObjectFileDB.cpp
+26 -25
View File
@@ -113,10 +113,10 @@ const ObjectFileData& ObjectFileDB::lookup_record(const ObjectFileRecord& rec) c
/*!
* Build an object file DB for the given list of DGOs.
*/
ObjectFileDB::ObjectFileDB(const std::vector<std::string>& _dgos,
const std::string& obj_file_name_map_file,
const std::vector<std::string>& object_files,
const std::vector<std::string>& str_files,
ObjectFileDB::ObjectFileDB(const std::vector<std::filesystem::path>& _dgos,
const std::filesystem::path& obj_file_name_map_file,
const std::vector<std::filesystem::path>& object_files,
const std::vector<std::filesystem::path>& str_files,
const Config& config) {
Timer timer;
@@ -125,7 +125,8 @@ ObjectFileDB::ObjectFileDB(const std::vector<std::string>& _dgos,
if (!obj_file_name_map_file.empty()) {
lg::info("-Loading obj name map file...");
load_map_file(file_util::read_text_file(file_util::get_file_path({obj_file_name_map_file})));
load_map_file(
file_util::read_text_file(file_util::get_jak_project_dir() / obj_file_name_map_file));
} else {
lg::warn(
"Not using an obj name map file! The decompiler will automatically generate object file "
@@ -138,14 +139,14 @@ ObjectFileDB::ObjectFileDB(const std::vector<std::string>& _dgos,
try {
get_objs_from_dgo(dgo, config);
} catch (std::runtime_error& e) {
lg::warn("Error when reading DGOs: {} on {}", e.what(), dgo);
lg::warn("Error when reading DGOs: {} on {}", e.what(), dgo.string());
}
}
lg::info("-Loading {} plain object files...", object_files.size());
for (auto& obj : object_files) {
auto data = file_util::read_binary_file(obj);
auto name = obj_filename_to_name(obj);
auto name = obj_filename_to_name(obj.string());
if (auto it = config.object_patches.find(name); it != config.object_patches.end()) {
// print the file CRC
fmt::print("CRC for {} is: 0x{:X}\n", name, crc32(data.data(), data.size()));
@@ -206,7 +207,7 @@ ObjectFileDB::ObjectFileDB(const std::vector<std::string>& _dgos,
for (auto& obj : str_files) {
StrFileReader reader(obj);
// name from the file name
std::string base_name = obj_filename_to_name(obj);
std::string base_name = obj_filename_to_name(obj.string());
// name from inside the file (this does a lot of sanity checking)
auto obj_name = reader.get_full_name(base_name + ".STR");
for (int i = 0; i < reader.chunk_count(); i++) {
@@ -257,7 +258,7 @@ void ObjectFileDB::load_map_file(const std::string& map_data) {
/*!
* Load the objects stored in the given DGO into the ObjectFileDB
*/
void ObjectFileDB::get_objs_from_dgo(const std::string& filename, const Config& config) {
void ObjectFileDB::get_objs_from_dgo(const std::filesystem::path& filename, const Config& config) {
auto dgo_data = file_util::read_binary_file(filename);
stats.total_dgo_bytes += dgo_data.size();
@@ -268,7 +269,7 @@ void ObjectFileDB::get_objs_from_dgo(const std::string& filename, const Config&
BinaryReader reader(dgo_data);
auto header = reader.read<DgoHeader>();
auto dgo_base_name = file_util::base_name(filename);
auto dgo_base_name = filename.filename().string();
ASSERT(header.name == dgo_base_name);
assert_string_empty_after(header.name, 60);
@@ -515,7 +516,7 @@ void ObjectFileDB::process_labels() {
/*!
* Dump object files and their linking data to text files for debugging
*/
void ObjectFileDB::write_object_file_words(const std::string& output_dir,
void ObjectFileDB::write_object_file_words(const std::filesystem::path& output_dir,
bool dump_data,
bool dump_code) {
lg::info("- Writing object file dumps (code? {} data? {})...", dump_code, dump_data);
@@ -527,7 +528,7 @@ void ObjectFileDB::write_object_file_words(const std::string& output_dir,
if ((obj.linked_data.segments == 3 && dump_code) ||
(obj.linked_data.segments != 3 && dump_data)) {
auto file_text = obj.linked_data.print_words();
auto file_name = file_util::combine_path(output_dir, obj.to_unique_name() + ".txt");
auto file_name = output_dir / (obj.to_unique_name() + ".txt");
total_bytes += file_text.size();
file_util::write_text_file(file_name, file_text);
total_files++;
@@ -544,7 +545,7 @@ void ObjectFileDB::write_object_file_words(const std::string& output_dir,
/*!
* Dump disassembly for object files containing code. Data zones will also be dumped.
*/
void ObjectFileDB::write_disassembly(const std::string& output_dir,
void ObjectFileDB::write_disassembly(const std::filesystem::path& output_dir,
bool disassemble_data,
bool disassemble_code,
bool print_hex) {
@@ -558,7 +559,7 @@ void ObjectFileDB::write_disassembly(const std::string& output_dir,
if ((obj.obj_version == 3 && disassemble_code) || (obj.obj_version != 3 && disassemble_data)) {
auto file_text = obj.linked_data.print_disassembly(print_hex);
asm_functions += obj.linked_data.print_asm_function_disassembly(obj.to_unique_name());
auto file_name = file_util::combine_path(output_dir, obj.to_unique_name() + ".asm");
auto file_name = output_dir / (obj.to_unique_name() + ".asm");
total_bytes += file_text.size();
file_util::write_text_file(file_name, file_text);
@@ -568,8 +569,7 @@ void ObjectFileDB::write_disassembly(const std::string& output_dir,
total_bytes += asm_functions.size();
total_files++;
file_util::write_text_file(file_util::combine_path(output_dir, "asm_functions.func"),
asm_functions);
file_util::write_text_file(output_dir / "asm_functions.func", asm_functions);
lg::info("Wrote functions dumps:");
lg::info(" Total {} files", total_files);
@@ -623,7 +623,7 @@ void ObjectFileDB::find_code(const Config& config) {
* Finds and writes all scripts into a file named all_scripts.lisp.
* Doesn't change any state in ObjectFileDB.
*/
void ObjectFileDB::find_and_write_scripts(const std::string& output_dir) {
void ObjectFileDB::find_and_write_scripts(const std::filesystem::path& output_dir) {
lg::info("- Finding scripts in object files...");
Timer timer;
std::string all_scripts;
@@ -638,14 +638,15 @@ void ObjectFileDB::find_and_write_scripts(const std::string& output_dir) {
}
});
auto file_name = file_util::combine_path(output_dir, "all_scripts.lisp");
auto file_name = output_dir / "all_scripts.lisp";
file_util::write_text_file(file_name, all_scripts);
lg::info("Found scripts:");
lg::info(" Total {:.3f} ms\n", timer.getMs());
}
std::string ObjectFileDB::process_tpages(TextureDB& tex_db) {
std::string ObjectFileDB::process_tpages(TextureDB& tex_db,
const std::filesystem::path& output_path) {
lg::info("- Finding textures in tpages...");
std::string tpage_string = "tpage-";
int total = 0, success = 0;
@@ -656,7 +657,7 @@ std::string ObjectFileDB::process_tpages(TextureDB& tex_db) {
std::string result;
for_each_obj([&](ObjectFileData& data) {
if (data.name_in_dgo.substr(0, tpage_string.length()) == tpage_string) {
auto statistics = process_tpage(data, tex_db);
auto statistics = process_tpage(data, tex_db, output_path);
total += statistics.total_textures;
success += statistics.successful_textures;
total_px += statistics.num_px;
@@ -795,16 +796,16 @@ void ObjectFileDB::extract_art_info() {
/*!
* Write out the art group information.
*/
void ObjectFileDB::dump_art_info(const std::string& output_dir) {
void ObjectFileDB::dump_art_info(const std::filesystem::path& output_dir) {
lg::info("Writing art group info...");
Timer timer;
if (!dts.art_group_info.empty()) {
file_util::create_dir_if_needed(file_util::combine_path(output_dir, "import"));
file_util::create_dir_if_needed(output_dir / "import");
}
for (const auto& [ag_name, info] : dts.art_group_info) {
auto ag_fname = ag_name + ".gc";
auto filename = file_util::get_file_path({output_dir, "import", ag_fname});
auto filename = output_dir / "import" / ag_fname;
std::string result = ";;-*-Lisp-*-\n";
result += "(in-package goal)\n\n";
result += fmt::format(";; {} - art group OpenGOAL import file\n", ag_fname);
@@ -819,9 +820,9 @@ void ObjectFileDB::dump_art_info(const std::string& output_dir) {
lg::info("Written art group info: in {:.2f} ms\n", timer.getMs());
}
void ObjectFileDB::dump_raw_objects(const std::string& output_dir) {
void ObjectFileDB::dump_raw_objects(const std::filesystem::path& output_dir) {
for_each_obj([&](ObjectFileData& data) {
auto dest = output_dir + "/" + data.to_unique_name();
auto dest = output_dir / data.to_unique_name();
if (data.obj_version != 3) {
dest += ".go";
}
decompiler/ObjectFile/ObjectFileDB.h
+16 -14
View File
@@ -144,28 +144,30 @@ struct LetRewriteStats {
class ObjectFileDB {
public:
ObjectFileDB(const std::vector<std::string>& _dgos,
const std::string& obj_file_name_map_file,
const std::vector<std::string>& object_files,
const std::vector<std::string>& str_files,
ObjectFileDB(const std::vector<std::filesystem::path>& _dgos,
const std::filesystem::path& obj_file_name_map_file,
const std::vector<std::filesystem::path>& object_files,
const std::vector<std::filesystem::path>& str_files,
const Config& config);
std::string generate_dgo_listing();
std::string generate_obj_listing(const std::unordered_set<std::string>& merged_objs);
void process_link_data(const Config& config);
void process_labels();
void find_code(const Config& config);
void find_and_write_scripts(const std::string& output_dir);
void find_and_write_scripts(const std::filesystem::path& output_dir);
void extract_art_info();
void dump_art_info(const std::string& output_dir);
void dump_raw_objects(const std::string& output_dir);
void write_object_file_words(const std::string& output_dir, bool dump_data, bool dump_code);
void write_disassembly(const std::string& output_dir,
void dump_art_info(const std::filesystem::path& output_dir);
void dump_raw_objects(const std::filesystem::path& output_dir);
void write_object_file_words(const std::filesystem::path& output_dir,
bool dump_data,
bool dump_code);
void write_disassembly(const std::filesystem::path& output_dir,
bool disassemble_data,
bool disassemble_code,
bool print_hex);
void analyze_functions_ir2(
const std::string& output_dir,
const std::filesystem::path& output_dir,
const Config& config,
const std::unordered_set<std::string>& skip_functions,
const std::unordered_map<std::string, std::unordered_set<std::string>>& skip_states = {});
@@ -183,7 +185,7 @@ class ObjectFileDB {
void ir2_rewrite_inline_asm_instructions(int seg, ObjectFileData& data);
void ir2_insert_anonymous_functions(int seg, ObjectFileData& data);
void ir2_symbol_definition_map(ObjectFileData& data);
void ir2_write_results(const std::string& output_dir,
void ir2_write_results(const std::filesystem::path& output_dir,
const Config& config,
const std::vector<std::string>& imports,
ObjectFileData& data);
@@ -191,7 +193,7 @@ class ObjectFileDB {
void ir2_do_segment_analysis_phase2(int seg, const Config& config, ObjectFileData& data);
void ir2_setup_labels(const Config& config, ObjectFileData& data);
void ir2_run_mips2c(const Config& config, ObjectFileData& data);
void ir2_analyze_all_types(const std::string& output_file,
void ir2_analyze_all_types(const std::filesystem::path& output_file,
const std::optional<std::string>& previous_game_types,
const std::unordered_set<std::string>& bad_types);
std::string ir2_to_file(ObjectFileData& data, const Config& config);
@@ -200,7 +202,7 @@ class ObjectFileDB {
const std::vector<std::string>& imports,
const std::unordered_set<std::string>& skip_functions);
std::string process_tpages(TextureDB& tex_db);
std::string process_tpages(TextureDB& tex_db, const std::filesystem::path& output_path);
std::string process_game_count_file();
std::string process_game_text_files(const Config& cfg);
@@ -214,7 +216,7 @@ class ObjectFileDB {
public:
void load_map_file(const std::string& map_data);
void get_objs_from_dgo(const std::string& filename, const Config& config);
void get_objs_from_dgo(const std::filesystem::path& filename, const Config& config);
void add_obj_from_dgo(const std::string& obj_name,
const std::string& name_in_dgo,
const uint8_t* obj_data,
decompiler/ObjectFile/ObjectFileDB_IR2.cpp
+7 -8
View File
@@ -38,7 +38,7 @@ namespace decompiler {
* functions, but nothing else.
*/
void ObjectFileDB::analyze_functions_ir2(
const std::string& output_dir,
const std::filesystem::path& output_dir,
const Config& config,
const std::unordered_set<std::string>& skip_functions,
const std::unordered_map<std::string, std::unordered_set<std::string>>& skip_states) {
@@ -100,7 +100,7 @@ void ObjectFileDB::analyze_functions_ir2(
imports = imports_it->second;
}
if (!output_dir.empty()) {
if (!output_dir.string().empty()) {
ir2_write_results(output_dir, config, imports, data);
} else {
if (!skip_functions.empty()) {
@@ -135,8 +135,7 @@ void ObjectFileDB::analyze_functions_ir2(
lg::info("Generating symbol definition map...");
map_builder.build_map();
std::string result = map_builder.convert_to_json();
auto file_name = file_util::combine_path(output_dir, "symbol_map.json");
file_util::write_text_file(file_name, result);
file_util::write_text_file(output_dir / "symbol_map.json", result);
}
}
@@ -301,7 +300,7 @@ void ObjectFileDB::ir2_top_level_pass(const Config& config) {
lg::info("{:4d} logins {:.2f}%\n", total_top_levels, 100.f * total_top_levels / total_functions);
}
void ObjectFileDB::ir2_analyze_all_types(const std::string& output_file,
void ObjectFileDB::ir2_analyze_all_types(const std::filesystem::path& output_file,
const std::optional<std::string>& previous_game_types,
const std::unordered_set<std::string>& bad_types) {
struct PerObject {
@@ -689,7 +688,7 @@ void ObjectFileDB::ir2_insert_anonymous_functions(int seg, ObjectFileData& data)
});
}
void ObjectFileDB::ir2_write_results(const std::string& output_dir,
void ObjectFileDB::ir2_write_results(const std::filesystem::path& output_dir,
const Config& config,
const std::vector<std::string>& imports,
ObjectFileData& obj) {
@@ -697,11 +696,11 @@ void ObjectFileDB::ir2_write_results(const std::string& output_dir,
// todo
auto file_text = ir2_to_file(obj, config);
auto file_name = file_util::combine_path(output_dir, obj.to_unique_name() + "_ir2.asm");
auto file_name = output_dir / (obj.to_unique_name() + "_ir2.asm");
file_util::write_text_file(file_name, file_text);
auto final = ir2_final_out(obj, imports, {});
auto final_name = file_util::combine_path(output_dir, obj.to_unique_name() + "_disasm.gc");
auto final_name = output_dir / (obj.to_unique_name() + "_disasm.gc");
file_util::write_text_file(final_name, final);
}
}
decompiler/config/jak1_ntsc_black_label.jsonc
+4 -4
View File
@@ -105,10 +105,10 @@
////////////////////////////
// PATCHING OPTIONS
////////////////////////////
// these are options related to xdelta3 patches on specific objects
// this allows us to get a more consistent input
// this allows us to get a more consistent input
// set to true to write new patch files
"write_patches": false,
// set to true to apply patch files
@@ -151,5 +151,5 @@
"out": "game/assets/jak1/patches/6common.xd3"
}
}
}
decompiler/data/StrFileReader.cpp
+1 -1
View File
@@ -14,7 +14,7 @@
#include "game/common/str_rpc_types.h"
namespace decompiler {
StrFileReader::StrFileReader(const std::string& file_path) {
StrFileReader::StrFileReader(const std::filesystem::path& file_path) {
auto data = file_util::read_binary_file(file_path);
ASSERT(data.size() >= SECTOR_SIZE); // must have at least the header sector
ASSERT(data.size() % SECTOR_SIZE == 0); // should be multiple of the sector size.
decompiler/data/StrFileReader.h
+2 -1
View File
@@ -5,6 +5,7 @@
* Utility class to read a .STR file and extract the full file name.
*/
#include <filesystem>
#include <string>
#include <vector>
@@ -13,7 +14,7 @@
namespace decompiler {
class StrFileReader {
public:
explicit StrFileReader(const std::string& file_path);
explicit StrFileReader(const std::filesystem::path& file_path);
int chunk_count() const;
const std::vector<u8>& get_chunk(int idx) const;
std::string get_full_name(const std::string& short_name) const;
decompiler/data/TextureDB.cpp
+1 -1
View File
@@ -45,7 +45,7 @@ void TextureDB::add_texture(u32 tpage,
}
}
void TextureDB::replace_textures(const std::string& path) {
void TextureDB::replace_textures(const std::filesystem::path& path) {
std::filesystem::path base_path(path);
for (auto& tex : textures) {
std::filesystem::path full_path =
decompiler/data/TextureDB.h
+2 -1
View File
@@ -1,5 +1,6 @@
#pragma once
#include <filesystem>
#include <set>
#include <string>
#include <unordered_map>
@@ -29,6 +30,6 @@ struct TextureDB {
const std::string& tpage_name,
const std::vector<std::string>& level_names);
void replace_textures(const std::string& path);
void replace_textures(const std::filesystem::path& path);
};
} // namespace decompiler
decompiler/data/streamed_audio.cpp
+12 -14
View File
@@ -89,7 +89,7 @@ struct VagFileHeader {
/*!
* Read the DIR file into an AudioDir
*/
AudioDir read_audio_dir(const std::string& path) {
AudioDir read_audio_dir(const std::filesystem::path& path) {
// matches the format in file.
struct DirEntry {
char name[8];
@@ -145,7 +145,8 @@ struct AudioFileInfo {
double length_seconds;
};
AudioFileInfo process_audio_file(const std::vector<u8>& data,
AudioFileInfo process_audio_file(const std::filesystem::path& output_folder,
const std::vector<u8>& data,
const std::string& name,
const std::string& suffix) {
BinaryReader reader(data);
@@ -168,10 +169,9 @@ AudioFileInfo process_audio_file(const std::vector<u8>& data,
ASSERT(reader.read<u8>() == 0);
}
file_util::create_dir_if_needed(file_util::get_file_path({"assets", "streaming_audio", suffix}));
file_util::create_dir_if_needed(output_folder / suffix);
auto file_name = fmt::format("{}.wav", remove_trailing_spaces(name));
write_wave_file_mono(decoded_samples, header.sample_rate,
file_util::get_file_path({"assets", "streaming_audio", suffix, file_name}));
write_wave_file_mono(decoded_samples, header.sample_rate, output_folder / suffix / file_name);
std::string vag_filename;
for (int i = 0; i < 16; i++) {
@@ -182,11 +182,10 @@ AudioFileInfo process_audio_file(const std::vector<u8>& data,
return {vag_filename, (double)decoded_samples.size() / header.sample_rate};
}
void process_streamed_audio(const std::string& dir, const std::vector<std::string>& audio_files) {
auto dir_file_name = file_util::combine_path(dir, "VAGDIR.AYB");
lg::info("Streaming audio: {}\n", dir_file_name);
file_util::create_dir_if_needed(file_util::get_file_path({"assets", "streaming_audio"}));
auto dir_data = read_audio_dir(file_util::get_file_path({"iso_data", dir_file_name}));
void process_streamed_audio(const std::filesystem::path& output_path,
const std::filesystem::path& input_dir,
const std::vector<std::string>& audio_files) {
auto dir_data = read_audio_dir(input_dir / "VAG" / "VAGDIR.AYB");
double audio_len = 0.f;
std::vector<std::string> langs;
@@ -201,14 +200,14 @@ void process_streamed_audio(const std::string& dir, const std::vector<std::strin
for (size_t lang_id = 0; lang_id < audio_files.size(); lang_id++) {
auto& file = audio_files[lang_id];
auto wad_data = file_util::read_binary_file(file_util::get_file_path({"iso_data", dir, file}));
auto wad_data = file_util::read_binary_file(input_dir / "VAG" / file);
auto suffix = std::filesystem::path(file).extension().u8string().substr(1);
langs.push_back(suffix);
dir_data.set_file_size(wad_data.size());
for (int i = 0; i < dir_data.entry_count(); i++) {
auto audio_data = read_entry(dir_data, wad_data, i);
lg::info("File {}, total {:.2f} minutes", dir_data.entries.at(i).name, audio_len / 60.0);
auto info = process_audio_file(audio_data, dir_data.entries.at(i).name, suffix);
auto info = process_audio_file(output_path, audio_data, dir_data.entries.at(i).name, suffix);
audio_len += info.length_seconds;
filename_data[i][lang_id + 1] = info.filename;
}
@@ -218,8 +217,7 @@ void process_streamed_audio(const std::string& dir, const std::vector<std::strin
file_list["names"] = filename_data;
file_list["languages"] = langs;
file_util::write_text_file(
file_util::get_file_path({"assets", "streaming_audio", "file_list.txt"}), file_list.dump(2));
file_util::write_text_file(output_path / "file_list.txt", file_list.dump(2));
}
} // namespace decompiler
decompiler/data/streamed_audio.h
+4 -1
View File
@@ -1,8 +1,11 @@
#pragma once
#include <filesystem>
#include <string>
#include <vector>
namespace decompiler {
void process_streamed_audio(const std::string& dir, const std::vector<std::string>& audio_files);
void process_streamed_audio(const std::filesystem::path& output_path,
const std::filesystem::path& input_dir,
const std::vector<std::string>& audio_files);
}
decompiler/data/tpage.cpp
+18 -32
View File
@@ -75,7 +75,9 @@ namespace {
*/
// texture format names.
std::unordered_map<u8, std::string> psms = {{0x02, "PSMCT16"}, {0x13, "PSMT8"}, {0x14, "PSMT4"}};
const std::unordered_map<u8, std::string> psms = {{0x02, "PSMCT16"},
{0x13, "PSMT8"},
{0x14, "PSMT4"}};
/*!
* GOAL texture type. Stores info about a single texture in a texture page.
@@ -417,7 +419,9 @@ TexturePage read_texture_page(ObjectFileData& data,
* Process a texture page.
* TODO - document
*/
TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) {
TPageResultStats process_tpage(ObjectFileData& data,
TextureDB& texture_db,
const std::filesystem::path& output_path) {
TPageResultStats stats;
auto& words = data.linked_data.words_by_seg.at(0);
const auto& level_names = data.dgo_names;
@@ -437,6 +441,8 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) {
// Read the texture_page struct
TexturePage texture_page = read_texture_page(data, words, 0, end_of_texture_page);
auto texture_dump_dir = output_path / texture_page.name;
file_util::create_dir_if_needed(texture_dump_dir);
// Get raw data for textures.
std::vector<u32> tex_data;
@@ -518,12 +524,8 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) {
}
// write texture to a PNG.
file_util::create_dir_if_needed(
file_util::get_file_path({"assets", "textures", texture_page.name}));
file_util::write_rgba_png(
fmt::format(file_util::get_file_path({"assets", "textures", texture_page.name, "{}.png"}),
tex.name),
out.data(), tex.w, tex.h);
file_util::write_rgba_png(texture_dump_dir / fmt::format("{}.png", tex.name), out.data(),
tex.w, tex.h);
texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
texture_page.name, level_names);
stats.successful_textures++;
@@ -565,12 +567,8 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) {
}
// write texture to a PNG.
file_util::create_dir_if_needed(
file_util::get_file_path({"assets", "textures", texture_page.name}));
file_util::write_rgba_png(
fmt::format(file_util::get_file_path({"assets", "textures", texture_page.name, "{}.png"}),
tex.name),
out.data(), tex.w, tex.h);
file_util::write_rgba_png(texture_dump_dir / fmt::format("{}.png", tex.name), out.data(),
tex.w, tex.h);
texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
texture_page.name, level_names);
stats.successful_textures++;
@@ -594,12 +592,8 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) {
}
// write texture to a PNG.
file_util::create_dir_if_needed(
file_util::get_file_path({"assets", "textures", texture_page.name}));
file_util::write_rgba_png(
fmt::format(file_util::get_file_path({"assets", "textures", texture_page.name, "{}.png"}),
tex.name),
out.data(), tex.w, tex.h);
file_util::write_rgba_png(texture_dump_dir / fmt::format("{}.png", tex.name), out.data(),
tex.w, tex.h);
texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
texture_page.name, level_names);
stats.successful_textures++;
@@ -639,12 +633,8 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) {
}
// write texture to a PNG.
file_util::create_dir_if_needed(
file_util::get_file_path({"assets", "textures", texture_page.name}));
file_util::write_rgba_png(
fmt::format(file_util::get_file_path({"assets", "textures", texture_page.name, "{}.png"}),
tex.name),
out.data(), tex.w, tex.h);
file_util::write_rgba_png(texture_dump_dir / fmt::format("{}.png", tex.name), out.data(),
tex.w, tex.h);
texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
texture_page.name, level_names);
stats.successful_textures++;
@@ -684,12 +674,8 @@ TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) {
}
// write texture to a PNG.
file_util::create_dir_if_needed(
file_util::get_file_path({"assets", "textures", texture_page.name}));
file_util::write_rgba_png(
fmt::format(file_util::get_file_path({"assets", "textures", texture_page.name, "{}.png"}),
tex.name),
out.data(), tex.w, tex.h);
file_util::write_rgba_png(texture_dump_dir / fmt::format("{}.png", tex.name), out.data(),
tex.w, tex.h);
texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
texture_page.name, level_names);
stats.successful_textures++;
decompiler/data/tpage.h
+5 -1
View File
@@ -1,5 +1,7 @@
#pragma once
#include <filesystem>
#include "decompiler/data/TextureDB.h"
namespace decompiler {
@@ -11,5 +13,7 @@ struct TPageResultStats {
int num_px = 0;
};
TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db);
TPageResultStats process_tpage(ObjectFileData& data,
TextureDB& texture_db,
const std::filesystem::path& output_path);
} // namespace decompiler
decompiler/extractor/main.cpp
+19 -14
View File
@@ -366,18 +366,18 @@ void decompile(std::filesystem::path jak1_input_files) {
.string(),
{});
std::vector<std::string> dgos, objs;
std::vector<std::filesystem::path> dgos, objs;
// grab all DGOS we need (level + common)
for (const auto& dgo_name : config.dgo_names) {
std::string common_name = "GAME.CGO";
if (dgo_name.length() > 3 && dgo_name.substr(dgo_name.length() - 3) == "DGO") {
// ends in DGO, it's a level
dgos.push_back((jak1_input_files / dgo_name).string());
dgos.push_back(jak1_input_files / dgo_name);
} else if (dgo_name.length() >= common_name.length() &&
dgo_name.substr(dgo_name.length() - common_name.length()) == common_name) {
// it's COMMON.CGO, we need that too.
dgos.push_back((jak1_input_files / dgo_name).string());
dgos.push_back(jak1_input_files / dgo_name);
}
}
@@ -385,16 +385,16 @@ void decompile(std::filesystem::path jak1_input_files) {
for (const auto& obj_name : config.object_file_names) {
if (obj_name.length() > 3 && obj_name.substr(obj_name.length() - 3) == "TXT") {
// ends in TXT
objs.push_back((jak1_input_files / obj_name).string());
objs.push_back(jak1_input_files / obj_name);
}
}
// set up objects
ObjectFileDB db(dgos, config.obj_file_name_map_file, objs, {}, config);
ObjectFileDB db(dgos, std::filesystem::path(config.obj_file_name_map_file), objs, {}, config);
// save object files
auto out_folder = (file_util::get_jak_project_dir() / "decompiler_out" / "jak1").string();
auto raw_obj_folder = file_util::combine_path(out_folder, "raw_obj");
auto out_folder = file_util::get_jak_project_dir() / "decompiler_out" / "jak1";
auto raw_obj_folder = out_folder / "raw_obj";
file_util::create_dir_if_needed(raw_obj_folder);
db.dump_raw_objects(raw_obj_folder);
@@ -404,22 +404,24 @@ void decompile(std::filesystem::path jak1_input_files) {
db.process_labels();
// ensure asset dir exists
file_util::create_dir_if_needed(file_util::get_file_path({"assets"}));
file_util::create_dir_if_needed(out_folder / "assets");
// text files
{
auto result = db.process_game_text_files(config);
if (!result.empty()) {
file_util::write_text_file(file_util::get_file_path({"assets", "game_text.txt"}), result);
file_util::write_text_file(out_folder / "assets" / "game_text.txt", result);
}
}
// textures
decompiler::TextureDB tex_db;
file_util::write_text_file(file_util::get_file_path({"assets", "tpage-dir.txt"}),
db.process_tpages(tex_db));
auto textures_out = out_folder / "textures";
file_util::create_dir_if_needed(textures_out);
file_util::write_text_file(textures_out / "tpage-dir.txt",
db.process_tpages(tex_db, textures_out));
// texture replacements
auto replacements_path = file_util::get_file_path({"texture_replacements"});
auto replacements_path = file_util::get_jak_project_dir() / "texture_replacements";
if (std::filesystem::exists(replacements_path)) {
tex_db.replace_textures(replacements_path);
}
@@ -428,14 +430,17 @@ void decompile(std::filesystem::path jak1_input_files) {
{
auto result = db.process_game_count_file();
if (!result.empty()) {
file_util::write_text_file(file_util::get_file_path({"assets", "game_count.txt"}), result);
file_util::write_text_file(out_folder / "assets" / "game_count.txt", result);
}
}
// levels
{
// TODO separate out dirs
auto level_out_path = file_util::get_jak_project_dir() / "out" / "fr3";
file_util::create_dir_if_needed(level_out_path);
extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config.hacks,
config.rip_levels, config.extract_collision);
config.rip_levels, config.extract_collision, level_out_path);
}
}
decompiler/level_extractor/extract_level.cpp
+15 -11
View File
@@ -202,7 +202,8 @@ std::vector<level_tools::TextureRemap> extract_bsp_from_level(const ObjectFileDB
void extract_common(const ObjectFileDB& db,
const TextureDB& tex_db,
const std::string& dgo_name,
bool dump_levels) {
bool dump_levels,
const std::filesystem::path& output_folder) {
if (db.obj_files_by_dgo.count(dgo_name) == 0) {
lg::warn("Skipping common extract for {} because the DGO was not part of the input", dgo_name);
return;
@@ -227,9 +228,9 @@ void extract_common(const ObjectFileDB& db,
print_memory_usage(tfrag_level, ser.get_save_result().second);
fmt::print("compressed: {} -> {} ({:.2f}%)\n", ser.get_save_result().second, compressed.size(),
100.f * compressed.size() / ser.get_save_result().second);
file_util::write_binary_file(file_util::get_file_path({fmt::format(
"assets/{}.fr3", dgo_name.substr(0, dgo_name.length() - 4))}),
compressed.data(), compressed.size());
file_util::write_binary_file(
output_folder / fmt::format("{}.fr3", dgo_name.substr(0, dgo_name.length() - 4)),
compressed.data(), compressed.size());
}
void extract_from_level(const ObjectFileDB& db,
@@ -237,7 +238,8 @@ void extract_from_level(const ObjectFileDB& db,
const std::string& dgo_name,
const DecompileHacks& hacks,
bool dump_level,
bool extract_collision) {
bool extract_collision,
const std::filesystem::path& output_folder) {
if (db.obj_files_by_dgo.count(dgo_name) == 0) {
lg::warn("Skipping extract for {} because the DGO was not part of the input", dgo_name);
return;
@@ -258,9 +260,9 @@ void extract_from_level(const ObjectFileDB& db,
print_memory_usage(level_data, ser.get_save_result().second);
fmt::print("compressed: {} -> {} ({:.2f}%)\n", ser.get_save_result().second, compressed.size(),
100.f * compressed.size() / ser.get_save_result().second);
file_util::write_binary_file(file_util::get_file_path({fmt::format(
"assets/{}.fr3", dgo_name.substr(0, dgo_name.length() - 4))}),
compressed.data(), compressed.size());
file_util::write_binary_file(
output_folder / fmt::format("{}.fr3", dgo_name.substr(0, dgo_name.length() - 4)),
compressed.data(), compressed.size());
}
void extract_all_levels(const ObjectFileDB& db,
@@ -269,12 +271,14 @@ void extract_all_levels(const ObjectFileDB& db,
const std::string& common_name,
const DecompileHacks& hacks,
bool debug_dump_level,
bool extract_collision) {
extract_common(db, tex_db, common_name, debug_dump_level);
bool extract_collision,
const std::filesystem::path& output_path) {
extract_common(db, tex_db, common_name, debug_dump_level, output_path);
SimpleThreadGroup threads;
threads.run(
[&](int idx) {
extract_from_level(db, tex_db, dgo_names[idx], hacks, debug_dump_level, extract_collision);
extract_from_level(db, tex_db, dgo_names[idx], hacks, debug_dump_level, extract_collision,
output_path);
},
dgo_names.size());
threads.join();
decompiler/level_extractor/extract_level.h
+3 -1
View File
@@ -1,5 +1,6 @@
#pragma once
#include <filesystem>
#include <vector>
#include "common/math/Vector.h"
@@ -15,5 +16,6 @@ void extract_all_levels(const ObjectFileDB& db,
const std::string& common_name,
const DecompileHacks& hacks,
bool debug_dump_level,
bool extract_collision);
bool extract_collision,
const std::filesystem::path& path);
} // namespace decompiler
decompiler/main.cpp
+35 -30
View File
@@ -86,62 +86,62 @@ int main(int argc, char** argv) {
return 1;
}
std::string in_folder = file_util::combine_path(argv[2], config.game_name);
std::string out_folder = file_util::combine_path(argv[3], config.game_name);
// std::string in_folder = file_util::combine_path(argv[2], config.game_name);
std::filesystem::path in_folder = std::filesystem::path(argv[2]) / config.game_name;
std::filesystem::path out_folder = std::filesystem::path(argv[3]) / config.game_name;
// Verify the in_folder is correct
// TODO - refactor to use ghc::filesystem, cleanup file_util
if (!file_util::file_exists(in_folder)) {
fmt::print("Aborting - 'in_folder' does not exist '{}'\n", in_folder);
if (!exists(in_folder)) {
fmt::print("Aborting - 'in_folder' does not exist '{}'\n", in_folder.string());
return 1;
}
// Warning message if expected ELF isn't found, user could be using bad assets / didn't extract
// the ISO properly
if (!config.expected_elf_name.empty() &&
!file_util::file_exists(file_util::combine_path(in_folder, config.expected_elf_name))) {
if (!config.expected_elf_name.empty() && !exists(in_folder / config.expected_elf_name)) {
fmt::print(
"WARNING - '{}' does not contain the expected ELF file '{}'. Was the ISO extracted "
"properly or is there a version mismatch?\n",
in_folder, config.expected_elf_name);
in_folder.string(), config.expected_elf_name);
}
std::vector<std::string> dgos, objs, strs;
std::vector<std::filesystem::path> dgos, objs, strs;
for (const auto& dgo_name : config.dgo_names) {
dgos.push_back(file_util::combine_path(in_folder, dgo_name));
dgos.push_back(in_folder / dgo_name);
}
for (const auto& obj_name : config.object_file_names) {
objs.push_back(file_util::combine_path(in_folder, obj_name));
objs.push_back(in_folder / obj_name);
}
for (const auto& str_name : config.str_file_names) {
strs.push_back(file_util::combine_path(in_folder, str_name));
strs.push_back(in_folder / str_name);
}
file_util::create_dir_if_needed(out_folder);
file_util::create_dir_if_needed(out_folder / "assets");
if (config.rip_levels) {
file_util::create_dir_if_needed(file_util::get_file_path({"debug_out"}));
file_util::create_dir_if_needed(file_util::get_jak_project_dir() / "debug_out");
}
fmt::print("[Mem] After config read: {} MB\n", get_peak_rss() / (1024 * 1024));
// build file database
lg::info("Setting up object file DB...");
ObjectFileDB db(dgos, config.obj_file_name_map_file, objs, strs, config);
ObjectFileDB db(dgos, std::filesystem::path(config.obj_file_name_map_file), objs, strs, config);
fmt::print("[Mem] After DB setup: {} MB\n", get_peak_rss() / (1024 * 1024));
// write out DGO file info
file_util::write_text_file(file_util::combine_path(out_folder, "dgo.txt"),
db.generate_dgo_listing());
file_util::write_text_file(out_folder / "dgo.txt", db.generate_dgo_listing());
// write out object file map (used for future decompilations, if desired)
file_util::write_text_file(file_util::combine_path(out_folder, "obj.txt"),
file_util::write_text_file(out_folder / "obj.txt",
db.generate_obj_listing(config.merged_objects));
// dump raw objs
if (config.dump_objs) {
auto path = file_util::combine_path(out_folder, "raw_obj");
auto path = out_folder / "raw_obj";
file_util::create_dir_if_needed(path);
db.dump_raw_objects(path);
}
@@ -176,16 +176,14 @@ int main(int argc, char** argv) {
if (config.generate_all_types) {
ASSERT_MSG(config.decompile_code, "Must decompile code to generate all-types");
db.ir2_analyze_all_types(file_util::combine_path(out_folder, "new-all-types.gc"),
config.old_all_types_file,
db.ir2_analyze_all_types(out_folder / "new-all-types.gc", config.old_all_types_file,
config.hacks.types_with_bad_inspect_methods);
}
fmt::print("[Mem] After decomp: {} MB\n", get_peak_rss() / (1024 * 1024));
// write out all symbols
file_util::write_text_file(file_util::combine_path(out_folder, "all-syms.gc"),
db.dts.dump_symbol_types());
file_util::write_text_file(out_folder / "all-syms.gc", db.dts.dump_symbol_types());
// write art groups
if (config.process_art_groups) {
@@ -204,7 +202,7 @@ int main(int argc, char** argv) {
if (config.process_game_text) {
auto result = db.process_game_text_files(config);
if (!result.empty()) {
file_util::write_text_file(file_util::get_file_path({"assets", "game_text.txt"}), result);
file_util::write_text_file(out_folder / "assets" / "game_text.txt", result);
}
}
@@ -212,15 +210,16 @@ int main(int argc, char** argv) {
decompiler::TextureDB tex_db;
if (config.process_tpages || config.levels_extract) {
auto result = db.process_tpages(tex_db);
auto textures_out = out_folder / "textures";
file_util::create_dir_if_needed(textures_out);
auto result = db.process_tpages(tex_db, textures_out);
if (!result.empty() && config.process_tpages) {
file_util::write_text_file(file_util::get_file_path({"assets", "tpage-dir.txt"}), result);
file_util::write_text_file(textures_out / "tpage-dir.txt", result);
}
}
fmt::print("[Mem] After textures: {} MB\n", get_peak_rss() / (1024 * 1024));
// todo config
auto replacements_path = file_util::get_file_path({"texture_replacements"});
auto replacements_path = file_util::get_jak_project_dir() / "texture_replacements";
if (std::filesystem::exists(replacements_path)) {
tex_db.replace_textures(replacements_path);
}
@@ -228,19 +227,25 @@ int main(int argc, char** argv) {
if (config.process_game_count) {
auto result = db.process_game_count_file();
if (!result.empty()) {
file_util::write_text_file(file_util::get_file_path({"assets", "game_count.txt"}), result);
file_util::write_text_file(out_folder / "assets" / "game_count.txt", result);
}
}
if (config.levels_extract) {
// TODO separate out dirs
auto level_out_path = file_util::get_jak_project_dir() / "out" / "fr3";
file_util::create_dir_if_needed(level_out_path);
extract_all_levels(db, tex_db, config.levels_to_extract, "GAME.CGO", config.hacks,
config.rip_levels, config.extract_collision);
config.rip_levels, config.extract_collision, level_out_path);
}
fmt::print("[Mem] After extraction: {} MB\n", get_peak_rss() / (1024 * 1024));
if (!config.audio_dir_file_name.empty()) {
process_streamed_audio(config.audio_dir_file_name, config.streamed_audio_file_names);
auto streaming_audio_in = in_folder / "VAG";
auto streaming_audio_out = out_folder / "assets" / "streaming_audio";
file_util::create_dir_if_needed(streaming_audio_out);
process_streamed_audio(streaming_audio_out, in_folder, config.streamed_audio_file_names);
}
lg::info("Decompiler has finished successfully in {:.2f} seconds.", decomp_timer.getSeconds());
game/assets/game_text.gp
+1 -1
View File
@@ -5,7 +5,7 @@
(text
;; NOTE : we compile using the fixed v2 encoding because it's what we use.
(file "assets/game_text.txt") ;; this is the decompiler-generated file!
(file "$DECOMP/assets/game_text.txt") ;; this is the decompiler-generated file!
;; "patch" files so we can fix some errors and perhaps maintain consistency
(file "game/assets/jak1/text/text_patch_ja.gs")
;; add custom files down here
game/graphics/opengl_renderer/loader/Loader.cpp
+4 -5
View File
@@ -16,7 +16,7 @@ std::string uppercase_string(const std::string& s) {
}
} // namespace
Loader::Loader() {
Loader::Loader(const std::filesystem::path& base_path) : m_base_path(base_path) {
m_loader_thread = std::thread(&Loader::loader_thread, this);
m_loader_stages = make_loader_stages();
}
@@ -116,8 +116,8 @@ void Loader::loader_thread() {
// load the fr3 file
Timer disk_timer;
auto data = file_util::read_binary_file(
file_util::get_file_path({fmt::format("assets/{}.fr3", uppercase_string(lev))}));
auto data =
file_util::read_binary_file(m_base_path / fmt::format("{}.fr3", uppercase_string(lev)));
double disk_load_time = disk_timer.getSeconds();
// the FR3 files are compressed
@@ -167,8 +167,7 @@ void Loader::loader_thread() {
* This should be called during initialization, before any threaded loading goes on.
*/
void Loader::load_common(TexturePool& tex_pool, const std::string& name) {
auto data =
file_util::read_binary_file(file_util::get_file_path({fmt::format("assets/{}.fr3", name)}));
auto data = file_util::read_binary_file(m_base_path / fmt::format("{}.fr3", name));
auto decomp_data = compression::decompress_zstd(data.data(), data.size());
Serializer ser(decomp_data.data(), decomp_data.size());
game/graphics/opengl_renderer/loader/Loader.h
+4 -1
View File
@@ -1,6 +1,7 @@
#pragma once
#include <condition_variable>
#include <filesystem>
#include <mutex>
#include <thread>
@@ -15,7 +16,7 @@ class Loader {
public:
static constexpr float TIE_LOAD_BUDGET = 1.5f;
static constexpr float SHARED_TEXTURE_LOAD_BUDGET = 3.f;
Loader();
Loader(const std::filesystem::path& base_path);
~Loader();
void update(TexturePool& tex_pool);
void update_blocking(TexturePool& tex_pool);
@@ -50,4 +51,6 @@ class Loader {
std::vector<std::string> m_desired_levels;
std::vector<std::unique_ptr<LoaderStage>> m_loader_stages;
std::filesystem::path m_base_path;
};
game/graphics/pipelines/opengl.cpp
+2 -1
View File
@@ -68,7 +68,8 @@ struct GraphicsData {
GraphicsData()
: dma_copier(EE_MAIN_MEM_SIZE),
texture_pool(std::make_shared<TexturePool>()),
loader(std::make_shared<Loader>()),
// TODO out dir
loader(std::make_shared<Loader>(file_util::get_jak_project_dir() / "out" / "fr3")),
ogl_renderer(texture_pool, loader) {}
};
goal_src/jak1/game.gp
+116 -87
View File
@@ -25,6 +25,50 @@
;; It is an error to not provide a step to make a required file.
;; It is an error to have a circular dependency and this will crash the compiler due to stack overflow.
;; our input/output directories aren't fixed ahead of time and some users may override them.
;; the map-path! command can be used to define path constants that can be used in in/out/dep and
;; inside certain tool files, like text project files.
;; The substitution is only used at the beginning of paths, and I hope to keep the number
;; of these to a minimum - it's quite confusing to have all these non-fixed paths everywhere.
;;;;;;;;;;;;;;;;;;;;;;;
;; Inputs from ISO
;;;;;;;;;;;;;;;;;;;;;;;
(cond
;; extractor can override everything by providing *use-iso-data-path*
(*use-iso-data-path*
(map-path! "$ISO" (string-append *iso-data* "/")))
;; user-specific places to put $ISO
((user? dass)
(map-path! "$ISO" "iso_data/jak1_us2/"))
;; for normal people, just use jak1.
(#t
(map-path! "$ISO" "iso_data/jak1/")))
;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Inputs from decompiler
;;;;;;;;;;;;;;;;;;;;;;;;;;
(cond
;; user-specific places to put $ISO
((user? dass)
(map-path! "$DECOMP" "decompiler_out/jak1_us2/"))
(#t
(map-path! "$DECOMP" "decompiler_out/jak1/")))
;;;;;;;;;;;;;;;;;;;;;;;
;; Output
;;;;;;;;;;;;;;;;;;;;;;;
;; NOTE: this isn't perfect yet, some tools still are hardcoded to look in out/ for stuff.
(map-path! "$OUT" "out/")
;; use defmacro to define goos macros.
(define defmacro defsmacro)
(define defun desfun)
;;;;;;;;;;;;;;;;;;;;;;;
;; Build Groups
;;;;;;;;;;;;;;;;;;;;;;;
@@ -36,18 +80,12 @@
(define *all-vag* '())
(define *all-gc* '())
;;;;;;;;;;;;;;;;;;;;;;;
;; Build system macros
;;;;;;;;;;;;;;;;;;;;;;;
;; use defmacro to define goos macros.
(define defmacro defsmacro)
(define defun desfun)
(defun gc-file->o-file (filename)
"Get the name of the object file for the given GOAL (*.gc) source file."
(string-append "out/obj/" (stem filename) ".o")
(string-append "$OUT/obj/" (stem filename) ".o")
)
(defmacro goal-src (src-file &rest deps)
@@ -105,7 +143,7 @@
(defun custom-level-cgo (output-name desc-file-name)
"Add a CGO with the given output name (in out/iso) and input name (in custom_levels/)"
(let ((out-name (string-append "out/iso/" output-name)))
(let ((out-name (string-append "$OUT/iso/" output-name)))
(defstep :in (string-append "custom_levels/" desc-file-name)
:tool 'dgo
:out `(,out-name)
@@ -116,7 +154,7 @@
(defun cgo (output-name desc-file-name)
"Add a CGO with the given output name (in out/iso) and input name (in goal_src/jak1/dgos)"
(let ((out-name (string-append "out/iso/" output-name)))
(let ((out-name (string-append "$OUT/iso/" output-name)))
(defstep :in (string-append "goal_src/jak1/dgos/" desc-file-name)
:tool 'dgo
:out `(,out-name)
@@ -131,16 +169,13 @@
)
(define *game-directory* (get-environment-variable "OPENGOAL_DECOMP_DIR" :default "jak1/"))
(when (user? dass)
(set! *game-directory* "jak1_us2/"))
(defmacro copy-texture (tpage-id)
"Copy a texture from the game, using the given tpage ID"
(let* ((path (string-append "decompiler_out/" *game-directory* "raw_obj/" (tpage-name tpage-id))))
(let* ((path (string-append "$DECOMP/raw_obj/" (tpage-name tpage-id))))
`(defstep :in ,path
:tool 'copy
:out '(,(string-append "out/obj/" (tpage-name tpage-id))))))
:out '(,(string-append "$OUT/obj/" (tpage-name tpage-id))))))
(defmacro copy-textures (&rest ids)
`(begin
@@ -149,10 +184,10 @@
)
(defmacro copy-go (name)
(let* ((path (string-append "decompiler_out/" *game-directory* "raw_obj/" name ".go")))
(let* ((path (string-append "$DECOMP/raw_obj/" name ".go")))
`(defstep :in ,path
:tool 'copy
:out '(,(string-append "out/obj/" name ".go")))))
:out '(,(string-append "$OUT/obj/" name ".go")))))
(defmacro copy-gos (&rest gos)
`(begin
@@ -164,18 +199,12 @@
(let* ((path (string-append "custom_levels/" name "/" name ".jsonc")))
`(defstep :in ,path
:tool 'build-level
:out '(,(string-append "out/obj/" name ".go")))))
:out '(,(string-append "$OUT/obj/" name ".go")))))
(defun get-iso-data-path ()
(if *use-iso-data-path*
(string-append *iso-data* "/")
(string-append "iso_data/" *game-directory* "/")
)
)
(defun copy-iso-file (name subdir ext)
(let* ((path (string-append (get-iso-data-path) subdir name ext))
(out-name (string-append "out/iso/" name ext)))
(let* ((path (string-append "$ISO/" subdir name ext))
(out-name (string-append "$OUT/iso/" name ext)))
(defstep :in path
:tool 'copy
:out `(,out-name))
@@ -248,36 +277,36 @@
;;;;;;;;;;;;;;;;;;;;;;;;
;; The tpage directory
(defstep :in "assets/tpage-dir.txt"
(defstep :in "$DECOMP/textures/tpage-dir.txt"
:tool 'tpage-dir
:out '("out/obj/dir-tpages.go")
:out '("$OUT/obj/dir-tpages.go")
)
;; the count file.
(defstep :in "assets/game_count.txt"
(defstep :in "$DECOMP/assets/game_count.txt"
:tool 'game-cnt
:out '("out/obj/game-cnt.go")
:out '("$OUT/obj/game-cnt.go")
)
;; the TWEAKVAL file
(defstep :in (string-append (get-iso-data-path) "MUS/TWEAKVAL.MUS")
(defstep :in "$ISO/MUS/TWEAKVAL.MUS"
:tool 'copy
:out '("out/iso/TWEAKVAL.MUS"))
:out '("$OUT/iso/TWEAKVAL.MUS"))
;; the VAGDIR file
(defstep :in (string-append (get-iso-data-path) "VAG/VAGDIR.AYB")
(defstep :in "$ISO/VAG/VAGDIR.AYB"
:tool 'copy
:out '("out/iso/VAGDIR.AYB"))
:out '("$OUT/iso/VAGDIR.AYB"))
;; the save icon file
(defstep :in (string-append (get-iso-data-path) "DRIVERS/SAVEGAME.ICO")
(defstep :in "$ISO/DRIVERS/SAVEGAME.ICO"
:tool 'copy
:out '("out/iso/SAVEGAME.ICO"))
:out '("$OUT/iso/SAVEGAME.ICO"))
;; the loading screen file
(defstep :in (string-append (get-iso-data-path) "DRIVERS/SCREEN1.USA")
(defstep :in "$ISO/DRIVERS/SCREEN1.USA"
:tool 'copy
:out '("out/iso/SCREEN1.USA"))
:out '("$OUT/iso/SCREEN1.USA"))
;;;;;;;;;;;;;;;;;;;;;
;; Textures (Common)
@@ -350,20 +379,20 @@
(defstep :in "game/assets/game_text.gp"
:tool 'text
:out '("out/iso/0COMMON.TXT"
"out/iso/1COMMON.TXT"
"out/iso/2COMMON.TXT"
"out/iso/3COMMON.TXT"
"out/iso/4COMMON.TXT"
"out/iso/5COMMON.TXT"
"out/iso/6COMMON.TXT")
:out '("$OUT/iso/0COMMON.TXT"
"$OUT/iso/1COMMON.TXT"
"$OUT/iso/2COMMON.TXT"
"$OUT/iso/3COMMON.TXT"
"$OUT/iso/4COMMON.TXT"
"$OUT/iso/5COMMON.TXT"
"$OUT/iso/6COMMON.TXT")
)
(defstep :in "game/assets/game_subtitle.gp"
:tool 'subtitle
:out '("out/iso/0SUBTIT.TXT"
"out/iso/3SUBTIT.TXT"
"out/iso/6SUBTIT.TXT")
:out '("$OUT/iso/0SUBTIT.TXT"
"$OUT/iso/3SUBTIT.TXT"
"$OUT/iso/6SUBTIT.TXT")
)
@@ -374,10 +403,10 @@
;; the engine group is a group of files required to boot the game engine with no levels
(group "engine"
"out/iso/0COMMON.TXT"
"out/iso/0SUBTIT.TXT"
"out/iso/KERNEL.CGO"
"out/iso/GAME.CGO"
"$OUT/iso/0COMMON.TXT"
"$OUT/iso/0SUBTIT.TXT"
"$OUT/iso/KERNEL.CGO"
"$OUT/iso/GAME.CGO"
)
;;;;;;;;;;;;;;;;;;;;;;;;
@@ -417,7 +446,7 @@
(goal-src-sequence
"levels/"
:deps ;; no idea what these depend on, make it depend on the whole engine
("out/obj/ticky.o")
("$OUT/obj/ticky.o")
"village_common/villagep-obs.gc"
"village_common/oracle.gc"
@@ -453,7 +482,7 @@
(goal-src-sequence
"levels/beach/"
:deps ("out/obj/ticky.o")
:deps ("$OUT/obj/ticky.o")
"air-h.gc"
"air.gc"
"wobbler.gc"
@@ -527,7 +556,7 @@
(goal-src-sequence
"levels/jungle/"
:deps ;; no idea what these depend on, make it depend on the whole engine
("out/obj/ticky.o")
("$OUT/obj/ticky.o")
"jungle-elevator.gc"
"bouncer.gc"
@@ -598,7 +627,7 @@
(goal-src-sequence
"levels/"
:deps ;; no idea what these depend on, make it depend on the whole engine
("out/obj/ticky.o")
("$OUT/obj/ticky.o")
"village1/farmer.gc"
"village1/explorer.gc"
@@ -677,7 +706,7 @@
(goal-src-sequence
"levels/jungleb/"
:deps ;; no idea what these depend on, make it depend on the whole engine
("out/obj/ticky.o")
("$OUT/obj/ticky.o")
"jungleb-obs.gc"
"plat-flip.gc"
@@ -710,7 +739,7 @@
(goal-src-sequence
"levels/misty/"
:deps ("out/obj/evilbro.o")
:deps ("$OUT/obj/evilbro.o")
"mistycannon.gc"
"babak-with-cannon.gc"
"misty-obs.gc"
@@ -778,7 +807,7 @@
(goal-src-sequence
"levels/swamp/"
:deps ("out/obj/ticky.o")
:deps ("$OUT/obj/ticky.o")
"swamp-obs.gc"
"swamp-bat.gc"
"swamp-rat.gc"
@@ -822,7 +851,7 @@
(goal-src-sequence
"levels/sunken/"
:deps ("out/obj/ticky.o")
:deps ("$OUT/obj/ticky.o")
"sunken-part.gc"
"sunken-part2.gc"
"sunken-part3.gc"
@@ -915,7 +944,7 @@
(goal-src-sequence
"levels/snow/"
:deps ("out/obj/ticky.o")
:deps ("$OUT/obj/ticky.o")
"target-snowball.gc"
"target-ice.gc"
"ice-cube.gc"
@@ -979,7 +1008,7 @@
(goal-src-sequence
"levels/firecanyon/"
:deps ;; no idea what these depend on, make it depend on the whole engine
("out/obj/ticky.o")
("$OUT/obj/ticky.o")
"firecanyon-part.gc"
"assistant-firecanyon.gc"
@@ -1011,7 +1040,7 @@
(goal-src-sequence
"levels/ogre/"
:deps ("out/obj/ticky.o")
:deps ("$OUT/obj/ticky.o")
"ogre-part.gc"
"ogreboss.gc"
"ogre-obs.gc"
@@ -1052,7 +1081,7 @@
(goal-src-sequence
"levels/village2/"
:deps ("out/obj/ticky.o")
:deps ("$OUT/obj/ticky.o")
"village2-part.gc"
"village2-obs.gc"
"village2-part2.gc"
@@ -1122,7 +1151,7 @@
(goal-src-sequence
"levels/rolling/"
:deps ("out/obj/ticky.o")
:deps ("$OUT/obj/ticky.o")
"rolling-obs.gc"
"rolling-lightning-mole.gc"
"rolling-robber.gc"
@@ -1164,7 +1193,7 @@
(goal-src-sequence
"levels/"
:deps ;; no idea what these depend on, make it depend on the whole engine
("out/obj/ticky.o")
("$OUT/obj/ticky.o")
"village3/village3-part.gc"
"village3/village3-obs.gc"
"village3/minecart.gc"
@@ -1222,7 +1251,7 @@
;; The code
(goal-src-sequence
"levels/training/"
:deps ("out/obj/ticky.o") ;; makes us depend on the whole engine
:deps ("$OUT/obj/ticky.o") ;; makes us depend on the whole engine
"training-obs.gc"
"training-part.gc"
@@ -1251,7 +1280,7 @@
(goal-src-sequence
"levels/"
:deps ;; no idea what these depend on, make it depend on the whole engine
("out/obj/ticky.o"
("$OUT/obj/ticky.o"
)
"maincave/cavecrystal-light.gc"
"darkcave/darkcave-obs.gc"
@@ -1318,7 +1347,7 @@
(goal-src-sequence
"levels/robocave/"
:deps ;; no idea what these depend on, make it depend on the whole engine
("out/obj/ticky.o")
("$OUT/obj/ticky.o")
"cave-trap.gc"
"spider-egg.gc"
"robocave-part.gc"
@@ -1346,7 +1375,7 @@
(goal-src-sequence
"levels/lavatube/"
:deps ("out/obj/ticky.o")
:deps ("$OUT/obj/ticky.o")
"lavatube-obs.gc"
"lavatube-energy.gc"
@@ -1391,7 +1420,7 @@
(goal-src-sequence
"levels/citadel/"
:deps ("out/obj/battlecontroller.o" "out/obj/snow-bunny.o")
:deps ("$OUT/obj/battlecontroller.o" "out/obj/snow-bunny.o")
"citadel-part.gc"
"citadel-obs.gc"
@@ -1455,7 +1484,7 @@
(goal-src-sequence
"levels/finalboss/"
:deps ("out/obj/assistant-citadel.o")
:deps ("$OUT/obj/assistant-citadel.o")
"robotboss-h.gc"
"robotboss-part.gc"
@@ -1510,7 +1539,7 @@
(goal-src-sequence
"levels/intro/"
:deps ;; no idea what these depend on, make it depend on the whole engine
("out/obj/ticky.o")
("$OUT/obj/ticky.o")
"evilbro.gc"
)
@@ -1537,7 +1566,7 @@
(goal-src-sequence
"levels/demo/"
:deps ;; no idea what these depend on, make it depend on the whole engine
("out/obj/ticky.o")
("$OUT/obj/ticky.o")
"static-screen.gc"
"demo-obs.gc"
@@ -1559,7 +1588,7 @@
(goal-src-sequence
"levels/title/"
:deps ;; no idea what these depend on, make it depend on the whole engine
("out/obj/ticky.o")
("$OUT/obj/ticky.o")
"title-obs.gc"
)
@@ -1600,10 +1629,10 @@
"engine/"
:deps
("out/obj/gcommon.o"
"out/obj/gstate.o"
"out/obj/gstring.o"
"out/obj/gkernel.o"
("$OUT/obj/gcommon.o"
"$OUT/obj/gstate.o"
"$OUT/obj/gstring.o"
"$OUT/obj/gkernel.o"
)
;; sources
@@ -1648,7 +1677,7 @@
"engine/"
:deps
("out/ps2/pad.o")
("$OUT/ps2/pad.o")
"gfx/hw/gs.gc"
"gfx/hw/display-h.gc"
"math/vector.gc"
@@ -1875,7 +1904,7 @@
"engine/"
:deps
("out/main.o")
("$OUT/main.o")
"collide/collide-cache.gc"
"entity/relocate.gc"
@@ -1919,7 +1948,7 @@
(goal-src-sequence
"levels/common/"
:deps ("out/obj/default-menu.o")
:deps ("$OUT/obj/default-menu.o")
"texture-upload.gc"
"rigid-body-h.gc"
"water-anim.gc"
@@ -1951,10 +1980,10 @@
;; the iso group is a group of files built by the "(mi)" command.
(group-list "iso"
`("out/iso/0COMMON.TXT"
"out/iso/0SUBTIT.TXT"
"out/iso/TWEAKVAL.MUS"
"out/iso/VAGDIR.AYB"
`("$OUT/iso/0COMMON.TXT"
"$OUT/iso/0SUBTIT.TXT"
"$OUT/iso/TWEAKVAL.MUS"
"$OUT/iso/VAGDIR.AYB"
,@(reverse *all-vis*)
,@(reverse *all-str*)
,@(reverse *all-sbk*)
@@ -1969,8 +1998,8 @@
(group-list "text"
`("out/iso/0COMMON.TXT"
"out/iso/0SUBTIT.TXT"
`("$OUT/iso/0COMMON.TXT"
"$OUT/iso/0SUBTIT.TXT"
)
)
goalc/build_level/build_level.cpp
+7 -4
View File
@@ -14,7 +14,9 @@
#include "third-party/fmt/core.h"
void save_pc_data(const std::string& nickname, tfrag3::Level& data) {
void save_pc_data(const std::string& nickname,
tfrag3::Level& data,
const std::filesystem::path& output_dir) {
Serializer ser;
data.serialize(ser);
auto compressed =
@@ -23,8 +25,8 @@ void save_pc_data(const std::string& nickname, tfrag3::Level& data) {
print_memory_usage(data, ser.get_save_result().second);
fmt::print("compressed: {} -> {} ({:.2f}%)\n", ser.get_save_result().second, compressed.size(),
100.f * compressed.size() / ser.get_save_result().second);
file_util::write_binary_file(file_util::get_file_path({fmt::format("assets/{}.fr3", nickname)}),
compressed.data(), compressed.size());
file_util::write_binary_file(output_dir / fmt::format("{}.fr3", nickname), compressed.data(),
compressed.size());
}
std::vector<std::string> get_build_level_deps(const std::string& input_file) {
@@ -106,7 +108,8 @@ bool run_build_level(const std::string& input_file, const std::string& output_fi
file_util::write_binary_file(save_path, result.data(), result.size());
// Save the PC level
save_pc_data(file.nickname, pc_level);
// TODO out dir
save_pc_data(file.nickname, pc_level, file_util::get_jak_project_dir() / "out" / "fr3");
return true;
}
goalc/make/MakeSystem.cpp
+31 -11
View File
@@ -54,6 +54,11 @@ MakeSystem::MakeSystem(const std::string& username) : m_goos(username) {
return handle_stem(obj, args, env);
});
m_goos.register_form("map-path!", [=](const goos::Object& obj, goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
return handle_map_path(obj, args, env);
});
m_goos.set_global_variable_to_symbol("ASSETS", "#t");
set_constant("*iso-data*", file_util::get_file_path({"iso_data"}));
@@ -98,7 +103,7 @@ goos::Object MakeSystem::handle_defstep(const goos::Object& form,
auto step = std::make_shared<MakeStep>();
goos::for_each_in_list(args.get_named("out"), [&](const goos::Object& obj) {
step->outputs.push_back(obj.as_string()->data);
step->outputs.push_back(m_path_map.apply_remaps(obj.as_string()->data));
});
step->tool = args.get_named("tool").as_symbol()->name;
@@ -111,16 +116,17 @@ goos::Object MakeSystem::handle_defstep(const goos::Object& form,
const auto& in = args.get_named("in");
if (in.is_pair()) {
step->input.clear();
goos::for_each_in_list(
in, [&](const goos::Object& o) { step->input.push_back(o.as_string()->data); });
goos::for_each_in_list(in, [&](const goos::Object& o) {
step->input.push_back(m_path_map.apply_remaps(o.as_string()->data));
});
} else {
step->input = {in.as_string()->data};
step->input = {m_path_map.apply_remaps(in.as_string()->data)};
}
}
if (args.has_named("dep")) {
goos::for_each_in_list(args.get_named("dep"), [&](const goos::Object& obj) {
step->deps.push_back(obj.as_string()->data);
step->deps.push_back(m_path_map.apply_remaps(obj.as_string()->data));
});
}
@@ -182,6 +188,20 @@ goos::Object MakeSystem::handle_stem(const goos::Object& form,
return goos::StringObject::make_new(input.stem().u8string());
}
goos::Object MakeSystem::handle_map_path(const goos::Object& form,
goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
m_goos.eval_args(&args, env);
va_check(form, args, {goos::ObjectType::STRING, goos::ObjectType::STRING}, {});
auto old_path = args.unnamed.at(0).as_string()->data;
if (old_path.empty() || old_path[0] != '$') {
throw std::runtime_error(fmt::format("Invalid path remap {}, must start with $", old_path));
}
auto new_path = args.unnamed.at(1).as_string()->data;
m_path_map.path_remap[old_path] = new_path;
return goos::Object::make_empty_list();
}
void MakeSystem::get_dependencies(const std::string& master_target,
const std::string& output,
std::vector<std::string>* result,
@@ -202,9 +222,9 @@ void MakeSystem::get_dependencies(const std::string& master_target,
}
const auto& rule = rule_it->second;
for (auto& dep :
m_tools.at(rule->tool)
->get_additional_dependencies({rule->input, rule->deps, rule->outputs, rule->arg})) {
for (auto& dep : m_tools.at(rule->tool)
->get_additional_dependencies(
{rule->input, rule->deps, rule->outputs, rule->arg}, m_path_map)) {
get_dependencies(master_target, dep, result, result_set);
}
@@ -247,7 +267,7 @@ std::vector<std::string> MakeSystem::filter_dependencies(const std::vector<std::
bool added = false;
if (tool->needs_run(task)) {
if (tool->needs_run(task, m_path_map)) {
result.push_back(to_make);
stale_deps.insert(to_make);
added = true;
@@ -267,7 +287,7 @@ std::vector<std::string> MakeSystem::filter_dependencies(const std::vector<std::
if (!added) {
// check transitive dependencies
for (auto& dep : tool->get_additional_dependencies(task)) {
for (auto& dep : tool->get_additional_dependencies(task, m_path_map)) {
if (stale_deps.find(dep) != stale_deps.end()) {
result.push_back(to_make);
stale_deps.insert(to_make);
@@ -334,7 +354,7 @@ bool MakeSystem::make(const std::string& target, bool force, bool verbose) {
bool success = false;
try {
success = tool->run({rule->input, rule->deps, rule->outputs, rule->arg});
success = tool->run({rule->input, rule->deps, rule->outputs, rule->arg}, m_path_map);
} catch (std::exception& e) {
fmt::print("\n");
fmt::print("Error: {}\n", e.what());
goalc/make/MakeSystem.h
+5
View File
@@ -30,6 +30,10 @@ class MakeSystem {
goos::Arguments&,
const std::shared_ptr<goos::EnvironmentObject>& env);
goos::Object handle_map_path(const goos::Object& obj,
goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env);
std::vector<std::string> get_dependencies(const std::string& target) const;
std::vector<std::string> filter_dependencies(const std::vector<std::string>& all_deps);
@@ -62,4 +66,5 @@ class MakeSystem {
std::unordered_map<std::string, std::shared_ptr<MakeStep>> m_output_to_step;
std::unordered_map<std::string, std::shared_ptr<Tool>> m_tools;
PathMap m_path_map;
};
goalc/make/Tool.cpp
+30 -2
View File
@@ -10,7 +10,7 @@
Tool::Tool(const std::string& name) : m_name(name) {}
bool Tool::needs_run(const ToolInput& task) {
bool Tool::needs_run(const ToolInput& task, const PathMap& path_map) {
// for this to return false, all outputs need to be newer than all inputs.
for (auto& in : task.input) {
@@ -33,7 +33,7 @@ bool Tool::needs_run(const ToolInput& task) {
}
}
for (auto& dep : get_additional_dependencies(task)) {
for (auto& dep : get_additional_dependencies(task, path_map)) {
auto dep_path = std::filesystem::path(file_util::get_file_path({dep}));
if (std::filesystem::exists(dep_path)) {
auto dep_time = std::filesystem::last_write_time(dep_path);
@@ -60,3 +60,31 @@ bool Tool::needs_run(const ToolInput& task) {
return false;
}
std::string PathMap::apply_remaps(const std::string& input) const {
if (!input.empty() && input[0] == '$') {
std::string prefix = "$";
size_t i = 1;
for (; i < input.size(); i++) {
char c = input[i];
if (c == '/') {
break;
} else {
prefix.push_back(c);
}
}
const auto& it = path_remap.find(prefix);
if (it == path_remap.end()) {
return input;
} else {
std::string result = it->second;
while (!result.empty() && result.back() == '/') {
result.pop_back();
}
result.insert(result.end(), input.begin() + i, input.end());
return result;
}
} else {
return input;
}
}
goalc/make/Tool.h
+11 -3
View File
@@ -6,6 +6,11 @@
#include "common/goos/Object.h"
struct PathMap {
std::unordered_map<std::string, std::string> path_remap;
std::string apply_remaps(const std::string& input) const;
};
struct ToolInput {
std::vector<std::string>& input; // the input file
std::vector<std::string>& deps; // explicit dependencies
@@ -16,9 +21,12 @@ struct ToolInput {
class Tool {
public:
Tool(const std::string& name);
virtual bool run(const ToolInput& task) = 0;
virtual std::vector<std::string> get_additional_dependencies(const ToolInput&) { return {}; }
virtual bool needs_run(const ToolInput& task);
virtual bool run(const ToolInput& task, const PathMap& path_map) = 0;
virtual std::vector<std::string> get_additional_dependencies(const ToolInput&,
const PathMap& /*path_map*/) {
return {};
}
virtual bool needs_run(const ToolInput& task, const PathMap& path_map);
virtual ~Tool() = default;
const std::string& name() const { return m_name; }
goalc/make/Tools.cpp
+32 -18
View File
@@ -16,7 +16,7 @@
CompilerTool::CompilerTool(Compiler* compiler) : Tool("goalc"), m_compiler(compiler) {}
bool CompilerTool::needs_run(const ToolInput& task) {
bool CompilerTool::needs_run(const ToolInput& task, const PathMap& path_map) {
if (task.input.size() != 1) {
throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name()));
}
@@ -24,10 +24,10 @@ bool CompilerTool::needs_run(const ToolInput& task) {
if (!m_compiler->knows_object_file(std::filesystem::path(task.input.at(0)).stem().u8string())) {
return true;
}
return Tool::needs_run(task);
return Tool::needs_run(task, path_map);
}
bool CompilerTool::run(const ToolInput& task) {
bool CompilerTool::run(const ToolInput& task, const PathMap& /*path_map*/) {
// todo check inputs
try {
CompilationOptions options;
@@ -76,7 +76,7 @@ DgoDescription parse_desc_file(const std::string& filename, goos::Reader& reader
DgoTool::DgoTool() : Tool("dgo") {}
bool DgoTool::run(const ToolInput& task) {
bool DgoTool::run(const ToolInput& task, const PathMap& /*path_map*/) {
if (task.input.size() != 1) {
throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name()));
}
@@ -85,10 +85,12 @@ bool DgoTool::run(const ToolInput& task) {
return true;
}
std::vector<std::string> DgoTool::get_additional_dependencies(const ToolInput& task) {
std::vector<std::string> DgoTool::get_additional_dependencies(const ToolInput& task,
const PathMap& /*path_map*/) {
std::vector<std::string> result;
auto desc = parse_desc_file(task.input.at(0), m_reader);
for (auto& x : desc.entries) {
// todo out
result.push_back(fmt::format("out/obj/{}", x.file_name));
}
return result;
@@ -96,7 +98,7 @@ std::vector<std::string> DgoTool::get_additional_dependencies(const ToolInput& t
TpageDirTool::TpageDirTool() : Tool("tpage-dir") {}
bool TpageDirTool::run(const ToolInput& task) {
bool TpageDirTool::run(const ToolInput& task, const PathMap& /*path_map*/) {
if (task.input.size() != 1) {
throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name()));
}
@@ -106,7 +108,7 @@ bool TpageDirTool::run(const ToolInput& task) {
CopyTool::CopyTool() : Tool("copy") {}
bool CopyTool::run(const ToolInput& task) {
bool CopyTool::run(const ToolInput& task, const PathMap& /*path_map*/) {
if (task.input.size() != 1) {
throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name()));
}
@@ -120,7 +122,7 @@ bool CopyTool::run(const ToolInput& task) {
GameCntTool::GameCntTool() : Tool("game-cnt") {}
bool GameCntTool::run(const ToolInput& task) {
bool GameCntTool::run(const ToolInput& task, const PathMap& /*path_map*/) {
if (task.input.size() != 1) {
throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name()));
}
@@ -130,63 +132,75 @@ bool GameCntTool::run(const ToolInput& task) {
TextTool::TextTool() : Tool("text") {}
bool TextTool::needs_run(const ToolInput& task) {
bool TextTool::needs_run(const ToolInput& task, const PathMap& path_map) {
if (task.input.size() != 1) {
throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name()));
}
std::vector<std::string> deps;
open_text_project("text", task.input.at(0), deps);
return Tool::needs_run({task.input, deps, task.output, task.arg});
for (auto& dep : deps) {
dep = path_map.apply_remaps(dep);
}
return Tool::needs_run({task.input, deps, task.output, task.arg}, path_map);
}
bool TextTool::run(const ToolInput& task) {
bool TextTool::run(const ToolInput& task, const PathMap& path_map) {
GameTextDB db;
std::vector<std::string> inputs;
open_text_project("text", task.input.at(0), inputs);
for (auto& in : inputs) {
in = path_map.apply_remaps(in);
}
compile_game_text(inputs, db);
return true;
}
GroupTool::GroupTool() : Tool("group") {}
bool GroupTool::run(const ToolInput&) {
bool GroupTool::run(const ToolInput&, const PathMap& /*path_map*/) {
return true;
}
SubtitleTool::SubtitleTool() : Tool("subtitle") {}
bool SubtitleTool::needs_run(const ToolInput& task) {
bool SubtitleTool::needs_run(const ToolInput& task, const PathMap& path_map) {
if (task.input.size() != 1) {
throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name()));
}
std::vector<std::string> deps;
open_text_project("subtitle", task.input.at(0), deps);
return Tool::needs_run({task.input, deps, task.output, task.arg});
for (auto& dep : deps) {
dep = path_map.apply_remaps(dep);
}
return Tool::needs_run({task.input, deps, task.output, task.arg}, path_map);
}
bool SubtitleTool::run(const ToolInput& task) {
bool SubtitleTool::run(const ToolInput& task, const PathMap& path_map) {
GameSubtitleDB db;
db.m_subtitle_groups = std::make_unique<GameSubtitleGroups>();
db.m_subtitle_groups->hydrate_from_asset_file();
std::vector<std::string> inputs;
open_text_project("subtitle", task.input.at(0), inputs);
for (auto& in : inputs) {
in = path_map.apply_remaps(in);
}
compile_game_subtitle(inputs, db);
return true;
}
BuildLevelTool::BuildLevelTool() : Tool("build-level") {}
bool BuildLevelTool::needs_run(const ToolInput& task) {
bool BuildLevelTool::needs_run(const ToolInput& task, const PathMap& path_map) {
if (task.input.size() != 1) {
throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name()));
}
auto deps = get_build_level_deps(task.input.at(0));
return Tool::needs_run({task.input, deps, task.output, task.arg});
return Tool::needs_run({task.input, deps, task.output, task.arg}, path_map);
}
bool BuildLevelTool::run(const ToolInput& task) {
bool BuildLevelTool::run(const ToolInput& task, const PathMap& /*path_map*/) {
if (task.input.size() != 1) {
throw std::runtime_error(fmt::format("Invalid amount of inputs to {} tool", name()));
}
goalc/make/Tools.h
+15 -14
View File
@@ -9,8 +9,8 @@ class Compiler;
class CompilerTool : public Tool {
public:
CompilerTool(Compiler* compiler);
bool run(const ToolInput& task) override;
bool needs_run(const ToolInput& task) override;
bool run(const ToolInput& task, const PathMap& path_map) override;
bool needs_run(const ToolInput& task, const PathMap& path_map) override;
private:
Compiler* m_compiler = nullptr;
@@ -19,8 +19,9 @@ class CompilerTool : public Tool {
class DgoTool : public Tool {
public:
DgoTool();
bool run(const ToolInput& task) override;
std::vector<std::string> get_additional_dependencies(const ToolInput&) override;
bool run(const ToolInput& task, const PathMap& path_map) override;
std::vector<std::string> get_additional_dependencies(const ToolInput&,
const PathMap& path_map) override;
private:
goos::Reader m_reader;
@@ -29,44 +30,44 @@ class DgoTool : public Tool {
class TpageDirTool : public Tool {
public:
TpageDirTool();
bool run(const ToolInput& task) override;
bool run(const ToolInput& task, const PathMap& path_map) override;
};
class CopyTool : public Tool {
public:
CopyTool();
bool run(const ToolInput& task) override;
bool run(const ToolInput& task, const PathMap& path_map) override;
};
class GameCntTool : public Tool {
public:
GameCntTool();
bool run(const ToolInput& task) override;
bool run(const ToolInput& task, const PathMap& path_map) override;
};
class TextTool : public Tool {
public:
TextTool();
bool run(const ToolInput& task) override;
bool needs_run(const ToolInput& task) override;
bool run(const ToolInput& task, const PathMap& path_map) override;
bool needs_run(const ToolInput& task, const PathMap& path_map) override;
};
class GroupTool : public Tool {
public:
GroupTool();
bool run(const ToolInput& task) override;
bool run(const ToolInput& task, const PathMap& path_map) override;
};
class SubtitleTool : public Tool {
public:
SubtitleTool();
bool run(const ToolInput& task) override;
bool needs_run(const ToolInput& task) override;
bool run(const ToolInput& task, const PathMap& path_map) override;
bool needs_run(const ToolInput& task, const PathMap& path_map) override;
};
class BuildLevelTool : public Tool {
public:
BuildLevelTool();
bool run(const ToolInput& task) override;
bool needs_run(const ToolInput& task) override;
bool run(const ToolInput& task, const PathMap& path_map) override;
bool needs_run(const ToolInput& task, const PathMap& path_map) override;
};
test/offline/offline_test_main.cpp
+6 -6
View File
@@ -264,20 +264,20 @@ Decompiler setup_decompiler(const std::vector<DecompilerFile>& files,
// don't try to do this because we can't write the file
dc.config->generate_symbol_definition_map = false;
std::vector<std::string> dgo_paths;
std::vector<std::filesystem::path> dgo_paths;
if (args.iso_data_path.empty()) {
for (auto& x : offline_config.dgos) {
dgo_paths.push_back((file_util::get_jak_project_dir() / "iso_data" / "jak1" / x).string());
dgo_paths.push_back(file_util::get_jak_project_dir() / "iso_data" / "jak1" / x);
}
} else {
for (auto& x : offline_config.dgos) {
dgo_paths.push_back(file_util::combine_path(args.iso_data_path, x));
dgo_paths.push_back(std::filesystem::path(args.iso_data_path) / x);
}
}
dc.db = std::make_unique<decompiler::ObjectFileDB>(dgo_paths, dc.config->obj_file_name_map_file,
std::vector<std::string>{},
std::vector<std::string>{}, *dc.config);
dc.db = std::make_unique<decompiler::ObjectFileDB>(
dgo_paths, dc.config->obj_file_name_map_file, std::vector<std::filesystem::path>{},
std::vector<std::filesystem::path>{}, *dc.config);
std::unordered_set<std::string> db_files;
for (auto& files_by_name : dc.db->obj_files_by_name) {