mirror/jak-project
1
0
Fork 0
mirror of https://github.com/open-goal/jak-project synced 2026-05-23 15:02:01 -04:00
Code Issues Packages Projects Releases Wiki Activity
Files
2f74c271d40b373f200d2ea0a1a5f9cdaac82f53
jak-project/decompiler/ObjectFile/ObjectFileDB.cpp
T
Tyler Wilding d3cc739e43 jakx: Commit existing work from other PRs (#4112) 
This attempts to get into master whatever work was done in this PR /
it's earlier PR https://github.com/open-goal/jak-project/pull/3965

I don't want this work to be lost / floating around in massive PRs.

However the changes are:
- switch to ntsc_v1 instead of PAL as the development target, as we have
done for all other games
- remove most of the copied-from-jak2/3 changes as they need to be
confirmed during the decompilation process not just assumed
- avoids committing any changes to `game/kernel/common` as it was not
clear to me if these were changes made in jak x's kernel that were not
properly broken out into it's own functions. We don't want to
accidentally introduce bugs into jak1-3's kernel code.
- in other words, if the change in the kernel only happens in jak x...it
should likely be specific to jak x's kernel, not common.

---------

Co-authored-by: VodBox <dillon@vodbox.io>
Co-authored-by: yodah <greenboyyodah@gmail.com>
2025-12-31 21:08:44 -05:00

1182 lines
41 KiB
C++
Raw Blame History

/*!
* @file ObjectFileDB.cpp
* A "database" of object files found in DGO files.
* Eliminates duplicate object files, and also assigns unique names to all object files
* (there may be different object files with the same name sometimes)
*/
#include "ObjectFileDB.h"
#include <algorithm>
#include <cstring>
#include <map>
#include <set>
#include "LinkedObjectFileCreation.h"
#include "common/link_types.h"
#include "common/log/log.h"
#include "common/texture/texture_slots.h"
#include "common/util/BinaryReader.h"
#include "common/util/BitUtils.h"
#include "common/util/FileUtil.h"
#include "common/util/Timer.h"
#include "common/util/crc32.h"
#include "common/util/dgo_util.h"
#include "common/util/json_util.h"
#include "decompiler/Function/BasicBlocks.h"
#include "decompiler/config.h"
#include "decompiler/data/StrFileReader.h"
#include "decompiler/data/dir_tpages.h"
#include "decompiler/data/game_count.h"
#include "decompiler/data/game_subs.h"
#include "decompiler/data/game_text.h"
#include "decompiler/data/tpage.h"
#include "third-party/xdelta3/xdelta3.h"
namespace decompiler {
namespace {
std::string strip_dgo_extension(const std::string& x) {
auto ext = x.substr(x.length() - 4, 4);
if (ext == ".CGO" || ext == ".cgo" || ext == ".DGO" || ext == ".dgo") {
return x.substr(0, x.length() - 4);
}
return x;
}
/*!
* Get an object name from a file name.
* Strips off the file extension and anything before the last slash.
*/
std::string obj_filename_to_name(const std::string& x) {
auto end = x.length();
// find last dot
auto last_dot = end;
for (; last_dot-- > 0;) {
if (x.at(last_dot) == '.') {
break;
}
}
if (last_dot == 0) {
last_dot = end;
}
auto last_slash = end;
for (; last_slash-- > 0;) {
if (x.at(last_slash) == '\\' || x.at(last_slash) == '/') {
break;
}
}
ASSERT(last_dot > last_slash + 1);
ASSERT(last_slash + 1 < x.length());
return x.substr(last_slash + 1, last_dot - last_slash - 1);
}
} // namespace
std::string ObjectFileData::to_unique_name() const {
if (!name_from_map.empty()) {
return name_from_map;
}
if (has_multiple_versions) {
std::string result = record.name + "-";
auto dgo_names_sorted = dgo_names;
std::sort(dgo_names_sorted.begin(), dgo_names_sorted.end());
for (const auto& x : dgo_names_sorted) {
result += strip_dgo_extension(x) + "-";
}
result.pop_back();
return result;
} else {
return record.name;
}
}
const ObjectFileData& ObjectFileDB::lookup_record(const ObjectFileRecord& rec) const {
const ObjectFileData* result = nullptr;
for (auto& x : obj_files_by_name.at(rec.name)) {
if (x.record.version == rec.version) {
ASSERT(x.record.hash == rec.hash);
ASSERT(!result);
result = &x;
}
}
ASSERT(result);
return *result;
}
/*!
* Build an object file DB for the given list of DGOs.
*/
ObjectFileDB::ObjectFileDB(const std::vector<fs::path>& _dgos,
const fs::path& obj_file_name_map_file,
const std::vector<fs::path>& object_files,
const std::vector<fs::path>& str_files,
const std::vector<fs::path>& str_tex_files,
const std::vector<fs::path>& str_art_files,
const Config& config)
: dts(config.game_version), m_version(config.game_version) {
Timer timer;
lg::info("-Loading types...");
dts.parse_type_defs({config.all_types_file});
if (!obj_file_name_map_file.empty()) {
lg::info("-Loading obj name map file...");
load_map_file(
file_util::read_text_file(file_util::get_jak_project_dir() / obj_file_name_map_file));
} else {
lg::warn(
"Not using an obj name map file! The decompiler will automatically generate object file "
"names and write them to out/objs.txt. It is recommended to reuse this map file to get "
"consistent naming when doing a partial decompilation.");
}
lg::info("-Loading {} DGOs...", _dgos.size());
for (auto& dgo : _dgos) {
try {
get_objs_from_dgo(dgo, config);
} catch (std::runtime_error& e) {
lg::warn("Error when reading DGOs: {} on {}", e.what(), dgo.string());
}
}
lg::info("-Loading {} plain object files...", object_files.size());
for (auto& obj : object_files) {
auto data = file_util::read_binary_file(obj);
auto name = obj_filename_to_name(obj.string());
if (auto it = config.object_patches.find(name); it != config.object_patches.end()) {
// print the file CRC
lg::print("CRC for {} is: 0x{:X}\n", name, crc32(data.data(), data.size()));
// write patch file if necessary
if (config.write_patches) {
// this is the "target" file we want to patch to
auto data2 = file_util::read_binary_file(
(file_util::get_jak_project_dir() / it->second.target_file).string());
// we need to allocate a buffer to store the delta patch
// we make it 2x the source file's size because... it seems like a good size?
// ideally the delta patch should never be that big.
int buf_sz = data.size() * 2;
u8* out_buf = (u8*)malloc(buf_sz);
size_t out_sz = 0; // this is where the actual used size of the delta patch will be stored
int xd3_res = xd3_encode_memory(data2.data(), data2.size(), data.data(), data.size(),
out_buf, &out_sz, buf_sz, 0);
if (xd3_res) {
lg::error("error patching {} with {} (out {}): {}", name, it->second.target_file,
it->second.patch_file, xd3_strerror(xd3_res));
} else {
std::vector<u8> patch_data(out_sz);
memcpy(patch_data.data(), out_buf, patch_data.size());
file_util::write_binary_file(
(file_util::get_jak_project_dir() / it->second.patch_file).string(),
patch_data.data(), patch_data.size());
}
free(out_buf);
}
// apply patch file if necessary
// note that xd3 doesnt really have any safety against bad files so we check crc ourselves
if (config.apply_patches && it->second.crc == crc32(data.data(), data.size())) {
// this is the delta patch file
auto data2 = file_util::read_binary_file(
(file_util::get_jak_project_dir() / it->second.patch_file).string());
// we need to allocate a buffer to store the patched file
// the delta patch isn't gonna be this big but we allocate 2x the source file's size + the
// delta patch's size
int buf_sz = data.size() * 2 + data2.size();
u8* out_buf = (u8*)malloc(buf_sz);
size_t out_sz = 0;
int xd3_res = xd3_decode_memory(data2.data(), data2.size(), data.data(), data.size(),
out_buf, &out_sz, buf_sz, 0);
if (xd3_res) {
lg::error("error patching {} with {} (out {}): {}", name, it->second.target_file,
it->second.patch_file, xd3_strerror(xd3_res));
} else {
data.resize(out_sz);
memcpy(data.data(), out_buf, data.size());
}
free(out_buf);
}
}
add_obj_from_dgo(name, name, data.data(), data.size(), "NO-XGO", config);
}
if (config.read_spools) {
lg::info("-Loading {} streaming object files...", str_files.size());
for (auto& obj : str_files) {
StrFileReader reader(obj, version());
// name from the file name
std::string base_name = obj_filename_to_name(obj.string());
// name from inside the file (this does a lot of sanity checking)
auto obj_name = reader.get_full_name(base_name + ".STR");
for (int i = 0; i < reader.chunk_count(); i++) {
// append the chunk ID to the full name
std::string name = obj_name + fmt::format("+{}", i);
auto& data = reader.get_chunk(i);
add_obj_from_dgo(name, name, data.data(), data.size(), "ALLSPOOL", config, obj_name);
}
}
}
if (!str_tex_files.empty()) {
lg::info("-Loading {} streaming texture files...", str_tex_files.size());
for (auto& obj : str_tex_files) {
StrFileReader reader(obj, version());
// name from the file name
std::string base_name = obj_filename_to_name(obj.string());
for (int i = 0; i < reader.chunk_count(); i++) {
auto name = reader.get_chunk_texture_name(i);
add_obj_from_dgo(name, name, reader.get_chunk(i).data(), reader.get_chunk(i).size(),
"TEXSPOOL", config, name);
}
}
}
if (!str_art_files.empty()) {
lg::info("-Loading {} streaming art files...", str_art_files.size());
for (auto& obj : str_art_files) {
StrFileReader reader(obj, version());
for (int i = 0; i < reader.chunk_count(); i++) {
auto name = reader.get_chunk_art_name(i);
add_obj_from_dgo(name, name, reader.get_chunk(i).data(), reader.get_chunk(i).size(),
"ARTSPOOL", config, name);
}
}
}
lg::info("ObjectFileDB Initialized");
if (obj_files_by_name.empty()) {
lg::error(
"No object files have been added. Check that there are input files and the allowed_objects "
"list.");
}
dts.bad_format_strings = config.bad_format_strings;
dts.format_ops_with_dynamic_string_by_func_name =
config.hacks.format_ops_with_dynamic_string_by_func_name;
}
void ObjectFileDB::load_map_file(const std::string& map_data) {
auto j = parse_commented_json(map_data, "ObjectFileDB Map File");
for (auto& x : j) {
auto mapped_name = x[0].get<std::string>();
auto game_name = x[1].get<std::string>();
auto dgo_names = x[3].get<std::vector<std::string>>();
bool is_ag = mapped_name.find("-ag") != std::string::npos;
auto game_name_with_ag = game_name;
if (is_ag) {
game_name_with_ag += "-ag";
}
// add dgo
for (auto& dgo : dgo_names) {
auto kv = dgo_obj_name_map[dgo].find(game_name_with_ag);
if (kv != dgo_obj_name_map[dgo].end()) {
lg::error("Object {} in dgo {} occurs more than one time.", game_name_with_ag, dgo);
ASSERT(false);
}
dgo_obj_name_map[dgo][game_name_with_ag] = mapped_name;
}
}
}
/*!
* Load the objects stored in the given DGO into the ObjectFileDB
*/
void ObjectFileDB::get_objs_from_dgo(const fs::path& filename, const Config& config) {
auto dgo_data = file_util::read_binary_file(filename);
stats.total_dgo_bytes += dgo_data.size();
if (file_util::dgo_header_is_compressed(dgo_data)) {
dgo_data = file_util::decompress_dgo(dgo_data);
}
BinaryReader reader(dgo_data);
auto header = reader.read<DgoHeader>();
auto dgo_base_name = filename.filename().string();
ASSERT(header.name == dgo_base_name);
assert_string_empty_after(header.name, 60);
// get all obj files...
for (uint32_t i = 0; i < header.object_count; i++) {
auto obj_header = reader.read<DgoHeader>();
assert_string_empty_after(obj_header.name, 60);
if (i == header.object_count - 1) {
if (reader.bytes_left() == obj_header.object_count - 0x30) {
if (config.is_pal) {
lg::warn("Skipping {} in {} because it is a broken PAL object", obj_header.name,
dgo_base_name);
reader.ffwd(reader.bytes_left());
continue;
} else {
ASSERT(false);
}
}
} else {
ASSERT(reader.bytes_left() >= obj_header.object_count);
}
if (std::string(obj_header.name).find("-ag") != std::string::npos) {
lg::error(
"Object file {} has \"-ag\" in its name. This will break any tools which use this to "
"detect an art group",
obj_header.name);
ASSERT(false);
}
auto name = get_object_file_name(obj_header.name, reader.here(), obj_header.object_count);
add_obj_from_dgo(name, obj_header.name, reader.here(), obj_header.object_count, dgo_base_name,
config);
reader.ffwd(align16(obj_header.object_count));
}
// check we're at the end
ASSERT(0 == reader.bytes_left());
}
/*!
* Are two object files the same?
* Unfortunately they seemed to have a memory bug in their art-group generator, so there's some
* uninitialized padding bytes.
*/
bool are_objects_the_same(const std::string& obj_name,
size_t size_a,
const u8* a,
size_t size_b,
const u8* b) {
if (size_a != size_b) {
return false;
}
// if they are byte-for-byte the same, it's a match
if (!memcmp(a, b, size_a)) {
return true;
}
// if it's an art group...
if (obj_name.size() > 3 && !obj_name.compare(obj_name.length() - 3, 3, "-ag")) {
// count up the number of differing bytes, and the location of the first one.
size_t first_diff = 0;
size_t last_diff = 0;
int num_diffs = 0;
bool found_first_diff = false;
for (size_t i = 0; i < size_a; i++) {
if (a[i] != b[i]) {
num_diffs++;
last_diff = i;
if (!found_first_diff) {
first_diff = i;
found_first_diff = true;
}
}
}
// find the gap between "code" (really data here) and link table. This has up to 15 bytes of
// uninitialized memory.
const auto* header = (const LinkHeaderV4*)a;
int link_data_offset = header->code_size + sizeof(LinkHeaderV4);
int start_off_diff_from_code_end = link_data_offset - (int)first_diff;
if (num_diffs < 16 && start_off_diff_from_code_end < 16 && (int)last_diff < link_data_offset) {
return true;
}
}
return false;
}
/*!
* Add an object file to the ObjectFileDB
*/
void ObjectFileDB::add_obj_from_dgo(const std::string& obj_name,
const std::string& name_in_dgo,
const uint8_t* obj_data,
uint32_t obj_size,
const std::string& dgo_name,
const Config& config,
const std::string& cut_name) {
if (config.banned_objects.find(obj_name) != config.banned_objects.end()) {
return;
}
if (!config.allowed_objects.empty()) {
if (config.allowed_objects.find(obj_name) == config.allowed_objects.end()) {
return;
}
}
stats.total_obj_files++;
ASSERT(obj_size > 128);
uint16_t version = *(const uint16_t*)(obj_data + 8);
auto hash = crc32(obj_data, obj_size);
bool duplicated = false;
// first, check to see if we already got it...
for (auto& e : obj_files_by_name[obj_name]) {
if (are_objects_the_same(obj_name, e.data.size(), e.data.data(), obj_size, obj_data)) {
// already got it!
e.reference_count++;
auto& rec = e.record;
ASSERT(name_in_dgo == e.name_in_dgo);
e.dgo_names.push_back(dgo_name);
obj_files_by_dgo[dgo_name].push_back(rec);
duplicated = true;
break;
} else {
e.has_multiple_versions = true;
}
}
if (duplicated) {
return;
}
// nope, have to add a new one.
ObjectFileData data(config.game_version);
data.data.resize(obj_size);
memcpy(data.data.data(), obj_data, obj_size);
data.record.hash = hash;
data.record.name = obj_name;
data.dgo_names.push_back(dgo_name);
if (obj_files_by_name[obj_name].empty()) {
// if this is the first time we've seen this object file name, add it in the order.
obj_file_order.push_back(obj_name);
}
data.base_name_from_chunk = cut_name;
data.record.version = obj_files_by_name[obj_name].size();
data.name_in_dgo = name_in_dgo;
data.obj_version = version;
if (!dgo_obj_name_map.empty()) {
auto dgo_kv = dgo_obj_name_map.find(strip_dgo_extension(dgo_name));
if (dgo_kv == dgo_obj_name_map.end()) {
lg::error("Object {} is from DGO {}, but this DGO was not in the map.", obj_name, dgo_name);
ASSERT(false);
}
auto name_kv = dgo_kv->second.find(obj_name);
if (name_kv == dgo_kv->second.end()) {
lg::error("Object {} from DGO {} was not found in the name map.", obj_name, dgo_name);
ASSERT(false);
}
data.name_from_map = name_kv->second;
}
obj_files_by_dgo[dgo_name].push_back(data.record);
obj_files_by_name[obj_name].emplace_back(std::move(data));
stats.unique_obj_files++;
stats.unique_obj_bytes += obj_size;
if (obj_files_by_name[obj_name].size() > 1) {
for (auto& e : obj_files_by_name[obj_name]) {
e.has_multiple_versions = true;
}
}
}
/*!
* Generate a listing of what object files go in which dgos
*/
std::string ObjectFileDB::generate_dgo_listing() {
std::string result = ";; DGO File Listing\n\n";
std::vector<std::string> dgo_names;
for (auto& kv : obj_files_by_dgo) {
dgo_names.push_back(kv.first);
}
std::sort(dgo_names.begin(), dgo_names.end());
for (const auto& name : dgo_names) {
result += "(\"" + name + "\"\n";
for (auto& obj_rec : obj_files_by_dgo[name]) {
auto& obj = lookup_record(obj_rec);
std::string extension = ".o";
if (obj.obj_version == 4 || obj.obj_version == 2) {
extension = ".go";
}
result += " (\"" + obj.to_unique_name() + extension + "\" \"" + obj.name_in_dgo + "\")\n";
}
result += " )\n\n";
}
return result;
}
namespace {
std::string pad_string(const std::string& in, size_t length) {
if (in.length() < length) {
return in + std::string(length - in.length(), ' ');
} else {
return in;
}
}
} // namespace
std::string ObjectFileDB::generate_obj_listing(const std::unordered_set<std::string>& merged_objs) {
std::string result = "[";
std::set<std::string> all_unique_names;
int unique_count = 0;
for (auto& obj_file : obj_file_order) {
for (auto& x : obj_files_by_name.at(obj_file)) {
std::string dgos = "[";
for (auto& name : x.dgo_names) {
ASSERT(name.length() >= 5);
std::string new_str =
(name == "NO-XGO" || name == "ALLSPOOL") ? name : name.substr(0, name.length() - 4);
dgos += "\"" + new_str + "\", ";
}
dgos.pop_back();
dgos.pop_back();
dgos += "]";
auto name = x.to_unique_name();
result += "[\"" + pad_string(name + "\", ", 50) + "\"" +
pad_string(x.name_in_dgo + "\", ", 50) + std::to_string(x.obj_version) + ", " +
dgos + ", \"\"],\n";
if (all_unique_names.find(name) != all_unique_names.end() &&
merged_objs.find(name) == merged_objs.end()) {
lg::error("Object file {} appears multiple times with the same name.", name);
}
if (merged_objs.find(name) == merged_objs.end() ||
all_unique_names.find(name) == all_unique_names.end()) {
unique_count++;
}
all_unique_names.insert(name);
}
}
// this check is extremely important. It makes sure we don't have any repeat names. This could
// be caused by two files with the same name, in the same DGOs, but different data.
if (int(all_unique_names.size()) != unique_count) {
lg::error(
"Object files are not named properly, data will be lost! Got {} objs, but only {} names\n",
unique_count, all_unique_names.size());
}
if (unique_count > 0) {
result.pop_back(); // kill last new line
result.pop_back(); // kill last comma
}
return result + "]";
}
/*!
* Process all of the linking data of all objects.
*/
void ObjectFileDB::process_link_data(const Config& config) {
lg::info("Processing Link Data...");
Timer process_link_timer;
LinkedObjectFile::Stats combined_stats;
for_each_obj([&](ObjectFileData& obj) {
obj.linked_data = to_linked_object_file(obj.data, obj.record.name, dts, config.game_version);
combined_stats.add(obj.linked_data.stats);
});
lg::info("Processed Link Data");
lg::info(" Total {:.2f} ms", process_link_timer.getMs());
// printf("\n");
}
/*!
* Process all of the labels generated from linking and give them reasonable names.
*/
void ObjectFileDB::process_labels() {
lg::info("Processing Labels...");
Timer process_label_timer;
uint32_t total = 0;
for_each_obj([&](ObjectFileData& obj) { total += obj.linked_data.set_ordered_label_names(); });
lg::info("Processed Labels:");
lg::info(" Total {} labels", total);
lg::info(" Total {:.2f} ms", process_label_timer.getMs());
}
/*!
* Dump object files and their linking data to text files for debugging
*/
void ObjectFileDB::write_object_file_words(const fs::path& output_dir,
bool dump_data,
bool dump_code) {
lg::info("- Writing object file dumps (code? {} data? {})...", dump_code, dump_data);
Timer timer;
uint32_t total_bytes = 0, total_files = 0;
for_each_obj([&](ObjectFileData& obj) {
if ((obj.linked_data.segments == 3 && dump_code) ||
(obj.linked_data.segments != 3 && dump_data)) {
auto file_text = obj.linked_data.print_words();
auto file_name = output_dir / (obj.to_unique_name() + ".txt");
total_bytes += file_text.size();
file_util::write_text_file(file_name, file_text);
total_files++;
}
});
lg::info("Wrote object file dumps:");
lg::info(" Total {} files", total_files);
lg::info(" Total {:.3f} MB", total_bytes / ((float)(1u << 20u)));
lg::info(" Total {} ms ({:.3f} MB/sec)", timer.getMs(),
total_bytes / ((1u << 20u) * timer.getSeconds()));
}
/*!
* Dump disassembly for object files containing code. Data zones will also be dumped.
*/
void ObjectFileDB::write_disassembly(const fs::path& output_dir,
bool disassemble_data,
bool disassemble_code,
bool print_hex) {
lg::info("- Writing functions...");
Timer timer;
uint32_t total_bytes = 0, total_files = 0;
std::string asm_functions;
for_each_obj([&](ObjectFileData& obj) {
if (((obj.obj_version == 3 || (obj.obj_version == 5 && obj.linked_data.has_any_functions())) &&
disassemble_code) ||
(obj.obj_version != 3 && disassemble_data)) {
auto file_text = obj.linked_data.print_disassembly(print_hex);
asm_functions += obj.linked_data.print_asm_function_disassembly(obj.to_unique_name());
auto file_name = output_dir / (obj.to_unique_name() + ".asm");
total_bytes += file_text.size();
file_util::write_text_file(file_name, file_text);
total_files++;
}
});
total_bytes += asm_functions.size();
total_files++;
file_util::write_text_file(output_dir / "asm_functions.func", asm_functions);
lg::info("Wrote functions dumps:");
lg::info(" Total {} files", total_files);
lg::info(" Total {} MB", total_bytes / ((float)(1u << 20u)));
lg::info(" Total {} ms ({:.3f} MB/sec)", timer.getMs(),
total_bytes / ((1u << 20u) * timer.getSeconds()));
}
/*!
* Find code/data zones, identify functions, and disassemble
*/
void ObjectFileDB::find_code(const Config& config) {
lg::info("Finding code in object files...");
LinkedObjectFile::Stats combined_stats;
Timer timer;
for_each_obj([&](ObjectFileData& obj) {
// printf("fc %s\n", obj.record.to_unique_name().c_str());
obj.linked_data.find_code();
obj.linked_data.find_functions(config.game_version);
obj.linked_data.disassemble_functions();
if (config.game_version == GameVersion::Jak1 || obj.to_unique_name() != "effect-control-v0") {
obj.linked_data.process_fp_relative_links();
} else {
lg::warn("Skipping process_fp_relative_links in {}", obj.to_unique_name().c_str());
}
auto& obj_stats = obj.linked_data.stats;
if (obj_stats.code_bytes / 4 > obj_stats.decoded_ops) {
lg::warn("Failed to decode all in {} ({} / {})", obj.to_unique_name().c_str(),
obj_stats.decoded_ops, obj_stats.code_bytes / 4);
}
combined_stats.add(obj.linked_data.stats);
});
lg::info("Found code:");
lg::info(" Code {:.3f} MB", combined_stats.code_bytes / (float)(1 << 20));
lg::info(" Data {:.3f} MB", combined_stats.data_bytes / (float)(1 << 20));
lg::info(" Functions: {}", combined_stats.function_count);
lg::info(" fp uses resolved: {} / {} ({:.3f} %)", combined_stats.n_fp_reg_use_resolved,
combined_stats.n_fp_reg_use,
100.f * (float)combined_stats.n_fp_reg_use_resolved / combined_stats.n_fp_reg_use);
auto total_ops = combined_stats.code_bytes / 4;
lg::info(" Decoded {} / {} ({:.3f} %)", combined_stats.decoded_ops, total_ops,
100.f * (float)combined_stats.decoded_ops / total_ops);
lg::info(" Total {:.3f} ms", timer.getMs());
}
/*!
* Finds and writes all scripts into a file named all_scripts.lisp.
* Doesn't change any state in ObjectFileDB.
*/
void ObjectFileDB::find_and_write_scripts(const fs::path& output_dir) {
lg::info("- Finding scripts in object files...");
Timer timer;
std::string all_scripts;
for_each_obj([&](ObjectFileData& obj) {
auto scripts = obj.linked_data.print_scripts();
if (!scripts.empty()) {
all_scripts += ";--------------------------------------\n";
all_scripts += "; " + obj.to_unique_name() + "\n";
all_scripts += ";---------------------------------------\n";
all_scripts += scripts;
}
});
auto file_name = output_dir / "all_scripts.lisp";
file_util::write_text_file(file_name, all_scripts);
lg::info("Found scripts:");
lg::info(" Total {:.3f} ms", timer.getMs());
}
std::string ObjectFileDB::process_tpages(TextureDB& tex_db,
const fs::path& output_path,
const Config& cfg,
const fs::path& dump_out) {
lg::info("- Finding textures in tpages...");
std::string tpage_string = "tpage-";
int total = 0, success = 0;
int tpage_dir_count = 0;
u64 total_px = 0;
Timer timer;
std::vector<std::string> animated_slots;
switch (m_version) {
case GameVersion::Jak1: // no animated
break;
case GameVersion::Jak2:
animated_slots = jak2_animated_texture_slots();
break;
case GameVersion::Jak3:
animated_slots = jak3_animated_texture_slots();
break;
case GameVersion::JakX:
break;
default:
ASSERT_NOT_REACHED();
}
for (size_t i = 0; i < animated_slots.size(); i++) {
tex_db.animated_tex_output_to_anim_slot[animated_slots[i]] = i;
}
std::string result;
for_each_obj([&](ObjectFileData& data) {
if (data.name_in_dgo.substr(0, tpage_string.length()) == tpage_string) {
auto statistics =
process_tpage(data, tex_db, output_path, cfg.animated_textures, cfg.save_texture_pngs);
total += statistics.total_textures;
success += statistics.successful_textures;
total_px += statistics.num_px;
} else if (data.name_in_dgo == "dir-tpages") {
result = process_dir_tpages(data).to_source();
tpage_dir_count++;
}
});
ASSERT(tpage_dir_count <= 1);
lg::info("Processed {} / {} textures ({} px) {:.2f}% in {:.2f} ms", success, total, total_px,
100.f * float(success) / float(total), timer.getMs());
if (tpage_dir_count == 0) {
lg::warn("Did not find tpage-dir.");
return {};
}
if (cfg.write_tpage_imports) {
file_util::create_dir_if_needed(dump_out);
std::string tpage_dump;
std::string tex_dump;
for (auto& tpage : tex_db.tpage_names) {
tpage_dump += print_tpage_for_dump(tpage.second, tpage.first);
}
for (auto& tex : tex_db.textures) {
auto tpage_name = tex_db.tpage_names[tex.second.page];
dts.textures.emplace(tex.first, TexInfo{tex.second.name, tpage_name, tex.first & 0x0000ffff});
tex_dump += print_tex_for_dump(tex.second.name, tpage_name, tex.first & 0x0000ffff);
}
auto tpage_dump_out = dump_out / "tpages.gc";
auto tex_dump_out = dump_out / "textures.gc";
file_util::write_text_file(tpage_dump_out, tpage_dump);
file_util::write_text_file(tex_dump_out, tex_dump);
}
return result;
}
std::string ObjectFileDB::process_all_spool_subtitles(const Config& cfg,
const fs::path& image_out) {
try {
lg::info("- Finding spool subtitles...");
Timer timer;
int obj_count = 0;
int string_count = 0;
int image_count = 0;
int subs_count = 0;
std::unordered_map<std::string, std::vector<SpoolSubtitleRange>> all_subs;
for_each_obj_in_dgo("ALLSPOOL", [&](ObjectFileData& data) {
int this_string_count = 0;
int this_image_count = 0;
obj_count++;
auto this_spool_subs = process_spool_subtitles(data, cfg.text_version);
if (!this_spool_subs.empty()) {
for (auto& s : this_spool_subs) {
subs_count++;
for (int i = 0; i < 8; ++i) {
if (s.message[i].kind == SpoolSubtitleMessage::Kind::IMAGE) {
this_image_count++;
} else if (s.message[i].kind == SpoolSubtitleMessage::Kind::STRING) {
this_string_count++;
}
}
}
auto& spool_subs = all_subs[data.base_name_from_chunk];
for (auto& x : this_spool_subs) {
bool skip = false;
for (auto& other : spool_subs) {
skip |= other == x;
}
if (!skip) {
all_subs[data.base_name_from_chunk].push_back(x);
image_count += this_image_count;
string_count += this_string_count;
}
}
}
});
lg::info("Processed {} subtitles in {} spool objects ({} strings, {} images) in {:.2f} ms",
subs_count, obj_count, string_count, image_count, timer.getMs());
return write_spool_subtitles(cfg.text_version, image_out, all_subs);
} catch (std::runtime_error& e) {
lg::warn("Error when extracting spool subtitles: {}", e.what());
return {};
}
}
std::string ObjectFileDB::process_game_text_files(const Config& cfg, std::string text_string) {
try {
lg::info("- Finding game text...");
Timer timer;
int file_count = 0;
int string_count = 0;
int char_count = 0;
std::unordered_map<int, std::unordered_map<int, std::string>> text_by_language_by_id;
for_each_obj([&](ObjectFileData& data) {
if (data.name_in_dgo.ends_with(text_string)) {
file_count++;
auto statistics = process_game_text(data, cfg.text_version);
string_count += statistics.total_text;
char_count += statistics.total_chars;
if (text_by_language_by_id.find(statistics.language) != text_by_language_by_id.end()) {
ASSERT(false);
}
text_by_language_by_id[statistics.language] = std::move(statistics.text);
}
});
lg::info("Processed {} text files ({} strings, {} characters) in {:.2f} ms", file_count,
string_count, char_count, timer.getMs());
if (text_by_language_by_id.empty()) {
return {};
}
return write_game_text(cfg.text_version, text_by_language_by_id);
} catch (std::runtime_error& e) {
lg::warn("Error when extracting game text: {}", e.what());
return {};
}
}
std::string ObjectFileDB::process_game_count_file() {
lg::info("- Finding game count file...");
bool found = false;
std::string result;
for_each_obj([&](ObjectFileData& data) {
if (data.name_in_dgo == "game-cnt") {
ASSERT(!found);
found = true;
result = write_game_count(process_game_count(data));
}
});
if (!found) {
lg::warn("did not find game-cnt file");
}
return result;
}
namespace {
struct JointGeo {
u32 offset{};
std::string name;
u32 length{};
};
void get_joint_info(ObjectFileDB& db, ObjectFileData& obj, JointGeo jg) {
const auto& words = obj.linked_data.words_by_seg.at(MAIN_SEGMENT);
for (size_t i = 0; i < jg.length; ++i) {
u32 label = 0x0;
if (db.version() == GameVersion::Jak3 || db.version() == GameVersion::JakX) {
label = words.at((jg.offset / 4) + 11 + i).label_id();
} else {
label = words.at((jg.offset / 4) + 7 + i).label_id();
}
const auto& joint = obj.linked_data.labels.at(label);
const auto& name =
obj.linked_data.get_goal_string_by_label(words.at(joint.offset / 4).label_id());
// lg::print("{} joint idx {}/{}: {}\n", jg.name, i + 1, jg.length, name);
db.dts.add_joint_node(jg.name, name, i + 1);
}
}
void get_art_info(ObjectFileDB& db, ObjectFileData& obj) {
// jak 1/2
if (obj.obj_version == 4) {
const auto& words = obj.linked_data.words_by_seg.at(MAIN_SEGMENT);
if (words.at(0).kind() == LinkedWord::Kind::TYPE_PTR &&
words.at(0).symbol_name() == "art-group") {
auto obj_unique_name = obj.to_unique_name();
// lg::print("art-group {}:\n", obj.to_unique_name());
auto name = obj.linked_data.get_goal_string_by_label(words.at(2).label_id());
int length = words.at(3).data;
// lg::print(" length: {}\n", length);
for (int i = 0; i < length; ++i) {
const auto& word = words.at(8 + i);
if (word.kind() == LinkedWord::Kind::SYM_PTR && word.symbol_name() == "#f") {
continue;
}
const auto& label = obj.linked_data.labels.at(word.label_id());
auto elt_name =
obj.linked_data.get_goal_string_by_label(words.at(label.offset / 4 + 1).label_id());
auto unique_name = elt_name;
auto ag_name = obj_unique_name;
int elt_index = i;
auto& word_master_ag = words.at(label.offset / 4 + 7);
auto& word_master_idx = words.at(label.offset / 4 + 8);
if (word_master_ag.kind() == LinkedWord::Kind::PTR &&
word_master_idx.kind() == LinkedWord::Kind::PLAIN_DATA) {
ag_name = obj.linked_data.get_goal_string_by_label(word_master_ag.label_id()) + "-ag";
elt_index = word_master_idx.data;
}
std::string elt_type = words.at(label.offset / 4 - 1).symbol_name();
if (elt_type == "art-joint-geo") {
// the skeleton!
unique_name += "-jg";
JointGeo jg;
jg.offset = label.offset;
jg.name = unique_name;
jg.length = words.at(label.offset / 4 + 2).data;
get_joint_info(db, obj, jg);
} else if (elt_type == "merc-ctrl" || elt_type == "shadow-geo") {
// (maybe mesh-geo as well but that doesnt exist)
// the skin!
unique_name += "-mg";
} else if (elt_type == "art-joint-anim") {
// the animations!
unique_name += "-ja";
} else if (elt_type == "art-cloth-geo") {
// cloth geometry (jak 3)
unique_name += "-cg";
} else {
// the something idk!
throw std::runtime_error(
fmt::format("unknown art elt type {} in {}", elt_type, obj.to_unique_name()));
}
// lg::print(" {}: {} ({}) -> {} @ {}\n", i, elt_name, elt_type, unique_name, elt_index);
db.dts.add_art_group_elt(ag_name, unique_name, elt_index);
}
}
}
// jak 3
if (obj.obj_version == 5 && obj.linked_data.segments == 1) {
const auto& words = obj.linked_data.words_by_seg.at(MAIN_SEGMENT);
if (words.at(0).kind() == LinkedWord::Kind::TYPE_PTR &&
words.at(0).symbol_name() == "art-group") {
auto obj_unique_name = obj.to_unique_name();
// lg::print("art-group {}:\n", obj.to_unique_name());
auto name = obj.linked_data.get_goal_string_by_label(words.at(2).label_id());
int length = words.at(3).data;
// lg::print(" length: {}\n", length);
for (int i = 0; i < length; ++i) {
const auto& word = words.at(8 + i);
if (word.kind() == LinkedWord::Kind::SYM_PTR && word.symbol_name() == "#f") {
continue;
}
const auto& label = obj.linked_data.labels.at(word.label_id());
auto elt_name =
obj.linked_data.get_goal_string_by_label(words.at(label.offset / 4 + 1).label_id());
auto unique_name = elt_name;
auto ag_name = obj_unique_name;
int elt_index = i;
std::string elt_type = words.at(label.offset / 4 - 1).symbol_name();
auto& word_master_ag = words.at(label.offset / 4 + 4);
auto& word_master_idx = words.at(label.offset / 4 + 5);
if (word_master_ag.kind() == LinkedWord::Kind::PTR &&
word_master_idx.kind() == LinkedWord::Kind::PLAIN_DATA) {
ag_name = obj.linked_data.get_goal_string_by_label(word_master_ag.label_id()) + "-ag";
if (elt_type != "art-cloth-geo") {
elt_index = word_master_idx.data;
}
}
if (elt_type == "art-joint-geo") {
// the skeleton!
unique_name += "-jg";
JointGeo jg;
jg.offset = label.offset;
jg.name = unique_name;
jg.length = words.at(label.offset / 4 + 2).data;
get_joint_info(db, obj, jg);
} else if (elt_type == "merc-ctrl" || elt_type == "shadow-geo") {
// (maybe mesh-geo as well but that doesnt exist)
// the skin!
unique_name += "-mg";
} else if (elt_type == "art-joint-anim") {
// the animations!
unique_name += "-ja";
} else if (elt_type == "art-cloth-geo") {
// cloth geometry (jak 3)
unique_name += "-cg";
} else {
// the something idk!
throw std::runtime_error(
fmt::format("unknown art elt type {} in {}", elt_type, obj.to_unique_name()));
}
// lg::print(" {}: {} ({}) -> {} @ {}\n", i, elt_name, elt_type, unique_name, elt_index);
db.dts.add_art_group_elt(ag_name, unique_name, elt_index);
}
}
}
}
} // namespace
/*!
* Get information from art groups.
*/
void ObjectFileDB::extract_art_info() {
lg::info("Processing art groups...");
Timer timer;
for_each_obj([&](ObjectFileData& obj) {
try {
get_art_info(*this, obj);
} catch (std::runtime_error& e) {
lg::warn("Error when extracting art group info: {}", e.what());
}
});
lg::info("Processed art groups: in {:.2f} ms", timer.getMs());
}
/*!
* Write out the art group information.
*/
void ObjectFileDB::dump_art_info(const fs::path& output_dir) {
lg::info("Writing art group info...");
Timer timer;
if (!dts.art_group_info.empty() || !dts.jg_info.empty()) {
file_util::create_dir_if_needed(output_dir / "import");
}
auto ag_fpath = output_dir / "import" / "art-elts.gc";
std::string ag_result;
for (const auto& [ag_name, info] : dts.art_group_info) {
// auto ag_fname = ag_name + ".gc";
// auto filename = output_dir / "import" / ag_fname;
// std::string result = ";;-*-Lisp-*-\n";
// result += "(in-package goal)\n\n";
// result += fmt::format(";; {} - art group OpenGOAL import file\n", ag_fname);
// result += ";; THIS FILE IS AUTOMATICALLY GENERATED!\n\n";
for (const auto& [idx, elt_name] : info) {
ag_result += print_art_elt_for_dump(ag_name, elt_name, idx);
}
ag_result += "\n";
}
file_util::write_text_file(ag_fpath, ag_result);
auto jg_fpath = output_dir / "import" / "joint-nodes.gc";
std::string jg_result;
for (const auto& [jg_name, info] : dts.jg_info) {
for (const auto& [idx, joint] : info) {
jg_result += print_jg_for_dump(jg_name, joint, idx);
}
jg_result += "\n";
}
file_util::write_text_file(jg_fpath, jg_result);
lg::info("Written art group info: in {:.2f} ms", timer.getMs());
}
void ObjectFileDB::dump_part_group_table(
const fs::path& output_dir,
const std::unordered_map<u32, std::string>& part_group_table) {
lg::info("Writing part group table...");
Timer timer;
if (!part_group_table.empty()) {
file_util::create_dir_if_needed(output_dir / "import");
}
auto ptable_fpath = output_dir / "import" / "part-groups.gc";
std::string result;
for (const auto& [id, name] : part_group_table) {
result += fmt::format("(defconstant {} (-> *part-group-id-table* {}))", name, id);
result += "\n";
}
file_util::write_text_file(ptable_fpath, result);
auto ptable_dump_fpath = output_dir / "dump" / "part-groups.min.json";
nlohmann::json json = part_group_table;
file_util::create_dir_if_needed_for_file(ptable_dump_fpath);
file_util::write_text_file(ptable_dump_fpath, json.dump(-1));
lg::info("Written part group table in {:.2f} ms", timer.getMs());
}
void ObjectFileDB::dump_raw_objects(const fs::path& output_dir) {
for_each_obj([&](ObjectFileData& data) {
auto dest = output_dir / data.to_unique_name();
if (data.obj_version != 3) {
dest += ".go";
}
file_util::write_binary_file(dest, data.data.data(), data.data.size());
});
}
std::string print_art_elt_for_dump(const std::string& group_name,
const std::string& name,
int idx) {
return fmt::format("(def-art-elt {} {} {})\n", group_name, name, idx);
}
std::string print_jg_for_dump(const std::string& jg_name, const std::string& joint_name, int idx) {
return fmt::format("(def-joint-node {} \"{}\" {})\n", jg_name, joint_name, idx);
}
std::string print_tpage_for_dump(const std::string& debug_name, u32 id) {
return fmt::format("(defconstant {} {})\n", debug_name, id);
}
std::string print_tex_for_dump(const std::string& name, const std::string& page_name, u32 idx) {
return fmt::format("(def-tex {} {} {})\n", name, page_name, idx);
}
} // namespace decompiler
Reference in New Issue View Git Blame Copy Permalink