diff --git a/common/util/read_iso_file.cpp b/common/util/read_iso_file.cpp index cf4312ffce..ed35889659 100644 --- a/common/util/read_iso_file.cpp +++ b/common/util/read_iso_file.cpp @@ -34,10 +34,18 @@ void IsoFile::Entry::print(std::string* result, const std::string& prefix) const namespace { constexpr int SECTOR_SIZE = 0x800; +int fseek_64(FILE* fp, u64 offset, int origin) { +#ifdef _WIN32 + return _fseeki64(fp, offset, origin); +#else + return fseek(fp, offset, origin); +#endif +} + template T read_file(FILE* fp, u32 sector, u32 offset_in_sector) { T result; - if (fseek(fp, sector * SECTOR_SIZE + offset_in_sector, SEEK_SET)) { + if (fseek_64(fp, u64(sector) * SECTOR_SIZE + offset_in_sector, SEEK_SET)) { ASSERT_MSG(false, "Failed to fseek iso"); } if (fread(&result, sizeof(T), 1, fp) != 1) { @@ -95,7 +103,7 @@ void unpack_entry(FILE* fp, lg::info("Extracting {}...", entry.name); } std::vector buffer(entry.size); - if (fseek(fp, entry.offset_in_file, SEEK_SET)) { + if (fseek_64(fp, entry.offset_in_file, SEEK_SET)) { ASSERT_MSG(false, "Failed to fseek iso when unpacking"); } if (fread(buffer.data(), buffer.size(), 1, fp) != 1) { diff --git a/decompiler/CMakeLists.txt b/decompiler/CMakeLists.txt index ac74081582..accb6d249a 100644 --- a/decompiler/CMakeLists.txt +++ b/decompiler/CMakeLists.txt @@ -111,7 +111,7 @@ target_link_libraries(decompiler add_executable(extractor - extractor/main.cpp extractor/extractor_util.cpp) + extractor/main.cpp) target_link_libraries(extractor decomp diff --git a/decompiler/extractor/extractor_util.cpp b/decompiler/extractor/extractor_util.cpp index fd565cc033..7a3fc785fb 100644 --- a/decompiler/extractor/extractor_util.cpp +++ b/decompiler/extractor/extractor_util.cpp @@ -15,9 +15,6 @@ #include "third-party/json.hpp" #include "third-party/zstd/lib/common/xxhash.h" -const std::unordered_map game_iso_flag_names = { - {"jak1-black-label", FLAG_JAK1_BLACK_LABEL}}; - const std::unordered_map game_iso_territory_map = { {GAME_TERRITORY_SCEA, "NTSC-U"}, {GAME_TERRITORY_SCEE, "PAL"}, @@ -30,47 +27,100 @@ std::string get_territory_name(int territory) { return game_iso_territory_map.at(territory); } -// used for - decompiler_out/ and iso_data/ -const std::unordered_map data_subfolders = {{"jak1", "jak1"}}; - const ISOMetadata jak1_ntsc_black_label_info = {"Jak & Daxter™: The Precursor Legacy (Black Label)", GAME_TERRITORY_SCEA, 337, - 11363853835861842434U, + {11363853835861842434U}, "ntsc_v1", "jak1", {"jak1-black-label"}}; // { SERIAL : { ELF_HASH : ISOMetadataDatabase } } -const std::unordered_map> iso_database = { - {"SCUS-97124", - {{7280758013604870207U, jak1_ntsc_black_label_info}, - {744661860962747854, - {"Jak & Daxter™: The Precursor Legacy", - GAME_TERRITORY_SCEA, - 338, - 8538304367812415885U, - "ntsc_v2", - "jak1", - {}}}}}, - {"SCES-50361", - {{12150718117852276522U, - {"Jak & Daxter™: The Precursor Legacy", - GAME_TERRITORY_SCEE, - 338, - 16850370297611763875U, - "pal", - "jak1", - {}}}}}, - {"SCPS-15021", - {{16909372048085114219U, - {"ジャックＸダクスター　～　旧世界の遺産", - GAME_TERRITORY_SCEI, - 338, - 1262350561338887717, - "jp", - "jak1", - {}}}}}}; +const std::unordered_map>& +extractor_iso_database() { + static const std::unordered_map> database = + { + {"SCUS-97124", + {{7280758013604870207U, jak1_ntsc_black_label_info}, + {744661860962747854, + {"Jak & Daxter™: The Precursor Legacy", + GAME_TERRITORY_SCEA, + 338, + {8538304367812415885U}, + "ntsc_v2", + "jak1", + {}}}}}, + {"SCES-50361", + {{12150718117852276522U, + {"Jak & Daxter™: The Precursor Legacy", + GAME_TERRITORY_SCEE, + 338, + {16850370297611763875U}, + "pal", + "jak1", + {}}}}}, + {"SCPS-15021", + {{16909372048085114219U, + {"ジャックＸダクスター　～　旧世界の遺産", + GAME_TERRITORY_SCEI, + 338, + {1262350561338887717U}, + "jp", + "jak1", + {}}}}}, + {"SCPS-56003", + {{7280758013604870207U, + {"Jak & Daxter: 구세계의 유산", + GAME_TERRITORY_SCEA, + 338, + {13924540661438229398U}, + "ntsc_v1", + "jak1", + {}}}}}, + // Jak 2, NTSC-U v1 and v2. + // we put both of them together because they have the same serial and ELF. + {"SCUS-97265", // serial from ELF name + {{18445016742498932084U, // hash of ELF + {"Jak II", // canonical name + GAME_TERRITORY_SCEA, + 593, // number of files + {4835330407820245819U, 5223305410190549348U}, // iso hash + "ntsc_v1", // decompiler config + "jak2", + {}}}}}, + // Jak 2 PAL + {"SCES-51608", // serial from ELF name + {{18188891052467821088U, // hash of ELF + {"Jak II: Renegade", // canonical name + GAME_TERRITORY_SCEE, + 593, // number of files + {8410801891219727031U}, // iso hash + "pal", // decompiler config + "jak2", + {}}}}}, + // Jak 2 NTSC-J + {"SCPS-15057", // serial from ELF name + {{7409991384254810731U, // hash of ELF + {"ジャックＸダクスター2", // canonical name + GAME_TERRITORY_SCEI, + 593, // number of files + {1686904681401593185U}, // iso hash + "jp", // decompiler config + "jak2", + {}}}}}, + // Jak 2 NTSC-K + {"SCKA-20010", // serial from ELF name + {{8398029689314218575U, // hash of ELF + {"Jak II", // canonical name + GAME_TERRITORY_SCEI, + 593, // number of files + {4637199624374114440U}, // iso hash + "ko", // decompiler config + "jak2", + {}}}}}, + }; + return database; +} void to_json(nlohmann::json& j, const BuildInfo& info) { j = nlohmann::json{{"serial", info.serial}, {"elf_hash", info.elf_hash}}; @@ -100,8 +150,8 @@ std::optional get_version_info_from_build_info(const BuildInfo& bui if (build_info.serial.empty() || build_info.elf_hash == 0) { return {}; } - auto dbEntry = iso_database.find(build_info.serial); - if (dbEntry == iso_database.end()) { + auto dbEntry = extractor_iso_database().find(build_info.serial); + if (dbEntry == extractor_iso_database().end()) { return {}; } @@ -168,10 +218,8 @@ void log_potential_new_db_entry(ExtractorErrorCode error_code, lg::info( "If this is a new release or version that should be supported, consider adding the " "following serial entry to the database:"); - lg::info( - "\t'{{\"{}\", {{{{{}U, {{\"GAME_TITLE\", \"NTSC-U/PAL/NTSC-J\", {}, {}U, " - "\"DECOMP_CONFIG_FILENAME_NO_EXTENSION\", \"jak1|jak2|jak3|jakx\", {}}}}}}}}}'", - serial, elf_hash, files_extracted, contents_hash); + lg::info("serial {}, elf hash {}, files {}, hash {}", serial, elf_hash, files_extracted, + contents_hash); } else if (error_code == ExtractorErrorCode::VALIDATION_ELF_MISSING_FROM_DB) { lg::info( "If this is a new release or version that should be supported, consider adding the " diff --git a/decompiler/extractor/extractor_util.h b/decompiler/extractor/extractor_util.h index acfaf69fc0..87cbdc83c2 100644 --- a/decompiler/extractor/extractor_util.h +++ b/decompiler/extractor/extractor_util.h @@ -1,19 +1,13 @@ #pragma once #include -#include +#include #include -#include "common/log/log.h" -#include "common/util/Assert.h" #include "common/util/FileUtil.h" -#include "common/util/json_util.h" #include "common/util/read_iso_file.h" -#include "game/kernel/common/kboot.h" - #include "third-party/json.hpp" -#include "third-party/zstd/lib/common/xxhash.h" enum class ExtractorErrorCode { SUCCESS = 0, @@ -33,30 +27,21 @@ enum class ExtractorErrorCode { enum GameIsoFlags { FLAG_JAK1_BLACK_LABEL = (1 << 0) }; -extern const std::unordered_map game_iso_flag_names; - -extern const std::unordered_map game_iso_territory_map; - -// used for - decompiler_out/ and iso_data/ -extern const std::unordered_map data_subfolders; - std::string get_territory_name(int territory); struct ISOMetadata { std::string canonical_name; int region; // territory code int num_files; - uint64_t contents_hash; + std::set contents_hash; std::string decomp_config_version; std::string game_name; std::vector flags; }; -extern const ISOMetadata jak1_ntsc_black_label_info; - // { SERIAL : { ELF_HASH : ISOMetadataDatabase } } -extern const std::unordered_map> - iso_database; +const std::unordered_map>& +extractor_iso_database(); // This is all we need to re-fetch info from the database // - if this changes such that we have a collision in the future, diff --git a/decompiler/extractor/main.cpp b/decompiler/extractor/main.cpp index 87eccd6e0d..cbee12c675 100644 --- a/decompiler/extractor/main.cpp +++ b/decompiler/extractor/main.cpp @@ -1,24 +1,22 @@ -#include -#include -#include - -#include "extractor_util.h" - #include "common/log/log.h" +#include "common/util/Assert.h" #include "common/util/FileUtil.h" -#include "common/util/json_util.h" #include "common/util/read_iso_file.h" #include "common/util/term_util.h" #include "common/util/unicode_util.h" -#include "decompiler/Disasm/OpcodeInfo.h" #include "decompiler/ObjectFile/ObjectFileDB.h" #include "decompiler/config.h" +#include "decompiler/extractor/extractor_util.h" #include "decompiler/level_extractor/extract_level.h" #include "goalc/compiler/Compiler.h" #include "third-party/CLI11.hpp" +// used for - decompiler_out/ and iso_data/ +const std::unordered_map data_subfolders = {{"jak1", "jak1"}, + {"jak2", "jak2"}}; + IsoFile extract_files(fs::path input_file_path, fs::path extracted_iso_path) { lg::info( "Note: Provided game data path '{}' points to a file, not a directory. Assuming it's an ISO " @@ -52,6 +50,7 @@ std::tuple, ExtractorErrorCode> validate( } // Find the game in our tracking database + const auto& iso_database = extractor_iso_database(); auto dbEntry = iso_database.find(serial.value()); if (dbEntry == iso_database.end()) { lg::error("Serial '{}' not found in the validation database", serial.value()); @@ -87,9 +86,13 @@ std::tuple, ExtractorErrorCode> validate( return {std::nullopt, ExtractorErrorCode::VALIDATION_INCORRECT_EXTRACTION_COUNT}; } // Check the ISO Hash - if (version_info.contents_hash != expected_hash) { - lg::error("Overall ISO content's hash does not match. Expected '{}', Actual '{}'", - version_info.contents_hash, expected_hash); + if (version_info.contents_hash.count(expected_hash) == 0) { + std::string all_expected; + for (const auto& hash : version_info.contents_hash) { + all_expected += fmt::format("{}, ", hash); + } + lg::error("Overall ISO content's hash does not match. Expected '{}', Actual '{}'", all_expected, + expected_hash); return {std::nullopt, ExtractorErrorCode::VALIDATION_FILE_CONTENTS_UNEXPECTED}; } @@ -193,6 +196,9 @@ void decompile(const fs::path& iso_data_path, const std::string& data_subfolder) } } +const std::unordered_map game_iso_flag_names = { + {"jak1-black-label", FLAG_JAK1_BLACK_LABEL}}; + ExtractorErrorCode compile(const fs::path& iso_data_path, const std::string& data_subfolder) { // Determine which config to use from the database const auto version_info = get_version_info_or_default(iso_data_path); @@ -226,8 +232,10 @@ ExtractorErrorCode compile(const fs::path& iso_data_path, const std::string& dat return ExtractorErrorCode::SUCCESS; } -void launch_game() { - system(fmt::format("\"{}\"", (file_util::get_jak_project_dir() / "../gk").string()).c_str()); +void launch_game(const std::string& game_version) { + system(fmt::format("\"{}\" -g {}", (file_util::get_jak_project_dir() / "../gk").string(), + game_version) + .c_str()); } int main(int argc, char** argv) { @@ -252,7 +260,7 @@ int main(int argc, char** argv) { ->required(); app.add_option("--proj-path", project_path_override, "Explicitly set the location of the 'data/' folder"); - app.add_flag("-g,--game", game_name, "Specify the game name, defaults to 'jak1'"); + app.add_option("-g,--game", game_name, "Specify the game name, defaults to 'jak1'"); app.add_flag("-a,--all", flag_runall, "Run all steps, from extraction to playing the game"); app.add_flag("-e,--extract", flag_extract, "Extract the ISO"); app.add_flag("-v,--validate", flag_fail_on_validation, @@ -413,7 +421,7 @@ int main(int argc, char** argv) { } if (flag_play) { - launch_game(); + launch_game(game_name); } return 0; diff --git a/lsp/protocol/completion.cpp b/lsp/protocol/completion.cpp index de58fae8ec..684b433e61 100644 --- a/lsp/protocol/completion.cpp +++ b/lsp/protocol/completion.cpp @@ -9,6 +9,6 @@ void LSPSpec::from_json(const json& j, CompletionParams& obj) { j.at("position").get_to(obj.m_position); } -void LSPSpec::to_json(json& j, const CompletionList& obj) {} +void LSPSpec::to_json(json& /*j*/, const CompletionList& /*obj*/) {} -void LSPSpec::from_json(const json& j, CompletionList& obj) {} +void LSPSpec::from_json(const json& /*j*/, CompletionList& /*obj*/) {}