diff --git a/.gitmodules b/.gitmodules index 6716a1dddd..5de199ecb2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,3 +8,7 @@ [submodule "OTRExporter"] path = OTRExporter url = https://github.com/harbourmasters/OTRExporter +[submodule "soh/assets/yml"] + path = soh/assets/yml + url = https://github.com/briaguya0/soh-asset-yml.git + branch = code diff --git a/CMakeLists.txt b/CMakeLists.txt index 63b3bb9283..7a18428eab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -190,6 +190,32 @@ set(INCLUDE_MPQ_SUPPORT ON) ################################################################################ add_compile_definitions(CONTROLLERBUTTONS_T=uint32_t) +################################################################################ +# Torch (ROM asset extraction) +################################################################################ +find_package(ZLIB REQUIRED) + +include(FetchContent) + +set(USE_STANDALONE OFF CACHE BOOL "" FORCE) # static lib, no CLI +set(PORT_VERSION_ENDIANNESS ON CACHE BOOL "" FORCE) # 7-byte portVersion +set(ROM_CRC_BSWAP OFF CACHE BOOL "" FORCE) +set(BUILD_UI OFF CACHE BOOL "" FORCE) # would fetch a 2nd libultraship +set(BUILD_STORMLIB OFF CACHE BOOL "" FORCE) # target name clashes with LUS's `storm` +set(BUILD_NAUDIO OFF CACHE BOOL "" FORCE) + +set(BUILD_OOT ON CACHE BOOL "" FORCE) +foreach(_torch_game SM64 MK64 SF64 PM64 FZERO BK64 MARIO_ARTIST) + set(BUILD_${_torch_game} OFF CACHE BOOL "" FORCE) +endforeach() + +FetchContent_Declare( + torch + GIT_REPOSITORY https://github.com/HarbourMasters/Torch.git + GIT_TAG 4cae44160693e1beb562e39dc301bd42278be1f9 +) +FetchContent_MakeAvailable(torch) + ################################################################################ # Sub-projects ################################################################################ diff --git a/soh/CMakeLists.txt b/soh/CMakeLists.txt index 615fdeafe6..8d15b6f4ff 100644 --- a/soh/CMakeLists.txt +++ b/soh/CMakeLists.txt @@ -104,10 +104,6 @@ if (NOT TARGET libultraship) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../libultraship ${CMAKE_BINARY_DIR}/libultraship) endif() -if (NOT TARGET ZAPDLib) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../ZAPDTR/ZAPD ${CMAKE_BINARY_DIR}/ZAPD) -endif() - set(PROJECT_NAME soh) ################################################################################ @@ -322,7 +318,6 @@ target_include_directories(${PROJECT_NAME} PRIVATE assets ${CMAKE_CURRENT_SOURCE_DIR}/include/ ${CMAKE_CURRENT_SOURCE_DIR}/src/ ${CMAKE_CURRENT_SOURCE_DIR}/../libultraship/include - ${CMAKE_CURRENT_SOURCE_DIR}/../ZAPDTR/ZAPD/resource/type ${SDL2-INCLUDE} ${SDL2-NET-INCLUDE} ${CMAKE_CURRENT_SOURCE_DIR}/assets/ @@ -598,32 +593,22 @@ if (CMAKE_GENERATOR MATCHES "Visual Studio") add_custom_command( TARGET ${PROJECT_NAME} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_directory_if_different ${CMAKE_SOURCE_DIR}/soh/assets/extractor ${CMAKE_BINARY_DIR}/soh/assets - COMMAND ${CMAKE_COMMAND} -E copy_directory_if_different ${CMAKE_SOURCE_DIR}/soh/assets/xml ${CMAKE_BINARY_DIR}/soh/assets/xml - ) -endif() -if(NOT CMAKE_SYSTEM_NAME MATCHES "NintendoSwitch|CafeOS") - add_custom_command( - TARGET ${PROJECT_NAME} - POST_BUILD - COMMENT "Copying asset xmls..." - COMMAND ${CMAKE_COMMAND} -E copy_directory_if_different ${CMAKE_SOURCE_DIR}/soh/assets/extractor $/assets - COMMAND ${CMAKE_COMMAND} -E copy_directory_if_different ${CMAKE_SOURCE_DIR}/soh/assets/xml $/assets/xml - COMMAND ${CMAKE_COMMAND} -E make_directory $/assets/symbols - # COMMAND ${VS_COPY_ASSETS_CMD} + COMMAND ${CMAKE_COMMAND} -E copy_directory_if_different ${CMAKE_SOURCE_DIR}/soh/assets/yml ${CMAKE_BINARY_DIR}/soh/assets ) endif() +add_custom_command( + TARGET ${PROJECT_NAME} + POST_BUILD + COMMENT "Copying asset ymls..." + COMMAND ${CMAKE_COMMAND} -E copy_directory_if_different ${CMAKE_SOURCE_DIR}/soh/assets/yml $/assets +) ################################################################################ # Dependencies ################################################################################ add_dependencies(${PROJECT_NAME} libultraship + torch ) -if(NOT CMAKE_SYSTEM_NAME MATCHES "NintendoSwitch|CafeOS") -add_dependencies(${PROJECT_NAME} - ZAPDLib -) -endif() if (CMAKE_SYSTEM_NAME STREQUAL "Windows") find_package(glfw3 REQUIRED) @@ -639,7 +624,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Windows") if(SOH_WINDOWS_64BIT) set(ADDITIONAL_LIBRARY_DEPENDENCIES "libultraship;" - "ZAPDLib;" + "torch;" "glu32;" "SDL2::SDL2;" "SDL2::SDL2main;" @@ -659,7 +644,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Windows") elseif("${CMAKE_VS_PLATFORM_NAME}" STREQUAL "Win32") set(ADDITIONAL_LIBRARY_DEPENDENCIES "libultraship;" - "ZAPDLib;" + "torch;" "glu32;" "SDL2::SDL2;" "SDL2::SDL2main;" @@ -701,7 +686,7 @@ else() find_package(OpusFile REQUIRED) set(ADDITIONAL_LIBRARY_DEPENDENCIES "libultraship;" - "ZAPDLib;" + "torch;" SDL2::SDL2 "Ogg::ogg" "Vorbis::vorbis" diff --git a/soh/assets/yml b/soh/assets/yml new file mode 160000 index 0000000000..523be1de94 --- /dev/null +++ b/soh/assets/yml @@ -0,0 +1 @@ +Subproject commit 523be1de942b3810f3b53214264cb4503c525231 diff --git a/soh/soh/Extractor/Extract.cpp b/soh/soh/Extractor/Extract.cpp index 7819df6b1a..c0903a1837 100644 --- a/soh/soh/Extractor/Extract.cpp +++ b/soh/soh/Extractor/Extract.cpp @@ -5,7 +5,9 @@ #pragma comment(lib, "Shlwapi.lib") #endif #include "Extract.h" +#include "TorchExtract.h" #include "portable-file-dialogs.h" +#include "spdlog/spdlog.h" #include #include "soh/ShipUtils.h" #include "variables.h" @@ -581,36 +583,38 @@ bool Extractor::IsMasterQuest() const { } } -const char* Extractor::GetZapdVerStr() const { +// Version directories in the asset yml tree, matching the `path` torch resolves each ROM +// hash to in config.yml. +const char* Extractor::GetTorchVersionDir() const { switch (GetRomVerCrc()) { case OOT_PAL_GC: - return "GC_NMQ_PAL_F"; + return "pal_gc"; case OOT_PAL_MQ: - return "GC_MQ_PAL_F"; + return "pal_mq"; case OOT_PAL_GC_DBG1: - return "GC_NMQ_D"; + return "pal_gc_dbg"; case OOT_PAL_GC_MQ_DBG: - return "GC_MQ_D"; + return "pal_mq_dbg"; case OOT_PAL_10: - return "N64_PAL_10"; + return "pal_1-0"; case OOT_PAL_11: - return "N64_PAL_11"; + return "pal_1-1"; case OOT_NTSC_US_GC: - return "GC_NMQ_NTSC_U"; + return "ntsc_u_gc"; case OOT_NTSC_JP_GC: - return "GC_NMQ_NTSC_J"; + return "ntsc_j_gc"; case OOT_NTSC_JP_GC_CE: - return "GC_NMQ_NTSC_J_CE"; + return "ntsc_j_gc_collection"; case OOT_NTSC_US_MQ: - return "GC_MQ_NTSC_U"; + return "ntsc_u_mq"; case OOT_NTSC_JP_MQ: - return "GC_MQ_NTSC_J"; + return "ntsc_j_mq"; case OOT_NTSC_10: - return "N64_NTSC_10"; + return "ntsc_1-0"; case OOT_NTSC_11: - return "N64_NTSC_11"; + return "ntsc_1-1"; case OOT_NTSC_12: - return "N64_NTSC_12"; + return "ntsc_1-2"; default: // We should never be in a state where this path happens. UNREACHABLE; @@ -635,70 +639,38 @@ std::string Extractor::Mkdtemp() { return tmppath; } -extern "C" int zapd_report(int argc, char** argv, std::atomic* extractCount, std::atomic* totalExtract); static void MessageboxWorker(); -bool Extractor::CallZapd(std::string installPath, std::string exportdir, std::atomic* extractCount, - std::atomic* totalExtract) { - constexpr int argc = 22; - char xmlPath[1024]; - char confPath[1024]; +bool Extractor::CallTorch(std::string installPath, std::string exportdir, std::atomic* extractCount, + std::atomic* totalExtract) { char portVersion[18]; // 5 digits for int16_max (x3) + separators + terminator - std::array argv; - const char* version = GetZapdVerStr(); - const char* otrFile = IsMasterQuest() ? "oot-mq.o2r" : "oot.o2r"; + snprintf(portVersion, 18, "%d.%d.%d", gBuildVersionMajor, gBuildVersionMinor, gBuildVersionPatch); + const char* archiveName = IsMasterQuest() ? "oot-mq.o2r" : "oot.o2r"; std::string romPath = std::filesystem::absolute(mCurrentRomPath).string(); - installPath = std::filesystem::absolute(installPath).string(); + std::string srcDir = std::filesystem::absolute(installPath).string() + "/assets"; exportdir = std::filesystem::absolute(exportdir).string(); // Work this out in the temporary folder std::string tempdir = Mkdtemp(); - std::string curdir = std::filesystem::current_path().string(); -#ifdef _WIN32 - std::filesystem::copy(installPath + "/assets", tempdir + "/assets", - std::filesystem::copy_options::recursive | std::filesystem::copy_options::update_existing); -#else - std::filesystem::create_symlink(installPath + "/assets", tempdir + "/assets"); -#endif - std::filesystem::current_path(tempdir); + *totalExtract = SohTorch::CountAssetFiles(srcDir + "/" + GetTorchVersionDir()); + *extractCount = 0; - snprintf(xmlPath, 1024, "assets/xml/%s", version); - snprintf(confPath, 1024, "assets/Config_%s.xml", version); - snprintf(portVersion, 18, "%d.%d.%d", gBuildVersionMajor, gBuildVersionMinor, gBuildVersionPatch); + bool success = SohTorch::Extract(romPath, srcDir, tempdir, portVersion, archiveName, extractCount); - argv[0] = "ZAPD"; - argv[1] = "ed"; - argv[2] = "-i"; - argv[3] = xmlPath; - argv[4] = "-b"; - argv[5] = romPath.c_str(); - argv[6] = "-fl"; - argv[7] = "assets/filelists"; - argv[8] = "-gsf"; - argv[9] = "0"; - argv[10] = "-rconf"; - argv[11] = confPath; - argv[12] = "-se"; - argv[13] = "OTR"; - argv[14] = "--otrfile"; - argv[15] = otrFile; - argv[16] = "--portVer"; - argv[17] = portVersion; - argv[18] = "-o"; - argv[19] = "placeholder"; - argv[20] = "-osf"; - argv[21] = "placeholder"; + std::error_code ec; + if (success) { + std::filesystem::copy(tempdir + "/" + archiveName, exportdir + "/" + archiveName, + std::filesystem::copy_options::overwrite_existing, ec); + if (ec) { + SPDLOG_ERROR("Failed to copy {} to {}: {}", archiveName, exportdir, ec.message()); + success = false; + } + } - zapd_report(argc, (char**)argv.data(), extractCount, totalExtract); + std::filesystem::remove_all(tempdir, ec); - std::filesystem::copy(otrFile, exportdir + "/" + otrFile, std::filesystem::copy_options::overwrite_existing); - - // Go back to where this game was executed from - std::filesystem::current_path(curdir); - std::filesystem::remove_all(tempdir); - - return false; + return success; } static void MessageboxWorker() { diff --git a/soh/soh/Extractor/Extract.h b/soh/soh/Extractor/Extract.h index 552d5bf4eb..5ec3160864 100644 --- a/soh/soh/Extractor/Extract.h +++ b/soh/soh/Extractor/Extract.h @@ -41,7 +41,7 @@ class Extractor { bool ValidateRom(bool skipCrcBox = false); bool ValidateNotCompressed() const; - const char* GetZapdVerStr() const; + const char* GetTorchVersionDir() const; void SetRomInfo(const std::string& path); @@ -63,9 +63,8 @@ class Extractor { void GetRoms(std::vector& roms); bool RunFileStandalone(std::string file); bool Run(std::string searchPath, RomSearchMode searchMode = RomSearchMode::Both); - bool CallZapd(std::string installPath, std::string exportdir, std::atomic* extractCount, - std::atomic* totalExtract); - const char* GetZapdStr(); + bool CallTorch(std::string installPath, std::string exportdir, std::atomic* extractCount, + std::atomic* totalExtract); std::string Mkdtemp(); }; #endif diff --git a/soh/soh/Extractor/TorchExtract.cpp b/soh/soh/Extractor/TorchExtract.cpp new file mode 100644 index 0000000000..1640c3138b --- /dev/null +++ b/soh/soh/Extractor/TorchExtract.cpp @@ -0,0 +1,69 @@ +#include "TorchExtract.h" + +#include +#include +#include + +#include "spdlog/spdlog.h" + +#include "Companion.h" +#include "factories/BaseFactory.h" + +namespace fs = std::filesystem; + +namespace SohTorch { + +size_t CountAssetFiles(const std::string& ymlDir) { + std::error_code ec; + if (!fs::is_directory(ymlDir, ec)) { + return 0; + } + + size_t count = 0; + for (fs::recursive_directory_iterator it(ymlDir, ec), end; it != end; it.increment(ec)) { + if (ec) { + break; + } + if (it->is_regular_file(ec) && it->path().extension() == ".yml") { + count++; + } + } + return count; +} + +bool Extract(const std::string& romPath, const std::string& srcDir, const std::string& destDir, + const std::string& portVersion, const std::string& archiveName, std::atomic* progress) { + try { + // Companion::Instance is a raw global with no getter; factories dereference it. + auto companion = std::make_unique(fs::path(romPath), ArchiveType::O2R, false, srcDir, destDir); + Companion::Instance = companion.get(); + companion->SetVersion(portVersion); + companion->SetPhaseCallback([progress](int) { + if (progress != nullptr) { + (*progress)++; + } + }); + + // Init is the whole run; it calls Process() internally. + companion->Init(ExportType::Binary); + + // Companion holds every parsed asset, so don't leak it into the game's lifetime. + companion.reset(); + Companion::Instance = nullptr; + } catch (const std::exception& e) { + SPDLOG_ERROR("Torch extraction failed: {}", e.what()); + Companion::Instance = nullptr; + return false; + } catch (...) { + SPDLOG_ERROR("Torch extraction failed with an unknown exception"); + Companion::Instance = nullptr; + return false; + } + + // Process() returns void and several fatal paths only log and return, so confirm the + // archive exists rather than trusting the run. + std::error_code ec; + return fs::exists(fs::path(destDir) / archiveName, ec); +} + +} // namespace SohTorch diff --git a/soh/soh/Extractor/TorchExtract.h b/soh/soh/Extractor/TorchExtract.h new file mode 100644 index 0000000000..f942dfa9fa --- /dev/null +++ b/soh/soh/Extractor/TorchExtract.h @@ -0,0 +1,24 @@ +#ifndef TORCHEXTRACT_H +#define TORCHEXTRACT_H + +#include +#include +#include + +// No torch types here; TorchExtract.cpp is the only TU that includes Companion.h. +namespace SohTorch { + +// Count of .yml files under a version directory. Torch's phase callback fires once per file, +// so this is the progress denominator. +size_t CountAssetFiles(const std::string& ymlDir); + +// Extracts romPath into destDir/archiveName, where archiveName is what config.yml names the +// output for that ROM. Torch picks the version directory under srcDir by hashing the ROM. +// Increments progress once per asset file. False if it threw or produced no archive. +bool Extract(const std::string& romPath, const std::string& srcDir, const std::string& destDir, + const std::string& portVersion, const std::string& archiveName, + std::atomic* progress); + +} // namespace SohTorch + +#endif diff --git a/soh/soh/OTRGlobals.cpp b/soh/soh/OTRGlobals.cpp index 32b64fc504..5e824cbb3c 100644 --- a/soh/soh/OTRGlobals.cpp +++ b/soh/soh/OTRGlobals.cpp @@ -593,14 +593,14 @@ void OTRGlobals::RunExtract(int argc, char* argv[]) { std::string msg = "Archive for current ROM, " + archive + ", already exists.\nExtract again?"; SohGui::RegisterPopup("Confirm Re-extract", msg.c_str(), "Yes", "No", [&]() { extractionTask = threadPool->submit_task([&]() -> void { - extract.CallZapd(installPath, Ship::Context::GetAppDirectoryPath(appShortName), + extract.CallTorch(installPath, Ship::Context::GetAppDirectoryPath(appShortName), &extractCount, &totalExtract); extractCount = totalExtract = 0; }); }); } else { extractionTask = threadPool->submit_task([&]() -> void { - extract.CallZapd(installPath, Ship::Context::GetAppDirectoryPath(appShortName), + extract.CallTorch(installPath, Ship::Context::GetAppDirectoryPath(appShortName), &extractCount, &totalExtract); extractCount = totalExtract = 0; }); @@ -655,7 +655,7 @@ void OTRGlobals::RunExtract(int argc, char* argv[]) { continue; } extractionTask = threadPool->submit_task([&]() -> void { - extract.CallZapd(installPath, Ship::Context::GetAppDirectoryPath(appShortName), + extract.CallTorch(installPath, Ship::Context::GetAppDirectoryPath(appShortName), &extractCount, &totalExtract); generatedIsMQ = extract.IsMasterQuest(); promptStep = PS_SECOND; @@ -673,7 +673,7 @@ void OTRGlobals::RunExtract(int argc, char* argv[]) { extractStep = ES_VERIFY; } else { extractionTask = threadPool->submit_task([&]() -> void { - extract.CallZapd(installPath, Ship::Context::GetAppDirectoryPath(appShortName), + extract.CallTorch(installPath, Ship::Context::GetAppDirectoryPath(appShortName), &extractCount, &totalExtract); extractStep = ES_VERIFY; extractCount = 0;