mirror/Shipwright
1
0
Fork 0
mirror of https://github.com/HarbourMasters/Shipwright synced 2026-08-12 20:22:37 -04:00
Code Issues Packages Projects Releases Wiki Activity

Extract ROMs with Torch instead of ZAPD

Browse Source 
Adds soh/assets/yml as a submodule (briaguya0/soh-asset-yml, 20,353 asset
definitions) and FetchContents torch at 4cae4416, configured exactly as
Gate A' measured it: static lib, OoT only, no UI, no StormLib.

find_package(ZLIB) has to run before torch is declared. Torch declares
zlib with OVERRIDE_FIND_PACKAGE, so it takes over the whole build's
find_package(ZLIB), and its copy provides no ZLIB::ZLIB -- which both
StormLib and CMake's FindPNG link by name. Filed upstream as Torch#233.

CallZapd becomes CallTorch: no chdir, no symlinked assets dir, no 22-entry
argv. Everything that needed the working directory is now a parameter.
TorchExtract.cpp is the only TU that includes Companion.h, since torch
exports its whole lib/ as PUBLIC includes.

Progress uses torch's phase callback, which fires once per yml file, with
the denominator counted off disk. Gate A confirmed the two match exactly
for all three yml counts, so the bar runs 0-100 without the sawtooth the
old per-file counter would have produced.

GetZapdVerStr becomes GetTorchVersionDir, returning the version directory
under the yml tree rather than a ZAPD xml directory.

soh.elf links torch and no longer references zapd_report.
This commit is contained in:
briaguya
2026-07-25 01:53:34 -04:00
parent 9c29824f38
commit d1e6e5974d
9 changed files with 180 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitmodules
+4
View File
@@ -8,3 +8,7 @@
[submodule "OTRExporter"]
path = OTRExporter
url = https://github.com/harbourmasters/OTRExporter
[submodule "soh/assets/yml"]
path = soh/assets/yml
url = https://github.com/briaguya0/soh-asset-yml.git
branch = code
CMakeLists.txt
+26
View File
@@ -190,6 +190,32 @@ set(INCLUDE_MPQ_SUPPORT ON)
################################################################################
add_compile_definitions(CONTROLLERBUTTONS_T=uint32_t)
################################################################################
# Torch (ROM asset extraction)
################################################################################
find_package(ZLIB REQUIRED)
include(FetchContent)
set(USE_STANDALONE OFF CACHE BOOL "" FORCE) # static lib, no CLI
set(PORT_VERSION_ENDIANNESS ON CACHE BOOL "" FORCE) # 7-byte portVersion
set(ROM_CRC_BSWAP OFF CACHE BOOL "" FORCE)
set(BUILD_UI OFF CACHE BOOL "" FORCE) # would fetch a 2nd libultraship
set(BUILD_STORMLIB OFF CACHE BOOL "" FORCE) # target name clashes with LUS's `storm`
set(BUILD_NAUDIO OFF CACHE BOOL "" FORCE)
set(BUILD_OOT ON CACHE BOOL "" FORCE)
foreach(_torch_game SM64 MK64 SF64 PM64 FZERO BK64 MARIO_ARTIST)
set(BUILD_${_torch_game} OFF CACHE BOOL "" FORCE)
endforeach()
FetchContent_Declare(
torch
GIT_REPOSITORY https://github.com/HarbourMasters/Torch.git
GIT_TAG 4cae44160693e1beb562e39dc301bd42278be1f9
)
FetchContent_MakeAvailable(torch)
################################################################################
# Sub-projects
################################################################################
soh/CMakeLists.txt
+11 -26
View File
@@ -104,10 +104,6 @@ if (NOT TARGET libultraship)
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../libultraship ${CMAKE_BINARY_DIR}/libultraship)
endif()
if (NOT TARGET ZAPDLib)
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../ZAPDTR/ZAPD ${CMAKE_BINARY_DIR}/ZAPD)
endif()
set(PROJECT_NAME soh)
################################################################################
@@ -322,7 +318,6 @@ target_include_directories(${PROJECT_NAME} PRIVATE assets
${CMAKE_CURRENT_SOURCE_DIR}/include/
${CMAKE_CURRENT_SOURCE_DIR}/src/
${CMAKE_CURRENT_SOURCE_DIR}/../libultraship/include
${CMAKE_CURRENT_SOURCE_DIR}/../ZAPDTR/ZAPD/resource/type
${SDL2-INCLUDE}
${SDL2-NET-INCLUDE}
${CMAKE_CURRENT_SOURCE_DIR}/assets/
@@ -598,32 +593,22 @@ if (CMAKE_GENERATOR MATCHES "Visual Studio")
add_custom_command(
TARGET ${PROJECT_NAME}
POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_directory_if_different ${CMAKE_SOURCE_DIR}/soh/assets/extractor ${CMAKE_BINARY_DIR}/soh/assets
COMMAND ${CMAKE_COMMAND} -E copy_directory_if_different ${CMAKE_SOURCE_DIR}/soh/assets/xml ${CMAKE_BINARY_DIR}/soh/assets/xml
)
endif()
if(NOT CMAKE_SYSTEM_NAME MATCHES "NintendoSwitch|CafeOS")
add_custom_command(
TARGET ${PROJECT_NAME}
POST_BUILD
COMMENT "Copying asset xmls..."
COMMAND ${CMAKE_COMMAND} -E copy_directory_if_different ${CMAKE_SOURCE_DIR}/soh/assets/extractor $<TARGET_FILE_DIR:soh>/assets
COMMAND ${CMAKE_COMMAND} -E copy_directory_if_different ${CMAKE_SOURCE_DIR}/soh/assets/xml $<TARGET_FILE_DIR:soh>/assets/xml
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:soh>/assets/symbols
# COMMAND ${VS_COPY_ASSETS_CMD}
COMMAND ${CMAKE_COMMAND} -E copy_directory_if_different ${CMAKE_SOURCE_DIR}/soh/assets/yml ${CMAKE_BINARY_DIR}/soh/assets
)
endif()
add_custom_command(
TARGET ${PROJECT_NAME}
POST_BUILD
COMMENT "Copying asset ymls..."
COMMAND ${CMAKE_COMMAND} -E copy_directory_if_different ${CMAKE_SOURCE_DIR}/soh/assets/yml $<TARGET_FILE_DIR:soh>/assets
)
################################################################################
# Dependencies
################################################################################
add_dependencies(${PROJECT_NAME}
libultraship
torch
)
if(NOT CMAKE_SYSTEM_NAME MATCHES "NintendoSwitch|CafeOS")
add_dependencies(${PROJECT_NAME}
ZAPDLib
)
endif()
if (CMAKE_SYSTEM_NAME STREQUAL "Windows")
find_package(glfw3 REQUIRED)
@@ -639,7 +624,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Windows")
if(SOH_WINDOWS_64BIT)
set(ADDITIONAL_LIBRARY_DEPENDENCIES
"libultraship;"
"ZAPDLib;"
"torch;"
"glu32;"
"SDL2::SDL2;"
"SDL2::SDL2main;"
@@ -659,7 +644,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Windows")
elseif("${CMAKE_VS_PLATFORM_NAME}" STREQUAL "Win32")
set(ADDITIONAL_LIBRARY_DEPENDENCIES
"libultraship;"
"ZAPDLib;"
"torch;"
"glu32;"
"SDL2::SDL2;"
"SDL2::SDL2main;"
@@ -701,7 +686,7 @@ else()
find_package(OpusFile REQUIRED)
set(ADDITIONAL_LIBRARY_DEPENDENCIES
"libultraship;"
"ZAPDLib;"
"torch;"
SDL2::SDL2
"Ogg::ogg"
"Vorbis::vorbis"
soh/assets/yml
Submodule
+1
Submodule soh/assets/yml added at 523be1de94
soh/soh/Extractor/Extract.cpp
+38 -66
View File
@@ -5,7 +5,9 @@
#pragma comment(lib, "Shlwapi.lib")
#endif
#include "Extract.h"
#include "TorchExtract.h"
#include "portable-file-dialogs.h"
#include "spdlog/spdlog.h"
#include <ship/utils/binarytools/BitConverter.h>
#include "soh/ShipUtils.h"
#include "variables.h"
@@ -581,36 +583,38 @@ bool Extractor::IsMasterQuest() const {
}
}
const char* Extractor::GetZapdVerStr() const {
// Version directories in the asset yml tree, matching the `path` torch resolves each ROM
// hash to in config.yml.
const char* Extractor::GetTorchVersionDir() const {
switch (GetRomVerCrc()) {
case OOT_PAL_GC:
return "GC_NMQ_PAL_F";
return "pal_gc";
case OOT_PAL_MQ:
return "GC_MQ_PAL_F";
return "pal_mq";
case OOT_PAL_GC_DBG1:
return "GC_NMQ_D";
return "pal_gc_dbg";
case OOT_PAL_GC_MQ_DBG:
return "GC_MQ_D";
return "pal_mq_dbg";
case OOT_PAL_10:
return "N64_PAL_10";
return "pal_1-0";
case OOT_PAL_11:
return "N64_PAL_11";
return "pal_1-1";
case OOT_NTSC_US_GC:
return "GC_NMQ_NTSC_U";
return "ntsc_u_gc";
case OOT_NTSC_JP_GC:
return "GC_NMQ_NTSC_J";
return "ntsc_j_gc";
case OOT_NTSC_JP_GC_CE:
return "GC_NMQ_NTSC_J_CE";
return "ntsc_j_gc_collection";
case OOT_NTSC_US_MQ:
return "GC_MQ_NTSC_U";
return "ntsc_u_mq";
case OOT_NTSC_JP_MQ:
return "GC_MQ_NTSC_J";
return "ntsc_j_mq";
case OOT_NTSC_10:
return "N64_NTSC_10";
return "ntsc_1-0";
case OOT_NTSC_11:
return "N64_NTSC_11";
return "ntsc_1-1";
case OOT_NTSC_12:
return "N64_NTSC_12";
return "ntsc_1-2";
default:
// We should never be in a state where this path happens.
UNREACHABLE;
@@ -635,70 +639,38 @@ std::string Extractor::Mkdtemp() {
return tmppath;
}
extern "C" int zapd_report(int argc, char** argv, std::atomic<size_t>* extractCount, std::atomic<size_t>* totalExtract);
static void MessageboxWorker();
bool Extractor::CallZapd(std::string installPath, std::string exportdir, std::atomic<size_t>* extractCount,
std::atomic<size_t>* totalExtract) {
constexpr int argc = 22;
char xmlPath[1024];
char confPath[1024];
bool Extractor::CallTorch(std::string installPath, std::string exportdir, std::atomic<size_t>* extractCount,
std::atomic<size_t>* totalExtract) {
char portVersion[18]; // 5 digits for int16_max (x3) + separators + terminator
std::array<const char*, argc> argv;
const char* version = GetZapdVerStr();
const char* otrFile = IsMasterQuest() ? "oot-mq.o2r" : "oot.o2r";
snprintf(portVersion, 18, "%d.%d.%d", gBuildVersionMajor, gBuildVersionMinor, gBuildVersionPatch);
const char* archiveName = IsMasterQuest() ? "oot-mq.o2r" : "oot.o2r";
std::string romPath = std::filesystem::absolute(mCurrentRomPath).string();
installPath = std::filesystem::absolute(installPath).string();
std::string srcDir = std::filesystem::absolute(installPath).string() + "/assets";
exportdir = std::filesystem::absolute(exportdir).string();
// Work this out in the temporary folder
std::string tempdir = Mkdtemp();
std::string curdir = std::filesystem::current_path().string();
#ifdef _WIN32
std::filesystem::copy(installPath + "/assets", tempdir + "/assets",
std::filesystem::copy_options::recursive | std::filesystem::copy_options::update_existing);
#else
std::filesystem::create_symlink(installPath + "/assets", tempdir + "/assets");
#endif
std::filesystem::current_path(tempdir);
*totalExtract = SohTorch::CountAssetFiles(srcDir + "/" + GetTorchVersionDir());
*extractCount = 0;
snprintf(xmlPath, 1024, "assets/xml/%s", version);
snprintf(confPath, 1024, "assets/Config_%s.xml", version);
snprintf(portVersion, 18, "%d.%d.%d", gBuildVersionMajor, gBuildVersionMinor, gBuildVersionPatch);
bool success = SohTorch::Extract(romPath, srcDir, tempdir, portVersion, archiveName, extractCount);
argv[0] = "ZAPD";
argv[1] = "ed";
argv[2] = "-i";
argv[3] = xmlPath;
argv[4] = "-b";
argv[5] = romPath.c_str();
argv[6] = "-fl";
argv[7] = "assets/filelists";
argv[8] = "-gsf";
argv[9] = "0";
argv[10] = "-rconf";
argv[11] = confPath;
argv[12] = "-se";
argv[13] = "OTR";
argv[14] = "--otrfile";
argv[15] = otrFile;
argv[16] = "--portVer";
argv[17] = portVersion;
argv[18] = "-o";
argv[19] = "placeholder";
argv[20] = "-osf";
argv[21] = "placeholder";
std::error_code ec;
if (success) {
std::filesystem::copy(tempdir + "/" + archiveName, exportdir + "/" + archiveName,
std::filesystem::copy_options::overwrite_existing, ec);
if (ec) {
SPDLOG_ERROR("Failed to copy {} to {}: {}", archiveName, exportdir, ec.message());
success = false;
}
}
zapd_report(argc, (char**)argv.data(), extractCount, totalExtract);
std::filesystem::remove_all(tempdir, ec);
std::filesystem::copy(otrFile, exportdir + "/" + otrFile, std::filesystem::copy_options::overwrite_existing);
// Go back to where this game was executed from
std::filesystem::current_path(curdir);
std::filesystem::remove_all(tempdir);
return false;
return success;
}
static void MessageboxWorker() {
soh/soh/Extractor/Extract.h
+3 -4
View File
@@ -41,7 +41,7 @@ class Extractor {
bool ValidateRom(bool skipCrcBox = false);
bool ValidateNotCompressed() const;
const char* GetZapdVerStr() const;
const char* GetTorchVersionDir() const;
void SetRomInfo(const std::string& path);
@@ -63,9 +63,8 @@ class Extractor {
void GetRoms(std::vector<std::string>& roms);
bool RunFileStandalone(std::string file);
bool Run(std::string searchPath, RomSearchMode searchMode = RomSearchMode::Both);
bool CallZapd(std::string installPath, std::string exportdir, std::atomic<size_t>* extractCount,
std::atomic<size_t>* totalExtract);
const char* GetZapdStr();
bool CallTorch(std::string installPath, std::string exportdir, std::atomic<size_t>* extractCount,
std::atomic<size_t>* totalExtract);
std::string Mkdtemp();
};
#endif
soh/soh/Extractor/TorchExtract.cpp
+69
View File
@@ -0,0 +1,69 @@
#include "TorchExtract.h"
#include <exception>
#include <filesystem>
#include <memory>
#include "spdlog/spdlog.h"
#include "Companion.h"
#include "factories/BaseFactory.h"
namespace fs = std::filesystem;
namespace SohTorch {
size_t CountAssetFiles(const std::string& ymlDir) {
std::error_code ec;
if (!fs::is_directory(ymlDir, ec)) {
return 0;
}
size_t count = 0;
for (fs::recursive_directory_iterator it(ymlDir, ec), end; it != end; it.increment(ec)) {
if (ec) {
break;
}
if (it->is_regular_file(ec) && it->path().extension() == ".yml") {
count++;
}
}
return count;
}
bool Extract(const std::string& romPath, const std::string& srcDir, const std::string& destDir,
const std::string& portVersion, const std::string& archiveName, std::atomic<size_t>* progress) {
try {
// Companion::Instance is a raw global with no getter; factories dereference it.
auto companion = std::make_unique<Companion>(fs::path(romPath), ArchiveType::O2R, false, srcDir, destDir);
Companion::Instance = companion.get();
companion->SetVersion(portVersion);
companion->SetPhaseCallback([progress](int) {
if (progress != nullptr) {
(*progress)++;
}
});
// Init is the whole run; it calls Process() internally.
companion->Init(ExportType::Binary);
// Companion holds every parsed asset, so don't leak it into the game's lifetime.
companion.reset();
Companion::Instance = nullptr;
} catch (const std::exception& e) {
SPDLOG_ERROR("Torch extraction failed: {}", e.what());
Companion::Instance = nullptr;
return false;
} catch (...) {
SPDLOG_ERROR("Torch extraction failed with an unknown exception");
Companion::Instance = nullptr;
return false;
}
// Process() returns void and several fatal paths only log and return, so confirm the
// archive exists rather than trusting the run.
std::error_code ec;
return fs::exists(fs::path(destDir) / archiveName, ec);
}
} // namespace SohTorch
soh/soh/Extractor/TorchExtract.h
+24
View File
@@ -0,0 +1,24 @@
#ifndef TORCHEXTRACT_H
#define TORCHEXTRACT_H
#include <atomic>
#include <cstddef>
#include <string>
// No torch types here; TorchExtract.cpp is the only TU that includes Companion.h.
namespace SohTorch {
// Count of .yml files under a version directory. Torch's phase callback fires once per file,
// so this is the progress denominator.
size_t CountAssetFiles(const std::string& ymlDir);
// Extracts romPath into destDir/archiveName, where archiveName is what config.yml names the
// output for that ROM. Torch picks the version directory under srcDir by hashing the ROM.
// Increments progress once per asset file. False if it threw or produced no archive.
bool Extract(const std::string& romPath, const std::string& srcDir, const std::string& destDir,
const std::string& portVersion, const std::string& archiveName,
std::atomic<size_t>* progress);
} // namespace SohTorch
#endif
soh/soh/OTRGlobals.cpp
+4 -4
View File
@@ -593,14 +593,14 @@ void OTRGlobals::RunExtract(int argc, char* argv[]) {
std::string msg = "Archive for current ROM, " + archive + ", already exists.\nExtract again?";
SohGui::RegisterPopup("Confirm Re-extract", msg.c_str(), "Yes", "No", [&]() {
extractionTask = threadPool->submit_task([&]() -> void {
extract.CallZapd(installPath, Ship::Context::GetAppDirectoryPath(appShortName),
extract.CallTorch(installPath, Ship::Context::GetAppDirectoryPath(appShortName),
&extractCount, &totalExtract);
extractCount = totalExtract = 0;
});
});
} else {
extractionTask = threadPool->submit_task([&]() -> void {
extract.CallZapd(installPath, Ship::Context::GetAppDirectoryPath(appShortName),
extract.CallTorch(installPath, Ship::Context::GetAppDirectoryPath(appShortName),
&extractCount, &totalExtract);
extractCount = totalExtract = 0;
});
@@ -655,7 +655,7 @@ void OTRGlobals::RunExtract(int argc, char* argv[]) {
continue;
}
extractionTask = threadPool->submit_task([&]() -> void {
extract.CallZapd(installPath, Ship::Context::GetAppDirectoryPath(appShortName),
extract.CallTorch(installPath, Ship::Context::GetAppDirectoryPath(appShortName),
&extractCount, &totalExtract);
generatedIsMQ = extract.IsMasterQuest();
promptStep = PS_SECOND;
@@ -673,7 +673,7 @@ void OTRGlobals::RunExtract(int argc, char* argv[]) {
extractStep = ES_VERIFY;
} else {
extractionTask = threadPool->submit_task([&]() -> void {
extract.CallZapd(installPath, Ship::Context::GetAppDirectoryPath(appShortName),
extract.CallTorch(installPath, Ship::Context::GetAppDirectoryPath(appShortName),
&extractCount, &totalExtract);
extractStep = ES_VERIFY;
extractCount = 0;