diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c30e019fa..0b49056827 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -246,6 +246,12 @@ add_executable(soh-torch target_include_directories(soh-torch PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/soh/soh/Extractor) target_link_libraries(soh-torch PRIVATE torch) +add_executable(soh-o2r-packer + ${CMAKE_CURRENT_SOURCE_DIR}/tools/soh-o2r-packer/main.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tools/soh-o2r-packer/PngTexture.cpp +) +target_link_libraries(soh-o2r-packer PRIVATE torch) + # Target to generate OTRs. SOH_ROM_PATH takes roms and/or directories of roms; torch names each # archive (oot.o2r or oot-mq.o2r) from its hash, so a vanilla and a master quest rom produce both # in one run. soh.o2r comes from GenerateSohOtr, chained below. @@ -275,50 +281,20 @@ add_custom_target( COMMAND ${CMAKE_COMMAND} -E false ) -# TEMPORARY (Torch migration): soh.o2r is copied from prebuilt/, not generated. -# -# The real producer drives ZAPD via OTRExporter/extract_assets.py, and the migration deletes -# both. Its replacement -- a small in-tree packer, Phase 4 of torch-migration/PLAN.md -- has to -# reproduce ZAPD's texture quantisation exactly, so rather than block Phases 2/3/5 on it, the -# archive is checked in and copied into place. See prebuilt/README.md for provenance and for how -# to restore a real build step (which must also bring back the libultraship shader copy that the -# copy-only version below has no use for -- the shaders are already inside the archive). -# -# Fail at configure time if the prebuilt archive's baked-in portVersion no longer matches the -# project version. OTRGlobals.cpp:283 demands exact major.minor.patch equality and RunExtract -# exit(1)s with "soh.o2r is outdated" -- an obscure symptom for what is really a stale checked-in -# file. portVersion is 7 bytes: endianness, then u16 major/minor/patch big-endian. -set(SOH_PREBUILT_O2R "${CMAKE_CURRENT_SOURCE_DIR}/prebuilt/soh.o2r") -if(NOT EXISTS "${SOH_PREBUILT_O2R}") - message(FATAL_ERROR "Missing ${SOH_PREBUILT_O2R} -- see prebuilt/README.md") -endif() -file(READ "${SOH_PREBUILT_O2R}" _soh_o2r_hex HEX) -string(FIND "${_soh_o2r_hex}" "706f727456657273696f6e" _pv_at) # "portVersion" as hex -if(_pv_at EQUAL -1) - message(FATAL_ERROR "No portVersion entry in ${SOH_PREBUILT_O2R} -- see prebuilt/README.md") -endif() -math(EXPR _pv_at "${_pv_at} + 22") # skip the 11-character name -string(SUBSTRING "${_soh_o2r_hex}" ${_pv_at} 14 _pv) -string(SUBSTRING "${_pv}" 2 4 _pv_major) -string(SUBSTRING "${_pv}" 6 4 _pv_minor) -string(SUBSTRING "${_pv}" 10 4 _pv_patch) -math(EXPR _pv_major "0x${_pv_major}") -math(EXPR _pv_minor "0x${_pv_minor}") -math(EXPR _pv_patch "0x${_pv_patch}") -if(NOT "${_pv_major}.${_pv_minor}.${_pv_patch}" STREQUAL "${CMAKE_PROJECT_VERSION}") - message(FATAL_ERROR - "prebuilt/soh.o2r is stamped ${_pv_major}.${_pv_minor}.${_pv_patch} but this build is " - "${CMAKE_PROJECT_VERSION}. The game would reject it at startup (\"soh.o2r is outdated\"). " - "Regenerate it -- see prebuilt/README.md.") -endif() -unset(_soh_o2r_hex) - -# Target to place soh.o2r (copy of the prebuilt archive) +# Target to generate only soh.o2r add_custom_target( GenerateSohOtr - COMMAND ${CMAKE_COMMAND} -E copy_if_different "${SOH_PREBUILT_O2R}" ${CMAKE_CURRENT_SOURCE_DIR}/soh.o2r - COMMAND ${CMAKE_COMMAND} -E copy_if_different "${SOH_PREBUILT_O2R}" ${CMAKE_BINARY_DIR}/soh/soh.o2r - COMMENT "Copying prebuilt soh.o2r (TEMPORARY -- see prebuilt/README.md)..." + # copy LUS default shaders into assets/custom + COMMAND ${CMAKE_COMMAND} -E rm -r -f ${CMAKE_CURRENT_SOURCE_DIR}/soh/assets/custom/shaders/ + COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/libultraship/src/fast/shaders/ ${CMAKE_CURRENT_SOURCE_DIR}/soh/assets/custom/shaders/ + + COMMAND $ + ${CMAKE_CURRENT_SOURCE_DIR}/soh/assets/custom + ${CMAKE_BINARY_DIR}/soh/soh.o2r + ${CMAKE_PROJECT_VERSION} + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_BINARY_DIR}/soh/soh.o2r ${CMAKE_CURRENT_SOURCE_DIR}/soh.o2r + COMMENT "Generating soh.o2r..." + DEPENDS soh-o2r-packer BYPRODUCTS ${CMAKE_SOURCE_DIR}/soh.o2r ${CMAKE_BINARY_DIR}/soh/soh.o2r ) diff --git a/prebuilt/README.md b/prebuilt/README.md deleted file mode 100644 index 2b27c59ec4..0000000000 --- a/prebuilt/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# Prebuilt `soh.o2r` — **temporary** - -This directory exists to unblock the Torch migration and is **meant to be deleted**. - -`soh.o2r` is the port's own asset archive: fonts, textures, presets, translations, shaders — -everything under `soh/assets/custom/`, plus a `portVersion` stamp. It is built by -`GenerateSohOtr`, which today drives ZAPD through `OTRExporter/extract_assets.py`. The migration -deletes ZAPDTR and OTRExporter, so that producer goes away and its replacement — a small in-tree -packer (Phase 4 of `torch-migration/PLAN.md`) — has to reproduce it byte-for-byte, including -ZAPD's specific texture quantisation. - -Rather than block every other phase on that packer, the archive is checked in here and -`GenerateSohOtr` just copies it into place. **Phases 2, 3 and 5 become testable without the -packer existing.** - -This file also serves as Phase 4's acceptance oracle: the packer is correct when its output -matches this archive entry-for-entry. That replaces Gate E's planned sha256 manifest — the -artifact is strictly more useful than a list of hashes of it. - -## Provenance - -| | | -|---|---| -| Generated | 2026-07-25 | -| Shipwright branch | `torch-migration` @ `fe2f52248` | -| `develop` at the time | `c4e92a70f` (inputs verified identical — `git diff develop HEAD -- soh/assets/custom libultraship CMakeLists.txt` was empty) | -| libultraship submodule | `c57da1b4` | -| `CMAKE_PROJECT_VERSION` | `9.2.3` | -| Producer | `cmake --build build --target GenerateSohOtr` (ZAPD `botr -se OTR --norom`) | -| Size / entries | 4,414,957 bytes / 1,042 entries (1,038 checked-in assets + 3 LUS shaders + `portVersion`) | -| sha256 | `1d23b582fbc29cebf803b6ee54856089d0978e5496e6d4b1dbcde876e13cf13c` | -| sha256 of inputs | `de8ba0b5167bec6956258dd0445d01ac0f6daac84aff091c678d951cc72598f6` | - -The input hash covers every file under `soh/assets/custom/` except `shaders/`, which is not -checked in — it is copied out of `libultraship/src/fast/shaders/` at generation time. Recompute it -with: - -```sh -find soh/assets/custom -type f -not -path '*/shaders/*' -print0 | sort -z \ - | xargs -0 sha256sum | sha256sum -``` - -(`-print0`/`sort -z` are required: several preset filenames contain spaces.) - -## Generation is content-deterministic, not byte-deterministic - -Two consecutive runs produce archives with **identical payloads for all 1,042 entries** but -**different whole-file hashes** — the zip stores a modification timestamp per entry. So: - -- The sha256 above identifies *this* file, not "any correct archive". -- **Phase 4's acceptance test must compare entry-by-entry**, never whole-file. A byte-identical - archive is not achievable and not the goal. - -## What this pins - -The archive freezes three inputs. If any changes while this workaround is in place, the checked-in -copy is stale and **nothing will tell you at runtime** except missing or wrong assets: - -1. **`soh/assets/custom/`** — edit an asset and your change is silently ignored. Compare the input - hash above. -2. **libultraship's shaders** — `libultraship/src/fast/shaders/` is copied in at generation time, - so a LUS submodule bump can drift the three shaders inside this archive. -3. **The project version.** `portVersion` here is `9.2.3`. `OTRGlobals.cpp:283` requires exact - `major.minor.patch` equality against the build's version, and `RunExtract` calls `exit(1)` with - "soh.o2r is outdated" if it fails. A configure-time check in the root `CMakeLists.txt` catches - this and tells you to regenerate, because the runtime symptom is not obvious. - -## Regenerating - -From a tree where ZAPDTR/OTRExporter still build (i.e. `develop`): - -```sh -cmake --build build --target GenerateSohOtr -cp soh.o2r prebuilt/soh.o2r -``` - -Then update the provenance table above. - -## Removing this - -When Phase 4's packer lands and its output matches this archive entry-for-entry, delete this -directory, drop the `!prebuilt/soh.o2r` negation from `.gitignore`, and restore `GenerateSohOtr` -to a real build step (including the libultraship shader copy, which the copy-only version skips). - -**Delete it before the migration PR merges** if at all possible. It is a 4.4 MB binary, and a blob -that reaches `develop` is in the history permanently; removed in the final tree and squash-merged, -it never lands there at all. diff --git a/prebuilt/soh.o2r b/prebuilt/soh.o2r deleted file mode 100644 index 21070d1da1..0000000000 Binary files a/prebuilt/soh.o2r and /dev/null differ diff --git a/tools/soh-o2r-packer/PngTexture.cpp b/tools/soh-o2r-packer/PngTexture.cpp new file mode 100644 index 0000000000..4405688333 --- /dev/null +++ b/tools/soh-o2r-packer/PngTexture.cpp @@ -0,0 +1,145 @@ +// Encoding a PNG into an N64 texture is the one part of building soh.o2r that nothing else +// provides. Torch decodes N64 texture data out of a rom and libultraship decodes it again for +// rendering, but neither goes the other way. Everything else the packer needs -- the resource +// header, the texture type enum, size arithmetic, directory walking, zipping, portVersion -- +// comes from torch, so this file is the whole of what had to be written. +// +// The quantisation follows ZAPD's ZTexture exactly, since these bytes have to match the archives +// ZAPD produced. n64graphics is not a substitute: it scales (x * 15 / 255) where ZAPD shifts +// (x >> 4), which differ for most inputs. + +#include "PngTexture.h" + +#include +#include +#include + +#include "binarytools/BinaryWriter.h" +#include "factories/BaseFactory.h" +#include "factories/ResourceType.h" +#include "n64graphics/stb_image.h" +#include "utils/TextureUtils.h" + +namespace fs = std::filesystem; + +namespace { + +TextureType TypeFromString(const std::string& format) { + if (format == "rgba32") return TextureType::RGBA32bpp; + if (format == "rgb5a1") return TextureType::RGBA16bpp; + if (format == "ci4") return TextureType::Palette4bpp; + if (format == "ci8") return TextureType::Palette8bpp; + if (format == "i4") return TextureType::Grayscale4bpp; + if (format == "i8") return TextureType::Grayscale8bpp; + if (format == "ia4") return TextureType::GrayscaleAlpha4bpp; + if (format == "ia8") return TextureType::GrayscaleAlpha8bpp; + if (format == "ia16") return TextureType::GrayscaleAlpha16bpp; + return TextureType::Error; +} + +std::vector Encode(const uint8_t* rgba, int w, int h, TextureType type) { + auto px = [&](int y, int x, int c) -> uint8_t { return rgba[(((size_t)y * w) + x) * 4 + c]; }; + std::vector out(TextureUtils::CalculateTextureSize(type, w, h)); + + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + const size_t i = (size_t)y * w + x; + + switch (type) { + case TextureType::RGBA32bpp: + out[i * 4 + 0] = px(y, x, 0); + out[i * 4 + 1] = px(y, x, 1); + out[i * 4 + 2] = px(y, x, 2); + out[i * 4 + 3] = px(y, x, 3); + break; + + case TextureType::RGBA16bpp: { + const uint16_t data = ((px(y, x, 0) >> 3) << 11) | ((px(y, x, 1) >> 3) << 6) | + ((px(y, x, 2) >> 3) << 1) | (px(y, x, 3) != 0); + out[i * 2 + 0] = (data & 0xFF00) >> 8; + out[i * 2 + 1] = (data & 0x00FF); + break; + } + + case TextureType::Grayscale4bpp: + if (x % 2 == 0) { + out[i / 2] = (uint8_t)(((px(y, x, 0) / 16) << 4) + (px(y, x + 1, 0) / 16)); + } + break; + + case TextureType::Grayscale8bpp: + out[i] = px(y, x, 0); + break; + + case TextureType::GrayscaleAlpha4bpp: + if (x % 2 == 0) { + const uint8_t hi = ((px(y, x, 0) >> 5) << 1) | (px(y, x, 3) != 0); + const uint8_t lo = ((px(y, x + 1, 0) >> 5) << 1) | (px(y, x + 1, 3) != 0); + out[i / 2] = (uint8_t)((hi << 4) | lo); + } + break; + + case TextureType::GrayscaleAlpha8bpp: + out[i] = (uint8_t)((((px(y, x, 0) >> 4) & 0xF) << 4) | ((px(y, x, 3) >> 4) & 0xF)); + break; + + case TextureType::GrayscaleAlpha16bpp: + out[i * 2 + 0] = px(y, x, 0); + out[i * 2 + 1] = px(y, x, 3); + break; + + // Palettes need a TLUT this packer has no way to build. + default: + return {}; + } + } + } + + return out; +} + +} // namespace + +namespace PngTexture { + +bool IsFormat(const std::string& format) { + return TypeFromString(format) != TextureType::Error; +} + +bool Convert(const fs::path& png, const fs::path& dest, const std::string& format) { + const TextureType type = TypeFromString(format); + + int w = 0, h = 0, channels = 0; + uint8_t* pixels = stbi_load(png.string().c_str(), &w, &h, &channels, 4); + if (pixels == nullptr) { + fprintf(stderr, "%s: %s\n", png.string().c_str(), stbi_failure_reason()); + return false; + } + + const std::vector encoded = Encode(pixels, w, h, type); + stbi_image_free(pixels); + + if (encoded.empty()) { + fprintf(stderr, "%s: cannot encode %s\n", png.string().c_str(), format.c_str()); + return false; + } + + LUS::BinaryWriter writer; + BaseExporter::WriteHeader(writer, Torch::ResourceType::Texture, 0); + writer.Write((uint32_t)type); + writer.Write((uint32_t)w); + writer.Write((uint32_t)h); + writer.Write((uint32_t)encoded.size()); + writer.Write((char*)encoded.data(), encoded.size()); + + const std::vector payload = writer.ToVector(); + writer.Close(); + + fs::create_directories(dest.parent_path()); + std::ofstream out(dest, std::ios::binary); + out.write(payload.data(), payload.size()); + + return out.good(); +} + +} // namespace PngTexture diff --git a/tools/soh-o2r-packer/PngTexture.h b/tools/soh-o2r-packer/PngTexture.h new file mode 100644 index 0000000000..674d6f5ddc --- /dev/null +++ b/tools/soh-o2r-packer/PngTexture.h @@ -0,0 +1,17 @@ +#ifndef PNGTEXTURE_H +#define PNGTEXTURE_H + +#include +#include + +namespace PngTexture { + +// Whether format names an N64 texture format, as in ..png +bool IsFormat(const std::string& format); + +// Decodes png and writes it to dest as a libultraship texture resource. +bool Convert(const std::filesystem::path& png, const std::filesystem::path& dest, const std::string& format); + +} // namespace PngTexture + +#endif diff --git a/tools/soh-o2r-packer/main.cpp b/tools/soh-o2r-packer/main.cpp new file mode 100644 index 0000000000..bad554bb8c --- /dev/null +++ b/tools/soh-o2r-packer/main.cpp @@ -0,0 +1,75 @@ +// Packs soh/assets/custom into soh.o2r -- the port's own assets, as opposed to anything +// extracted from a rom. +// +// usage: soh-o2r-packer + +#include +#include +#include +#include + +#include "Companion.h" +#include "PngTexture.h" + +namespace fs = std::filesystem; + +int main(int argc, char** argv) { + if (argc != 4) { + fprintf(stderr, "usage: %s \n", argv[0]); + return 1; + } + + const fs::path assetsDir = argv[1]; + const fs::path outPath = argv[2]; + const std::string version = argv[3]; + + if (!fs::is_directory(assetsDir)) { + fprintf(stderr, "not a directory: %s\n", assetsDir.string().c_str()); + return 1; + } + + // Companion::Pack archives a directory as-is, so stage the assets in the shape the archive + // should have and let torch do the rest. + const fs::path stage = outPath.string() + ".stage"; + std::error_code ec; + fs::remove_all(stage, ec); + fs::create_directories(stage); + + for (const auto& entry : fs::recursive_directory_iterator(assetsDir)) { + if (!entry.is_regular_file()) { + continue; + } + + const fs::path& path = entry.path(); + const std::string rel = fs::relative(path, assetsDir).generic_string(); + const std::string filename = path.filename().string(); + + // ..png becomes a texture resource archived as + if (std::count(filename.begin(), filename.end(), '.') >= 2 && path.extension() == ".png") { + const std::string stem = path.stem().string(); + const std::string format = stem.substr(stem.find_last_of('.') + 1); + + if (PngTexture::IsFormat(format)) { + const std::string arc = rel.substr(0, rel.size() - (format.size() + 5)); + if (!PngTexture::Convert(path, stage / arc, format)) { + return 1; + } + continue; + } + } + + // Only json is carried over from accessibility + if (rel.find("accessibility") != std::string::npos && path.extension() != ".json") { + continue; + } + + fs::create_directories((stage / rel).parent_path()); + fs::copy_file(path, stage / rel, fs::copy_options::overwrite_existing); + } + + fs::remove(outPath, ec); + Companion::Pack(stage.string(), outPath.string(), ArchiveType::O2R, version); + fs::remove_all(stage, ec); + + return 0; +}