mirror/Shipwright
1
0
Fork 0
mirror of https://github.com/HarbourMasters/Shipwright synced 2026-08-12 12:18:32 -04:00
Code Issues Packages Projects Releases Wiki Activity

Generate soh.o2r again, replacing the prebuilt archive

Browse Source 
soh-o2r-packer builds soh.o2r from soh/assets/custom, reproducing the
ZAPD/OTRExporter archive exactly: 1,042 entries, 0 missing, 0 extra, 0
content mismatch, checked entry-wise because zip stores per-entry
timestamps and whole-file hashes never match between runs.

Most of it is torch's: Companion::Pack walks the directory, zips it and
writes portVersion; BaseExporter::WriteHeader writes the resource header;
TextureType and CalculateTextureSize come from TextureUtils. The packer
stages the assets into the shape the archive should have and hands that
over. Encoding a PNG into an N64 texture is the only piece nothing else
provides -- torch decodes rom data that is already N64 format, never the
reverse -- so PngTexture.cpp is the whole of what had to be written, and
it follows ZAPD's quantisation rather than n64graphics', which scales
where ZAPD shifts.

GenerateSohOtr copies the libultraship shaders into assets/custom again
before packing; assets/custom/shaders is gitignored and nothing else
populates it, so a fresh clone would otherwise pack three files short.

Removes prebuilt/ and its configure-time portVersion guard.
This commit is contained in:
briaguya
2026-07-25 03:39:14 -04:00
parent 359060200e
commit e1f372d6a5
6 changed files with 255 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+18 -42
View File
@@ -246,6 +246,12 @@ add_executable(soh-torch
target_include_directories(soh-torch PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/soh/soh/Extractor)
target_link_libraries(soh-torch PRIVATE torch)
add_executable(soh-o2r-packer
${CMAKE_CURRENT_SOURCE_DIR}/tools/soh-o2r-packer/main.cpp
${CMAKE_CURRENT_SOURCE_DIR}/tools/soh-o2r-packer/PngTexture.cpp
)
target_link_libraries(soh-o2r-packer PRIVATE torch)
# Target to generate OTRs. SOH_ROM_PATH takes roms and/or directories of roms; torch names each
# archive (oot.o2r or oot-mq.o2r) from its hash, so a vanilla and a master quest rom produce both
# in one run. soh.o2r comes from GenerateSohOtr, chained below.
@@ -275,50 +281,20 @@ add_custom_target(
COMMAND ${CMAKE_COMMAND} -E false
)
# TEMPORARY (Torch migration): soh.o2r is copied from prebuilt/, not generated.
#
# The real producer drives ZAPD via OTRExporter/extract_assets.py, and the migration deletes
# both. Its replacement -- a small in-tree packer, Phase 4 of torch-migration/PLAN.md -- has to
# reproduce ZAPD's texture quantisation exactly, so rather than block Phases 2/3/5 on it, the
# archive is checked in and copied into place. See prebuilt/README.md for provenance and for how
# to restore a real build step (which must also bring back the libultraship shader copy that the
# copy-only version below has no use for -- the shaders are already inside the archive).
#
# Fail at configure time if the prebuilt archive's baked-in portVersion no longer matches the
# project version. OTRGlobals.cpp:283 demands exact major.minor.patch equality and RunExtract
# exit(1)s with "soh.o2r is outdated" -- an obscure symptom for what is really a stale checked-in
# file. portVersion is 7 bytes: endianness, then u16 major/minor/patch big-endian.
set(SOH_PREBUILT_O2R "${CMAKE_CURRENT_SOURCE_DIR}/prebuilt/soh.o2r")
if(NOT EXISTS "${SOH_PREBUILT_O2R}")
message(FATAL_ERROR "Missing ${SOH_PREBUILT_O2R} -- see prebuilt/README.md")
endif()
file(READ "${SOH_PREBUILT_O2R}" _soh_o2r_hex HEX)
string(FIND "${_soh_o2r_hex}" "706f727456657273696f6e" _pv_at) # "portVersion" as hex
if(_pv_at EQUAL -1)
message(FATAL_ERROR "No portVersion entry in ${SOH_PREBUILT_O2R} -- see prebuilt/README.md")
endif()
math(EXPR _pv_at "${_pv_at} + 22") # skip the 11-character name
string(SUBSTRING "${_soh_o2r_hex}" ${_pv_at} 14 _pv)
string(SUBSTRING "${_pv}" 2 4 _pv_major)
string(SUBSTRING "${_pv}" 6 4 _pv_minor)
string(SUBSTRING "${_pv}" 10 4 _pv_patch)
math(EXPR _pv_major "0x${_pv_major}")
math(EXPR _pv_minor "0x${_pv_minor}")
math(EXPR _pv_patch "0x${_pv_patch}")
if(NOT "${_pv_major}.${_pv_minor}.${_pv_patch}" STREQUAL "${CMAKE_PROJECT_VERSION}")
message(FATAL_ERROR
"prebuilt/soh.o2r is stamped ${_pv_major}.${_pv_minor}.${_pv_patch} but this build is "
"${CMAKE_PROJECT_VERSION}. The game would reject it at startup (\"soh.o2r is outdated\"). "
"Regenerate it -- see prebuilt/README.md.")
endif()
unset(_soh_o2r_hex)
# Target to place soh.o2r (copy of the prebuilt archive)
# Target to generate only soh.o2r
add_custom_target(
GenerateSohOtr
COMMAND ${CMAKE_COMMAND} -E copy_if_different "${SOH_PREBUILT_O2R}" ${CMAKE_CURRENT_SOURCE_DIR}/soh.o2r
COMMAND ${CMAKE_COMMAND} -E copy_if_different "${SOH_PREBUILT_O2R}" ${CMAKE_BINARY_DIR}/soh/soh.o2r
COMMENT "Copying prebuilt soh.o2r (TEMPORARY -- see prebuilt/README.md)..."
# copy LUS default shaders into assets/custom
COMMAND ${CMAKE_COMMAND} -E rm -r -f ${CMAKE_CURRENT_SOURCE_DIR}/soh/assets/custom/shaders/
COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/libultraship/src/fast/shaders/ ${CMAKE_CURRENT_SOURCE_DIR}/soh/assets/custom/shaders/
COMMAND $<TARGET_FILE:soh-o2r-packer>
${CMAKE_CURRENT_SOURCE_DIR}/soh/assets/custom
${CMAKE_BINARY_DIR}/soh/soh.o2r
${CMAKE_PROJECT_VERSION}
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_BINARY_DIR}/soh/soh.o2r ${CMAKE_CURRENT_SOURCE_DIR}/soh.o2r
COMMENT "Generating soh.o2r..."
DEPENDS soh-o2r-packer
BYPRODUCTS ${CMAKE_SOURCE_DIR}/soh.o2r ${CMAKE_BINARY_DIR}/soh/soh.o2r
)
prebuilt/README.md
-87
View File
@@ -1,87 +0,0 @@
# Prebuilt `soh.o2r` — **temporary**
This directory exists to unblock the Torch migration and is **meant to be deleted**.
`soh.o2r` is the port's own asset archive: fonts, textures, presets, translations, shaders —
everything under `soh/assets/custom/`, plus a `portVersion` stamp. It is built by
`GenerateSohOtr`, which today drives ZAPD through `OTRExporter/extract_assets.py`. The migration
deletes ZAPDTR and OTRExporter, so that producer goes away and its replacement — a small in-tree
packer (Phase 4 of `torch-migration/PLAN.md`) — has to reproduce it byte-for-byte, including
ZAPD's specific texture quantisation.
Rather than block every other phase on that packer, the archive is checked in here and
`GenerateSohOtr` just copies it into place. **Phases 2, 3 and 5 become testable without the
packer existing.**
This file also serves as Phase 4's acceptance oracle: the packer is correct when its output
matches this archive entry-for-entry. That replaces Gate E's planned sha256 manifest — the
artifact is strictly more useful than a list of hashes of it.
## Provenance
| | |
|---|---|
| Generated | 2026-07-25 |
| Shipwright branch | `torch-migration` @ `fe2f52248` |
| `develop` at the time | `c4e92a70f` (inputs verified identical — `git diff develop HEAD -- soh/assets/custom libultraship CMakeLists.txt` was empty) |
| libultraship submodule | `c57da1b4` |
| `CMAKE_PROJECT_VERSION` | `9.2.3` |
| Producer | `cmake --build build --target GenerateSohOtr` (ZAPD `botr -se OTR --norom`) |
| Size / entries | 4,414,957 bytes / 1,042 entries (1,038 checked-in assets + 3 LUS shaders + `portVersion`) |
| sha256 | `1d23b582fbc29cebf803b6ee54856089d0978e5496e6d4b1dbcde876e13cf13c` |
| sha256 of inputs | `de8ba0b5167bec6956258dd0445d01ac0f6daac84aff091c678d951cc72598f6` |
The input hash covers every file under `soh/assets/custom/` except `shaders/`, which is not
checked in — it is copied out of `libultraship/src/fast/shaders/` at generation time. Recompute it
with:
```sh
find soh/assets/custom -type f -not -path '*/shaders/*' -print0 | sort -z \
| xargs -0 sha256sum | sha256sum
```
(`-print0`/`sort -z` are required: several preset filenames contain spaces.)
## Generation is content-deterministic, not byte-deterministic
Two consecutive runs produce archives with **identical payloads for all 1,042 entries** but
**different whole-file hashes** — the zip stores a modification timestamp per entry. So:
- The sha256 above identifies *this* file, not "any correct archive".
- **Phase 4's acceptance test must compare entry-by-entry**, never whole-file. A byte-identical
archive is not achievable and not the goal.
## What this pins
The archive freezes three inputs. If any changes while this workaround is in place, the checked-in
copy is stale and **nothing will tell you at runtime** except missing or wrong assets:
1. **`soh/assets/custom/`** — edit an asset and your change is silently ignored. Compare the input
hash above.
2. **libultraship's shaders** — `libultraship/src/fast/shaders/` is copied in at generation time,
so a LUS submodule bump can drift the three shaders inside this archive.
3. **The project version.** `portVersion` here is `9.2.3`. `OTRGlobals.cpp:283` requires exact
`major.minor.patch` equality against the build's version, and `RunExtract` calls `exit(1)` with
"soh.o2r is outdated" if it fails. A configure-time check in the root `CMakeLists.txt` catches
this and tells you to regenerate, because the runtime symptom is not obvious.
## Regenerating
From a tree where ZAPDTR/OTRExporter still build (i.e. `develop`):
```sh
cmake --build build --target GenerateSohOtr
cp soh.o2r prebuilt/soh.o2r
```
Then update the provenance table above.
## Removing this
When Phase 4's packer lands and its output matches this archive entry-for-entry, delete this
directory, drop the `!prebuilt/soh.o2r` negation from `.gitignore`, and restore `GenerateSohOtr`
to a real build step (including the libultraship shader copy, which the copy-only version skips).
**Delete it before the migration PR merges** if at all possible. It is a 4.4 MB binary, and a blob
that reaches `develop` is in the history permanently; removed in the final tree and squash-merged,
it never lands there at all.
prebuilt/soh.o2r
BIN
View File
Binary file not shown.
tools/soh-o2r-packer/PngTexture.cpp
+145
View File
@@ -0,0 +1,145 @@
// Encoding a PNG into an N64 texture is the one part of building soh.o2r that nothing else
// provides. Torch decodes N64 texture data out of a rom and libultraship decodes it again for
// rendering, but neither goes the other way. Everything else the packer needs -- the resource
// header, the texture type enum, size arithmetic, directory walking, zipping, portVersion --
// comes from torch, so this file is the whole of what had to be written.
//
// The quantisation follows ZAPD's ZTexture exactly, since these bytes have to match the archives
// ZAPD produced. n64graphics is not a substitute: it scales (x * 15 / 255) where ZAPD shifts
// (x >> 4), which differ for most inputs.
#include "PngTexture.h"
#include <cstdio>
#include <fstream>
#include <vector>
#include "binarytools/BinaryWriter.h"
#include "factories/BaseFactory.h"
#include "factories/ResourceType.h"
#include "n64graphics/stb_image.h"
#include "utils/TextureUtils.h"
namespace fs = std::filesystem;
namespace {
TextureType TypeFromString(const std::string& format) {
if (format == "rgba32") return TextureType::RGBA32bpp;
if (format == "rgb5a1") return TextureType::RGBA16bpp;
if (format == "ci4") return TextureType::Palette4bpp;
if (format == "ci8") return TextureType::Palette8bpp;
if (format == "i4") return TextureType::Grayscale4bpp;
if (format == "i8") return TextureType::Grayscale8bpp;
if (format == "ia4") return TextureType::GrayscaleAlpha4bpp;
if (format == "ia8") return TextureType::GrayscaleAlpha8bpp;
if (format == "ia16") return TextureType::GrayscaleAlpha16bpp;
return TextureType::Error;
}
std::vector<uint8_t> Encode(const uint8_t* rgba, int w, int h, TextureType type) {
auto px = [&](int y, int x, int c) -> uint8_t { return rgba[(((size_t)y * w) + x) * 4 + c]; };
std::vector<uint8_t> out(TextureUtils::CalculateTextureSize(type, w, h));
for (int y = 0; y < h; y++) {
for (int x = 0; x < w; x++) {
const size_t i = (size_t)y * w + x;
switch (type) {
case TextureType::RGBA32bpp:
out[i * 4 + 0] = px(y, x, 0);
out[i * 4 + 1] = px(y, x, 1);
out[i * 4 + 2] = px(y, x, 2);
out[i * 4 + 3] = px(y, x, 3);
break;
case TextureType::RGBA16bpp: {
const uint16_t data = ((px(y, x, 0) >> 3) << 11) | ((px(y, x, 1) >> 3) << 6) |
((px(y, x, 2) >> 3) << 1) | (px(y, x, 3) != 0);
out[i * 2 + 0] = (data & 0xFF00) >> 8;
out[i * 2 + 1] = (data & 0x00FF);
break;
}
case TextureType::Grayscale4bpp:
if (x % 2 == 0) {
out[i / 2] = (uint8_t)(((px(y, x, 0) / 16) << 4) + (px(y, x + 1, 0) / 16));
}
break;
case TextureType::Grayscale8bpp:
out[i] = px(y, x, 0);
break;
case TextureType::GrayscaleAlpha4bpp:
if (x % 2 == 0) {
const uint8_t hi = ((px(y, x, 0) >> 5) << 1) | (px(y, x, 3) != 0);
const uint8_t lo = ((px(y, x + 1, 0) >> 5) << 1) | (px(y, x + 1, 3) != 0);
out[i / 2] = (uint8_t)((hi << 4) | lo);
}
break;
case TextureType::GrayscaleAlpha8bpp:
out[i] = (uint8_t)((((px(y, x, 0) >> 4) & 0xF) << 4) | ((px(y, x, 3) >> 4) & 0xF));
break;
case TextureType::GrayscaleAlpha16bpp:
out[i * 2 + 0] = px(y, x, 0);
out[i * 2 + 1] = px(y, x, 3);
break;
// Palettes need a TLUT this packer has no way to build.
default:
return {};
}
}
}
return out;
}
} // namespace
namespace PngTexture {
bool IsFormat(const std::string& format) {
return TypeFromString(format) != TextureType::Error;
}
bool Convert(const fs::path& png, const fs::path& dest, const std::string& format) {
const TextureType type = TypeFromString(format);
int w = 0, h = 0, channels = 0;
uint8_t* pixels = stbi_load(png.string().c_str(), &w, &h, &channels, 4);
if (pixels == nullptr) {
fprintf(stderr, "%s: %s\n", png.string().c_str(), stbi_failure_reason());
return false;
}
const std::vector<uint8_t> encoded = Encode(pixels, w, h, type);
stbi_image_free(pixels);
if (encoded.empty()) {
fprintf(stderr, "%s: cannot encode %s\n", png.string().c_str(), format.c_str());
return false;
}
LUS::BinaryWriter writer;
BaseExporter::WriteHeader(writer, Torch::ResourceType::Texture, 0);
writer.Write((uint32_t)type);
writer.Write((uint32_t)w);
writer.Write((uint32_t)h);
writer.Write((uint32_t)encoded.size());
writer.Write((char*)encoded.data(), encoded.size());
const std::vector<char> payload = writer.ToVector();
writer.Close();
fs::create_directories(dest.parent_path());
std::ofstream out(dest, std::ios::binary);
out.write(payload.data(), payload.size());
return out.good();
}
} // namespace PngTexture
tools/soh-o2r-packer/PngTexture.h
+17
View File
@@ -0,0 +1,17 @@
#ifndef PNGTEXTURE_H
#define PNGTEXTURE_H
#include <filesystem>
#include <string>
namespace PngTexture {
// Whether format names an N64 texture format, as in <name>.<format>.png
bool IsFormat(const std::string& format);
// Decodes png and writes it to dest as a libultraship texture resource.
bool Convert(const std::filesystem::path& png, const std::filesystem::path& dest, const std::string& format);
} // namespace PngTexture
#endif
tools/soh-o2r-packer/main.cpp
+75
View File
@@ -0,0 +1,75 @@
// Packs soh/assets/custom into soh.o2r -- the port's own assets, as opposed to anything
// extracted from a rom.
//
// usage: soh-o2r-packer <custom assets dir> <out.o2r> <M.m.p>
#include <algorithm>
#include <cstdio>
#include <filesystem>
#include <string>
#include "Companion.h"
#include "PngTexture.h"
namespace fs = std::filesystem;
int main(int argc, char** argv) {
if (argc != 4) {
fprintf(stderr, "usage: %s <custom assets dir> <out.o2r> <M.m.p>\n", argv[0]);
return 1;
}
const fs::path assetsDir = argv[1];
const fs::path outPath = argv[2];
const std::string version = argv[3];
if (!fs::is_directory(assetsDir)) {
fprintf(stderr, "not a directory: %s\n", assetsDir.string().c_str());
return 1;
}
// Companion::Pack archives a directory as-is, so stage the assets in the shape the archive
// should have and let torch do the rest.
const fs::path stage = outPath.string() + ".stage";
std::error_code ec;
fs::remove_all(stage, ec);
fs::create_directories(stage);
for (const auto& entry : fs::recursive_directory_iterator(assetsDir)) {
if (!entry.is_regular_file()) {
continue;
}
const fs::path& path = entry.path();
const std::string rel = fs::relative(path, assetsDir).generic_string();
const std::string filename = path.filename().string();
// <name>.<format>.png becomes a texture resource archived as <name>
if (std::count(filename.begin(), filename.end(), '.') >= 2 && path.extension() == ".png") {
const std::string stem = path.stem().string();
const std::string format = stem.substr(stem.find_last_of('.') + 1);
if (PngTexture::IsFormat(format)) {
const std::string arc = rel.substr(0, rel.size() - (format.size() + 5));
if (!PngTexture::Convert(path, stage / arc, format)) {
return 1;
}
continue;
}
}
// Only json is carried over from accessibility
if (rel.find("accessibility") != std::string::npos && path.extension() != ".json") {
continue;
}
fs::create_directories((stage / rel).parent_path());
fs::copy_file(path, stage / rel, fs::copy_options::overwrite_existing);
}
fs::remove(outPath, ec);
Companion::Pack(stage.string(), outPath.string(), ArchiveType::O2R, version);
fs::remove_all(stage, ec);
return 0;
}