From 647b23301542c16fbe4a6dfd42bb3e9c8654a77d Mon Sep 17 00:00:00 2001 From: Alex Vanderveen Date: Sat, 29 Aug 2026 20:10:44 -0400 Subject: [PATCH] packaging: ship the mod catalog in every release, and let Studio bundle one A built title stages its mod catalog next to the exe, but nothing put it in the shipped package. Every release went out with an empty Mods page. The packager already had --runtime-dir mods; it was opt-in and RUNTIME_DIRS defaulted to empty, so each title had to pass it from its own generated wrapper. An audit of the workspace found 21 of 23 ports never did, and the two that did used the -optional variant that warns instead of failing. A capability that exists, defaults off, and fails silently. - package_setup_host.sh stages mods/ by DEFAULT and REQUIRES it. runtime.cmake guarantees /mods for every non-oracle target, so a missing one means broken build wiring and should fail the release rather than quietly ship an empty catalog. --no-mods opts out a title with none; an explicit --runtime-dir-optional mods still wins. It also ships the SOURCE catalog, because the setup-host zip rebuilds on the player's machine and that rebuild re-runs POST_BUILD staging from the source tree -- without it a rebuilt exe silently drops every game-owned mod and keeps only framework builtins. - package_release.ps1 (Windows/MinGW) had no mod handling at all. It now stages the catalog before the stray-file scan, so the existing gate covers it, strips the build machine's mods/state.toml, and asserts at least one manifest survived -- matching the file's own assert-rather-than-trust idiom. - package_release_macos.sh shipped ONE package where the player should have seen five. It took the catalog from mods/preloaded "never the build tree", but framework builtins (PGXP, fast loading, CD speed, bezel) live in psxrecomp/mods/builtin and only ever appear next to the exe. The real hazard was never the build tree, it was state.toml inside it -- so take the build tree and delete that one file. --mods-src still overrides. - The wrapper template gains --project-dir src so future titles ship their game-owned trusted-plugin sources; CMakeLists.txt references them unconditionally, so the setup-host rebuild cannot configure without them. Studio could build a title but never bundle one: the only route to a distributable was install-CI, commit, push, wait. That is how an empty Mods page survived unnoticed -- nobody produced a package locally to look inside. buildops.export_release() runs the SAME wrapper CI runs, so a local zip and a released one are the same artifact by construction; exposed as `build export` on the CLI and an "Export release zip" button in the GUI. Artifact tags default to the host and match the CI matrix. Verified by packaging, not by reading: default produces 5 catalogs with no state.toml; --no-mods reproduces the old behaviour at 0. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_0194TYJAqgyG9a9PJe9uUcgQ --- .../project_studio/buildops.py | 84 +++++++++++++++++++ .../new_project_layout/project_studio/cli.py | 36 ++++++++ .../new_project_layout/project_studio/gui.py | 29 +++++++ .../templates/package_setup_release.sh.in | 7 ++ tools/package_release.ps1 | 28 ++++++- tools/package_release_macos.sh | 43 ++++++++-- tools/package_setup_host.sh | 13 +++ 7 files changed, 231 insertions(+), 9 deletions(-) diff --git a/tools/new_project_layout/project_studio/buildops.py b/tools/new_project_layout/project_studio/buildops.py index 6c7145bc..8c52446b 100644 --- a/tools/new_project_layout/project_studio/buildops.py +++ b/tools/new_project_layout/project_studio/buildops.py @@ -837,3 +837,87 @@ def launch_status() -> str: if code is None: return f"running pid={h.pid} ({h.exe.name})" return f"exited code={code} ({h.exe.name})" + + +# --------------------------------------------------------------------------- +# Local release export +# +# Studio could build a title but never bundle one: the only path to a +# distributable package was "install release.yml, commit, push, wait for CI". +# That made a local mingw/linux export something you did by hand, which is how +# a broken package (an empty Mods page) survived so long -- nobody produced one +# locally to look inside. +# +# This runs the SAME wrapper CI runs (scripts/package_setup_release.sh), so a +# local export and a CI release are the same artifact by construction rather +# than by convention. +# --------------------------------------------------------------------------- + +def release_version(root: Path) -> str: + """Version the packager will stamp: $RELEASE_VERSION, else the VERSION file.""" + env_v = os.environ.get("RELEASE_VERSION", "").strip() + if env_v: + return env_v + vf = root / "VERSION" + if vf.is_file(): + text = vf.read_text(encoding="utf-8", errors="replace").strip() + if text: + return text + return "0.0.0" + + +def default_artifact_tag(host: BuildHost | None = None) -> str: + """CI's matrix tag for this host, so local zips match the released names.""" + h = host or detect_host() + if h.label == "windows": + return "windows-x64" + if h.label == "macos": + return "macos-arm64" + return "linux-x64" + + +def export_release( + root: Path, + build_dir: str = "build-release", + artifact_tag: str | None = None, + recompiler_build: str = "build-recompiler", + *, + log: LogFn | None = None, +) -> CmdResult: + """Package a distributable zip from an existing build. Returns its path.""" + wrapper = root / "scripts" / "package_setup_release.sh" + if not wrapper.is_file(): + return CmdResult( + False, + "No scripts/package_setup_release.sh", + "Emit it first (Migrate -> packager), or install release CI.", + ) + build_path = root / build_dir + if not build_path.is_dir(): + return CmdResult(False, f"No build directory: {build_dir}", + "Build the runtime target first.") + # Fail here rather than inside the packager: this is the exact condition + # that produced mod-less releases, and the message should name the cause. + if not (build_path / "mods").is_dir(): + return CmdResult( + False, + f"{build_dir}/mods is missing", + "runtime.cmake stages the mod catalog next to the exe on every " + "build. Rebuild the runtime target; packaging without it would " + "ship a title whose Mods page is empty.", + ) + tag = artifact_tag or default_artifact_tag() + env = dict(os.environ) + env.setdefault("RELEASE_VERSION", release_version(root)) + res = _run_stream( + ["bash", str(wrapper), build_dir, tag, recompiler_build], + root, log=log, env=env, + ) + if not res.ok: + return res + zips = sorted((root / "dist").glob(f"*-{tag}.zip"), + key=lambda p: p.stat().st_mtime, reverse=True) + if not zips: + return CmdResult(False, "Packager reported success but wrote no zip", + res.detail) + return CmdResult(True, f"Exported {zips[0].name}", str(zips[0])) diff --git a/tools/new_project_layout/project_studio/cli.py b/tools/new_project_layout/project_studio/cli.py index cc2a65ac..0487b72b 100644 --- a/tools/new_project_layout/project_studio/cli.py +++ b/tools/new_project_layout/project_studio/cli.py @@ -1511,6 +1511,28 @@ def cmd_build_compile(args: argparse.Namespace) -> int: return 0 if r.ok else 1 +def cmd_build_export(args: argparse.Namespace) -> int: + """Bundle a distributable zip from an existing build, mods included.""" + from project_studio.buildops import export_release + + root = _root_or_die(args) + if root is None: + return 2 + r = export_release( + root, + build_dir=args.build_dir, + artifact_tag=args.artifact or None, + recompiler_build=args.recompiler_build, + log=print, + ) + print(f"[{'OK' if r.ok else 'FAIL'}] {r.message}") + if r.ok: + print(r.detail) + elif r.detail: + print(r.detail, file=sys.stderr) + return 0 if r.ok else 1 + + def cmd_build_run(args: argparse.Namespace) -> int: import shlex from pathlib import Path @@ -2354,6 +2376,20 @@ def build_parser() -> argparse.ArgumentParser: p_bb.add_argument("--jobs", type=int, default=0) p_bb.set_defaults(func=cmd_build_compile) + p_bx = build_sub.add_parser( + "export", + help="Bundle a distributable zip from an existing build (mods included)", + ) + add_build_root(p_bx) + p_bx.add_argument( + "--artifact", + default="", + help="Artifact tag (default: this host -- linux-x64 / windows-x64). " + "Matches the CI matrix so local and released zips share names.", + ) + p_bx.add_argument("--recompiler-build", default="build-recompiler") + p_bx.set_defaults(func=cmd_build_export) + p_br = build_sub.add_parser("run", help="Launch product binary with env") add_build_root(p_br) p_br.add_argument("--exe", default="", help="Override executable path") diff --git a/tools/new_project_layout/project_studio/gui.py b/tools/new_project_layout/project_studio/gui.py index e36ee78f..354303be 100644 --- a/tools/new_project_layout/project_studio/gui.py +++ b/tools/new_project_layout/project_studio/gui.py @@ -2405,6 +2405,18 @@ class ProjectStudioApp: height=30, command=self._build_configure_and_build, ).pack(side="left", padx=8) + # Local export runs the SAME wrapper CI runs, so a zip made here and a + # released one are the same artifact by construction. It exists because + # "build it and look inside the package" had no button: the only route + # to a distributable was commit-and-wait-for-CI, which is how releases + # shipped with an empty Mods page without anyone noticing. + ctk.CTkButton( + brow, + text="Export release zip", + width=160, + height=30, + command=self._build_export, + ).pack(side="left") launch_box = ctk.CTkFrame(tab, corner_radius=10) launch_box.pack(fill="x", padx=4, pady=4) @@ -2598,6 +2610,23 @@ class ProjectStudioApp: self._build_run_bg("Build", go) + def _build_export(self) -> None: + """Bundle a distributable zip from the current build (mods included).""" + from .buildops import export_release + + root = self._game_root() + if root is None: + return + + def go(): + return export_release( + root, + build_dir=self.build_dir_var.get().strip() or "build-release", + log=lambda m: self.root.after(0, lambda line=m: self._log(line)), + ) + + self._build_run_bg("Export release", go) + def _build_configure_and_build(self) -> None: from .buildops import build, configure diff --git a/tools/new_project_layout/templates/package_setup_release.sh.in b/tools/new_project_layout/templates/package_setup_release.sh.in index 22b0e8a9..0364ba03 100644 --- a/tools/new_project_layout/templates/package_setup_release.sh.in +++ b/tools/new_project_layout/templates/package_setup_release.sh.in @@ -18,6 +18,12 @@ if [[ -z "${BUILD_DIR}" || -z "${ARTIFACT_TAG}" ]]; then exit 2 fi +# mods/ is staged by the framework packager by default (catalog next to the +# exe, plus the source catalog for the setup-host rebuild) -- no flag needed. +# --project-dir src ships game-owned trusted-plugin sources; CMakeLists.txt +# references them unconditionally, so the rebuild cannot configure without it. +# copy_proj skips a path that does not exist, so it is inert for titles with +# no src/. PACKAGER="${ROOT}/psxrecomp/tools/package_setup_host.sh" if [[ ! -f "${PACKAGER}" ]]; then echo "error: missing ${PACKAGER} (psxrecomp submodule)" >&2 @@ -56,4 +62,5 @@ exec bash "${PACKAGER}" \ --project-file README.md \ --project-dir seeds \ --project-dir launcher_assets \ + --project-dir src \ "${EXTRA_PROJECT[@]}" diff --git a/tools/package_release.ps1 b/tools/package_release.ps1 index 4b3635fd..9526d56b 100644 --- a/tools/package_release.ps1 +++ b/tools/package_release.ps1 @@ -76,11 +76,37 @@ if ($bakedBios) { } Write-Host "Verified no baked absolute BIOS path in the exe" +# Mod catalog. runtime.cmake stages /mods for every non-oracle target, so +# a build that lacks it has broken wiring -- fail rather than ship a package +# whose Mods page is silently empty. +# +# Staged BEFORE the stray-file scan below on purpose: mods/state.toml is the +# BUILD MACHINE's per-user enable/disable state and must never ship (preloaded +# catalogs are default-disabled, so a dev's selections would become everyone's +# defaults). Strip it here -- a dev machine legitimately has one, so that is a +# strip, not a failure -- and let the stray scan confirm none survived. +$ModsSrc = Join-Path $BuildPath "mods" +$ModsDst = Join-Path $Stage "mods" +if (!(Test-Path $ModsSrc)) { + throw "Build has no mods/ next to the exe: $ModsSrc (rebuild the runtime target)" +} +Copy-Item -Recurse -Force $ModsSrc $ModsDst +Get-ChildItem $ModsDst -Recurse -File -ErrorAction SilentlyContinue | + Where-Object { $_.Name -eq "state.toml" -or $_.Name -eq "state.toml.tmp" } | + Remove-Item -Force +$manifestCount = (Get-ChildItem $ModsDst -Recurse -File -Filter "manifest.toml" ` + -ErrorAction SilentlyContinue | Measure-Object).Count +if ($manifestCount -lt 1) { + throw "Staged mods/ contains no manifest.toml; the catalog would ship empty" +} +Write-Host "Staged mod catalog: $manifestCount manifest(s)" + # No user-machine or copyrighted files may ride along in the stage. OpenBIOS # is intentionally present and redistributable; retail SCPH images remain # forbidden. $strayPatterns = @("SCPH*.BIN","*.cue","*.iso","*.mcd","bios.cfg","disc.cfg", - "settings.toml","keybinds.ini","overlay_captures.json") + "settings.toml","keybinds.ini","overlay_captures.json", + "state.toml") $stray = foreach ($pat in $strayPatterns) { Get-ChildItem $Stage -Recurse -File -Filter $pat -ErrorAction SilentlyContinue } if ($stray) { throw "Stage contains files that must never ship: $(($stray | ForEach-Object FullName) -join '; ')" diff --git a/tools/package_release_macos.sh b/tools/package_release_macos.sh index e978b26d..6e52b450 100755 --- a/tools/package_release_macos.sh +++ b/tools/package_release_macos.sh @@ -36,7 +36,9 @@ # --version VER release version, e.g. v0.0.1 (default: dev) # --out DIR output directory (default: /release-macos) # --game-config P game.toml to bundle, relative to repo -# --mods-src DIR reviewed catalog source (default: /mods/preloaded) +# --mods-src DIR override catalog source (default: the build tree's +# mods/, which carries framework builtins AND the title's +# own packages; per-machine state.toml is stripped) # --require-mod PKG path under mods/packages that must exist (repeatable) # --data FILE file the runtime reads BESIDE the executable, relative to # repo, e.g. keybinds.ini (repeatable) @@ -60,7 +62,7 @@ note() { echo " $*"; } [ "$(uname -s)" = "Darwin" ] || die "run this on macOS." REPO=""; BUILD=""; BIN_NAME=""; APP_NAME=""; PACKAGE=""; BUNDLE_ID="" -VERSION="dev"; OUT=""; GAME_CONFIG=""; MODS_SRC="" +VERSION="dev"; OUT=""; GAME_CONFIG=""; MODS_SRC=""; MODS_SRC_EXPLICIT="" DO_DMG=0; DO_ZIP=1 REQUIRE_MODS=(); DOCS=(); DATA_FILES=() @@ -75,7 +77,7 @@ while [ $# -gt 0 ]; do --version) VERSION="$2"; shift 2;; --out) OUT="$2"; shift 2;; --game-config) GAME_CONFIG="$2"; shift 2;; - --mods-src) MODS_SRC="$2"; shift 2;; + --mods-src) MODS_SRC="$2"; MODS_SRC_EXPLICIT=1; shift 2;; --require-mod) REQUIRE_MODS+=("$2"); shift 2;; --data) DATA_FILES+=("$2"); shift 2;; --doc) DOCS+=("$2"); shift 2;; @@ -153,14 +155,39 @@ stage_data() { # stage_data stage_data bios "$BUILD/bios" stage_data assets "$BUILD/assets" -# Take the mod catalog from SOURCE, never the build tree: a developer build dir -# accumulates mods/state.toml with enhancements switched on, and shipping that -# silently flips a title's default presentation for every player. -if [ -d "$MODS_SRC" ]; then +# Mod catalog. The hazard this guards against is real but narrow: a developer +# build dir accumulates mods/state.toml with enhancements switched on, and +# shipping that silently flips a title's default presentation for every player. +# +# Taking the SOURCE catalog instead used to be the remedy, but it drops every +# framework builtin (PGXP, fast loading, CD speed, bezel): those live in +# psxrecomp/mods/builtin and only ever appear NEXT TO THE EXE, staged there by +# runtime.cmake. A title with its own catalog therefore shipped one package +# where the player should have seen five. +# +# So take the build tree -- which is exactly what the runtime resolves at +# runtime -- and delete the one file that is actually unsafe. That matches +# package_setup_host.sh and package_release.ps1, so all three packagers now +# ship the same catalog. An explicit --mods-src still wins for anyone who +# wants the source-only set. +if [ -n "$MODS_SRC_EXPLICIT" ]; then + [ -d "$MODS_SRC" ] || die "--mods-src is not a directory: $MODS_SRC" stage_data mods "$MODS_SRC" elif [ -d "$BUILD/mods" ]; then - die "refusing to package mods/ from the build tree ($BUILD/mods): pass --mods-src pointing at the reviewed source catalog" + stage_data mods "$BUILD/mods" + find "$APPDIR/Contents/Resources/mods" \ + \( -name state.toml -o -name state.toml.tmp \) -delete +elif [ -d "$MODS_SRC" ]; then + stage_data mods "$MODS_SRC" +else + die "no mod catalog found: neither $BUILD/mods nor $MODS_SRC exists (rebuild the runtime target)" fi +_mod_manifests=$(find "$APPDIR/Contents/Resources/mods" -name manifest.toml 2>/dev/null | wc -l) +[ "$_mod_manifests" -ge 1 ] \ + || die "staged mods/ contains no manifest.toml; the catalog would ship empty" +[ -z "$(find "$APPDIR/Contents/Resources/mods" -name 'state.toml*' 2>/dev/null)" ] \ + || die "per-machine mods/state.toml survived staging" +note "mod catalog: ${_mod_manifests} manifest(s), no per-machine state" if [ -n "$GAME_CONFIG" ]; then stage_data "$(basename "$GAME_CONFIG")" "$REPO/$GAME_CONFIG" diff --git a/tools/package_setup_host.sh b/tools/package_setup_host.sh index 04302cb2..601a200e 100755 --- a/tools/package_setup_host.sh +++ b/tools/package_setup_host.sh @@ -112,6 +112,19 @@ if [[ "${STAGE_MODS}" -eq 1 ]]; then if [[ "${_mods_named}" -eq 0 ]]; then RUNTIME_DIRS+=("mods") fi + # Also ship the SOURCE catalog (mods/preloaded/...) when the title has one. + # The setup-host zip rebuilds on the player's machine, and that rebuild + # re-runs the title's POST_BUILD mod staging -- which reads the source tree, + # not the staged copy. Without this the rebuilt exe silently loses every + # game-owned mod and keeps only the framework builtins. copy_proj skips a + # path that does not exist, so this is inert for a title with no catalog. + _mods_proj=0 + for _d in "${PROJECT_DIRS[@]:-}"; do + [[ "${_d}" == "mods" ]] && _mods_proj=1 + done + if [[ "${_mods_proj}" -eq 0 ]]; then + PROJECT_DIRS+=("mods") + fi fi if [[ -z "${BUILD_DIR}" || -z "${ARTIFACT}" || -z "${ZIP_PREFIX}" || -z "${EXE_NAME}" ]]; then