diff --git a/docs/GAME_PROJECT_SETUP.md b/docs/GAME_PROJECT_SETUP.md index 618e5c85..bf105208 100644 --- a/docs/GAME_PROJECT_SETUP.md +++ b/docs/GAME_PROJECT_SETUP.md @@ -110,20 +110,22 @@ Details: [`LOCAL_CODEGEN_SDK.md`](LOCAL_CODEGEN_SDK.md), [`BUILDING.md`](BUILDIN ## Setup-host releases (self-build zip) -CI ships a zip **without** `generated/` game C and **without** retail BIOS -dumps. The zip includes: +CI ships a zip **without** `generated/` game C, retail BIOS dumps, or an +embedded `toolchain/`. The zip includes: - Setup host exe (`recomp-ui` + generate/rebuild wizard) - Your game sources (`game.toml`, seeds, CMake, host glue, …) - `psxrecomp/` (runtime + CLI + OpenBIOS profiles + emitters) - `recomp-ui/` sources (needed to rebuild) -- `toolchain/` — portable `cmake-clang-v1` from - [retcomm-toolchains](https://github.com/TechnicallyComputers/retcomm-toolchains) - On Windows: MinGW runtime DLLs beside the host and emitters Players (or [RetComM](https://github.com/TechnicallyComputers/RetComM-Launcher)) -run Generate & rebuild locally. After a successful rebuild the CLI can prune -`toolchain/` and build intermediates. +run Generate & rebuild locally. RetComM / the wizard download +`cmake-clang-v1` from +[retcomm-toolchains](https://github.com/TechnicallyComputers/retcomm-toolchains) +(or accept an offline zip / `RETCOMM_TOOLCHAIN_DIR`). Pass +`--embed-toolchain` to `package_setup_host.sh` only for special offline-first +packs. Do **not** set `PSX_PGO` in CI. PGO stays user-local when `[pgo] enabled = true`. @@ -137,8 +139,8 @@ Do **not** set `PSX_PGO` in CI. PGO stays user-local when `[pgo] enabled = true` | `tools/ci/clear_generated.sh` | Wipe `generated/` for setup-host CI | | `tools/ci/record_pins.sh` | Log submodule SHAs | | `tools/ci/build_emitters.sh` | Build `psxrecomp-game` + `psxrecomp-bios` | -| `tools/fetch_toolchain.sh` | Download/unpack portable cmake/clang | -| `tools/stage_setup_sdk.sh` | Emitters + OpenBIOS + `toolchain/` + MinGW DLLs | +| `tools/fetch_toolchain.sh` | Optional download/unpack (embed packs only) | +| `tools/stage_setup_sdk.sh` | Emitters + OpenBIOS + optional `toolchain/` + MinGW DLLs | | `tools/bundle_mingw_dlls.sh` | Windows runtime DLL copy | | `tools/package_setup_host.sh` | Full setup-host zip (title args) | @@ -146,13 +148,15 @@ Composite actions (from the game repo after checkout): ```yaml - uses: ./psxrecomp/.github/actions/build-emitters -- uses: ./psxrecomp/.github/actions/fetch-toolchain - with: - artifact: ${{ matrix.artifact }} +# Optional — only when packaging with --embed-toolchain: +# - uses: ./psxrecomp/.github/actions/fetch-toolchain +# with: +# artifact: ${{ matrix.artifact }} - uses: ./psxrecomp/.github/actions/stage-setup-sdk with: stage: dist/stage-setup-${{ matrix.artifact }} recompiler-build: build-recompiler + allow-no-toolchain: 'true' ``` --- @@ -196,9 +200,9 @@ Use this before tagging a setup-host release that matches other titles ### Packaging (shared helpers) - [ ] CI uses `./psxrecomp/.github/actions/build-emitters` -- [ ] CI uses `./psxrecomp/.github/actions/fetch-toolchain` -- [ ] Packager calls `psxrecomp/tools/package_setup_host.sh` (or - `stage_setup_sdk.sh` after a custom stage) +- [ ] Packager calls `psxrecomp/tools/package_setup_host.sh` (lean zip by + default; optional `--embed-toolchain`) or `stage_setup_sdk.sh` after a + custom stage - [ ] Staged tree includes `psxrecomp/psxrecomp_cli.py` - [ ] Staged `psxrecomp/bios/` has `OpenBIOS.toml`, `openbios.bin`, `OpenBIOS.LICENSE`, `SCPH1001.toml` — and **no** retail `.BIN` diff --git a/docs/LOCAL_CODEGEN_SDK.md b/docs/LOCAL_CODEGEN_SDK.md index 3f4598f5..c8afbb6d 100644 --- a/docs/LOCAL_CODEGEN_SDK.md +++ b/docs/LOCAL_CODEGEN_SDK.md @@ -23,17 +23,19 @@ rebuild links everything. | `psxrecomp_cli.py` | Generate / rebuild / verify-disc (in the submodule) | | `tools/package_setup_host.sh` | Universal setup-host zip packager | | `tools/ci/*.sh` | normalize version, clear generated, record pins, build emitters | -| `tools/fetch_toolchain.sh` | CI: download/unpack portable `cmake-clang-v1` | -| `tools/stage_setup_sdk.sh` | CI/pack: emitters, OpenBIOS checks, embed `toolchain/`, MinGW DLLs | +| `tools/fetch_toolchain.sh` | Optional: download/unpack `cmake-clang-v1` (CI embed or local) | +| `tools/toolchain_pack.py` | CLI: resolve / download / offline-unpack into shared cache | +| `tools/stage_setup_sdk.sh` | Pack: emitters, OpenBIOS checks, optional `toolchain/`, MinGW DLLs | | `tools/bundle_mingw_dlls.sh` | Windows: copy MinGW runtime DLLs next to host + emitters | -| Game zip `toolchain/` | Portable `cmake-clang-v1` (cmake/ninja/clang); pruned after rebuild | +| Shared toolchain cache | `~/.local/share/psxrecomp/toolchains/cmake-clang-v1/` (or RetComM’s) | | `docs/ci/` | Composite actions + [`templates/setup-release.yml`](ci/templates/setup-release.yml) | | `docs/GAME_PROJECT_SETUP.md` | Submodules, CI template usage, bundled-release checklist | | Game sources | `game.toml`, seeds, `CMakeLists.txt` at repo root; `psxrecomp/`, `recomp-ui/` submodules | -RetComM harvests emitters + toolchain into shared caches (no separate tools -zip required). Wizard rebuild uses `--prune-after toolchain,build-intermediates`. -Needs Python 3; cmake comes from `toolchain/` when bundled. +RetComM harvests emitters into the SDK cache and **downloads** `cmake-clang-v1` +(no separate tools zip; game zips stay lean). The wizard / CLI +`ensure-toolchain` use the same packs; offline builds accept +`--from-zip` / a file picker. Needs Python 3. ## Commands @@ -46,11 +48,15 @@ python psxrecomp/psxrecomp_cli.py generate \ [--bios path/to/SCPH1001.BIN] [--force-bios] \ [--skip-hash-check] [--force-prepare] [--json-progress] +python psxrecomp/psxrecomp_cli.py ensure-toolchain \ + [--project-root .] [--from-zip cmake-clang-v1-linux-x64.zip] [--no-download] + python psxrecomp/psxrecomp_cli.py rebuild \ --config game.toml --project-root . \ --build-dir build-release --target psx-runtime \ --exe-basename Bomberman_Party_Edition_Recompiled \ - [--disc path/to/game.cue] [--no-pgo] [--force-pgo] [--json-progress] + [--disc path/to/game.cue] [--toolchain-zip path/to/pack.zip] \ + [--no-toolchain-download] [--no-pgo] [--force-pgo] [--json-progress] python psxrecomp/psxrecomp_cli.py pgo-train \ --config game.toml --build-dir build-release \ diff --git a/docs/ci/README.md b/docs/ci/README.md index 0a09eeaf..effa2345 100644 --- a/docs/ci/README.md +++ b/docs/ci/README.md @@ -23,10 +23,10 @@ Template: [`templates/setup-release.yml`](templates/setup-release.yml) | `ci/clear_generated.sh` | Clear `generated/` for setup-host CI | | `ci/record_pins.sh` | Log `psxrecomp` / `recomp-ui` / `recomp-net` SHAs | | `ci/build_emitters.sh` | Build `psxrecomp-game` + `psxrecomp-bios` | -| `fetch_toolchain.sh` | Download/unpack `cmake-clang-v1` | -| `stage_setup_sdk.sh` | Emitters, OpenBIOS, `toolchain/`, MinGW DLLs | +| `fetch_toolchain.sh` | Optional download/unpack (only if embedding) | +| `stage_setup_sdk.sh` | Emitters, OpenBIOS, optional `toolchain/`, MinGW DLLs | | `bundle_mingw_dlls.sh` | Copy imported non-system DLLs next to Windows PEs | -| `package_setup_host.sh` | Full setup-host zip (title passes exe/prefix/files) | +| `package_setup_host.sh` | Lean setup-host zip (optional `--embed-toolchain`) | | `../cmake/toolchain-mingw-w64.cmake` | Linux→Windows MinGW cross toolchain | | `../host/psxrecomp_codegen_host.*` | Portable Generate & rebuild host (via CMake helper) | | `templates/game.gitignore` | Suggested gitignore for title repos | @@ -40,14 +40,13 @@ submodule — there is no separate `psxrecomp-sdk/` overlay. ```yaml - uses: ./psxrecomp/.github/actions/build-emitters -- uses: ./psxrecomp/.github/actions/fetch-toolchain - with: - artifact: ${{ matrix.artifact }} - +# Prefer package_setup_host.sh (allow-no-toolchain by default). +# fetch-toolchain + --embed-toolchain only for offline-first packs. - uses: ./psxrecomp/.github/actions/stage-setup-sdk with: stage: dist/stage-setup-${{ matrix.artifact }} recompiler-build: build-recompiler + allow-no-toolchain: 'true' runtime-bin: /mingw64/bin # Windows / MSYS2 ``` diff --git a/docs/ci/templates/setup-release.yml b/docs/ci/templates/setup-release.yml index 7b832137..e6b871ee 100644 --- a/docs/ci/templates/setup-release.yml +++ b/docs/ci/templates/setup-release.yml @@ -6,7 +6,7 @@ # Then replace every YOUR_* token. See docs/GAME_PROJECT_SETUP.md. # # Shared (do not fork these into the title): -# ./psxrecomp/.github/actions/{build-emitters,fetch-toolchain,stage-setup-sdk} +# ./psxrecomp/.github/actions/{build-emitters,fetch-toolchain (optional embed),stage-setup-sdk} # psxrecomp/tools/ci/*.sh # psxrecomp/tools/{fetch_toolchain,stage_setup_sdk,bundle_mingw_dlls,package_setup_host}.sh # psxrecomp/psxrecomp_cli.py (CLI lives in the submodule — no psxrecomp-sdk/) @@ -173,11 +173,6 @@ jobs: "${EXTRA[@]}" cmake --build build-ci --target psx-runtime -j"${JOBS}" - - name: Fetch portable toolchain pack - uses: ./psxrecomp/.github/actions/fetch-toolchain - with: - artifact: ${{ matrix.artifact }} - - name: Package setup zip run: | set -euo pipefail @@ -187,11 +182,8 @@ jobs: export BPE_RUNTIME_BIN_DIR=/mingw64/bin fi export RELEASE_VERSION - # Packager stages title files, then calls stage_setup_sdk.sh. - if [[ -z "${PSXRECOMP_TOOLCHAIN_DIR:-${TOOLCHAIN_DIR:-${BPE_TOOLCHAIN_DIR:-}}}" ]]; then - echo "::error::toolchain dir not set — fetch-toolchain failed?" - exit 1 - fi + # Lean zip (no embedded toolchain/). RetComM / wizard download + # cmake-clang-v1; pass --embed-toolchain only for offline-first packs. scripts/package_setup_release.sh build-ci "${{ matrix.artifact }}" build-recompiler ls -la dist/ diff --git a/host/psxrecomp_codegen_host.c b/host/psxrecomp_codegen_host.c index f78f9431..899b883f 100644 --- a/host/psxrecomp_codegen_host.c +++ b/host/psxrecomp_codegen_host.c @@ -2,6 +2,8 @@ #include "psxrecomp_codegen_host.h" +#include "common/launcher_files.h" + #include #include #include @@ -151,29 +153,29 @@ static int find_python(char* out, size_t cap) { return 0; } -static int resolve_toolchain_bin(char* out, size_t cap) { - char cand[1100], cmake[1200]; - if (!g_project_root[0]) - return 0; - if (!join_path(cand, sizeof(cand), g_project_root, "toolchain/bin")) - return 0; +static int toolchain_bin_has_cmake(const char* bin, char* out, size_t cap) { + char cmake[1200]; #if defined(_WIN32) - if (join_path(cmake, sizeof(cmake), cand, "cmake.exe") && path_is_file(cmake)) { - snprintf(out, cap, "%s", cand); + if (join_path(cmake, sizeof(cmake), bin, "cmake.exe") && path_is_file(cmake)) { + snprintf(out, cap, "%s", bin); return 1; } #else - if (join_path(cmake, sizeof(cmake), cand, "cmake") && path_is_file(cmake)) { - snprintf(out, cap, "%s", cand); + if (join_path(cmake, sizeof(cmake), bin, "cmake") && path_is_file(cmake)) { + snprintf(out, cap, "%s", bin); return 1; } #endif - /* Nested pack root: toolchain//bin */ - char wrap[1100]; - if (!join_path(wrap, sizeof(wrap), g_project_root, "toolchain")) - return 0; - if (!path_is_dir(wrap)) + return 0; +} + +static int resolve_toolchain_bin_under(const char* wrap, char* out, size_t cap) { + char cand[1100], cmake[1200]; + if (!wrap || !wrap[0] || !path_is_dir(wrap)) return 0; + if (join_path(cand, sizeof(cand), wrap, "bin") && + toolchain_bin_has_cmake(cand, out, cap)) + return 1; #if defined(_WIN32) WIN32_FIND_DATAA fd; char pattern[1200]; @@ -227,6 +229,24 @@ static int resolve_toolchain_bin(char* out, size_t cap) { #endif } +static int resolve_toolchain_bin(char* out, size_t cap) { + const char* env_keys[] = { + "PSXRECOMP_TOOLCHAIN_DIR", "RETCOMM_TOOLCHAIN_DIR", "TOOLCHAIN_DIR", + "BPE_TOOLCHAIN_DIR", NULL}; + for (int i = 0; env_keys[i]; ++i) { + const char* e = getenv(env_keys[i]); + if (e && e[0] && resolve_toolchain_bin_under(e, out, cap)) + return 1; + } + if (!g_project_root[0]) + return 0; + char wrap[1100]; + if (join_path(wrap, sizeof(wrap), g_project_root, "toolchain") && + resolve_toolchain_bin_under(wrap, out, cap)) + return 1; + return 0; +} + static void activate_toolchain_path(void) { g_toolchain_bin[0] = '\0'; if (!resolve_toolchain_bin(g_toolchain_bin, sizeof(g_toolchain_bin))) @@ -596,6 +616,105 @@ static int run_cli_posix(char* const argv[], } #endif +/* Download or offline-install cmake-clang-v1 when no pack / system cmake. */ +static int host_ensure_toolchain(RecompLauncherCPrepareProgressFn on_progress, + void* progress_ctx, char* err_msg, + size_t err_cap) { +#if defined(_WIN32) + char cmdline[4096]; +#else + char* argv[12]; + int argc = 0; +#endif + activate_toolchain_path(); + if (find_cmake(g_cmake, sizeof(g_cmake))) + return 1; + + if (on_progress) + on_progress(progress_ctx, 0.03f, + "Ensuring portable cmake/clang toolchain…"); + +#if defined(_WIN32) + snprintf(cmdline, sizeof(cmdline), + "\"%s\" \"%s\" ensure-toolchain --project-root \"%s\" " + "--json-progress", + g_python, g_cli_path, g_project_root); + if (run_cli_win(cmdline, on_progress, progress_ctx, err_msg, err_cap, + "ensure-toolchain")) { + activate_toolchain_path(); + if (find_cmake(g_cmake, sizeof(g_cmake))) + return 1; + } +#else + argc = 0; + argv[argc++] = g_python; + argv[argc++] = g_cli_path; + argv[argc++] = "ensure-toolchain"; + argv[argc++] = "--project-root"; + argv[argc++] = g_project_root; + argv[argc++] = "--json-progress"; + argv[argc] = NULL; + if (run_cli_posix(argv, on_progress, progress_ctx, err_msg, err_cap, + "ensure-toolchain")) { + activate_toolchain_path(); + if (find_cmake(g_cmake, sizeof(g_cmake))) + return 1; + } +#endif + + /* Offline / download failure: let the user pick a local pack zip. */ + if (on_progress) + on_progress(progress_ctx, 0.04f, + "Select cmake-clang-v1-*.zip (offline toolchain)…"); + { + static const char* pats[] = {"*.zip"}; + char zip_path[1100]; + if (!launcher_pick_file("Select cmake-clang-v1 toolchain zip", pats, 1, + "Toolchain zip archives", zip_path, + sizeof(zip_path))) { + snprintf(err_msg, err_cap, + "No portable toolchain. Connect to download " + "cmake-clang-v1, pick a local zip, set " + "PSXRECOMP_TOOLCHAIN_DIR / RETCOMM_TOOLCHAIN_DIR, " + "or install cmake on PATH."); + return 0; + } +#if defined(_WIN32) + snprintf(cmdline, sizeof(cmdline), + "\"%s\" \"%s\" ensure-toolchain --project-root \"%s\" " + "--from-zip \"%s\" --json-progress", + g_python, g_cli_path, g_project_root, zip_path); + if (!run_cli_win(cmdline, on_progress, progress_ctx, err_msg, err_cap, + "ensure-toolchain --from-zip")) + return 0; +#else + char zip_storage[1100]; + snprintf(zip_storage, sizeof(zip_storage), "%s", zip_path); + argc = 0; + argv[argc++] = g_python; + argv[argc++] = g_cli_path; + argv[argc++] = "ensure-toolchain"; + argv[argc++] = "--project-root"; + argv[argc++] = g_project_root; + argv[argc++] = "--from-zip"; + argv[argc++] = zip_storage; + argv[argc++] = "--json-progress"; + argv[argc] = NULL; + if (!run_cli_posix(argv, on_progress, progress_ctx, err_msg, err_cap, + "ensure-toolchain --from-zip")) + return 0; +#endif + activate_toolchain_path(); + if (find_cmake(g_cmake, sizeof(g_cmake))) + return 1; + } + + snprintf(err_msg, err_cap, + "Toolchain zip installed but cmake was not found. " + "Expected a cmake-clang-v1 pack with bin/cmake."); + return 0; +} + static int host_prepare_generate(const char* source_path, char* out_path, size_t out_cap, char* err_msg, size_t err_cap, RecompLauncherCPrepareProgressFn on_progress, @@ -711,13 +830,22 @@ static int write_windows_deferred_rebuild_helper(char* err_msg, size_t err_cap) " ping -n 2 127.0.0.1 >NUL\r\n" " goto waitloop\r\n" ")\r\n" - "echo Building...\r\n" + "echo Ensuring toolchain...\r\n" "cd /d \"%%ROOT%%\"\r\n" "if defined TC_BIN set \"PATH=%%TC_BIN%%;%%PATH%%\"\r\n" + "\"%%PYTHON%%\" \"%%CLI%%\" ensure-toolchain --project-root \"%%ROOT%%\"\r\n" + "if errorlevel 1 (\r\n" + " echo.\r\n" + " echo Toolchain missing. Download cmake-clang-v1 or set\r\n" + " echo PSXRECOMP_TOOLCHAIN_DIR / pass --toolchain-zip on rebuild.\r\n" + " pause\r\n" + " exit /b 1\r\n" + ")\r\n" + "echo Building...\r\n" "\"%%PYTHON%%\" \"%%CLI%%\" rebuild --project-root \"%%ROOT%%\" " "--config \"%%CONFIG%%\" --build-dir \"%%BUILD_DIR%%\" " "--target \"%%TARGET%%\" --exe-basename \"%%EXE_BASE%%\" " - "--no-pgo --prune-after toolchain,build-intermediates\r\n" + "--no-pgo --prune-after build-intermediates\r\n" "if errorlevel 1 (\r\n" " echo.\r\n" " echo Build failed. Fix the errors above, then rebuild manually.\r\n" @@ -742,9 +870,11 @@ static int host_rebuild_game(const char* disc_path, char* out_exe_path, return 0; } + if (!host_ensure_toolchain(on_progress, progress_ctx, err_msg, err_cap)) + return 0; + #if defined(_WIN32) (void)disc_path; - (void)g_cmake; activate_toolchain_path(); if (on_progress) on_progress(progress_ctx, 0.4f, @@ -786,7 +916,7 @@ static int host_rebuild_game(const char* disc_path, char* out_exe_path, } argv[argc++] = "--no-pgo"; argv[argc++] = "--prune-after"; - argv[argc++] = "toolchain,build-intermediates"; + argv[argc++] = "build-intermediates"; argv[argc++] = "--json-progress"; argv[argc] = NULL; @@ -923,8 +1053,9 @@ void psxrecomp_codegen_host_apply(RecompLauncherCGameInfo* gi, gi->prepare_busy_status = "Generating BIOS + game sources…"; gi->prepare_success_status = "Sources ready — building…"; - const int can_rebuild = find_cmake(g_cmake, sizeof(g_cmake)) && - resolve_build_paths(); + /* Rebuild is offered whenever the build tree can be formed; cmake may be + * downloaded (or an offline zip picked) at rebuild time. */ + const int can_rebuild = resolve_build_paths(); if (can_rebuild) { gi->prepare_disc_label = "Generate & rebuild…"; #if defined(_WIN32) @@ -932,8 +1063,8 @@ void psxrecomp_codegen_host_apply(RecompLauncherCGameInfo* gi, cfg->prepare_note_windows ? cfg->prepare_note_windows : "Uses your disc with the local psxrecomp SDK to regenerate " - "generated/, then quits and rebuilds via a helper so the " - "running .exe is not locked."; + "generated/, then downloads cmake-clang-v1 if needed (or " + "asks for an offline zip), quits, and rebuilds via a helper."; gi->rebuild_busy_status = "Scheduling rebuild…"; gi->rebuild_success_status = "Exiting for Windows rebuild — a console will finish the build…"; @@ -942,7 +1073,8 @@ void psxrecomp_codegen_host_apply(RecompLauncherCGameInfo* gi, cfg->prepare_note ? cfg->prepare_note : "Uses your disc with the local psxrecomp SDK to regenerate " - "generated/, then runs cmake --build and restarts."; + "generated/, ensures cmake-clang-v1 (download or offline " + "zip), then runs cmake --build and restarts."; gi->rebuild_busy_status = "Building…"; gi->rebuild_success_status = "Build complete — restarting…"; #endif @@ -955,7 +1087,8 @@ void psxrecomp_codegen_host_apply(RecompLauncherCGameInfo* gi, cfg->prepare_note_no_cmake ? cfg->prepare_note_no_cmake : "Regenerates generated/ with the local psxrecomp SDK. " - "CMake/build dir not found — rebuild manually, then relaunch."; + "Build dir could not be resolved — rebuild manually, then " + "relaunch."; gi->prepare_success_status = "Sources generated. Rebuild manually, then relaunch."; } diff --git a/psxrecomp_cli.py b/psxrecomp_cli.py index f524c31e..a9ed0ef1 100755 --- a/psxrecomp_cli.py +++ b/psxrecomp_cli.py @@ -26,6 +26,10 @@ from typing import Any, Optional ROOT = Path(__file__).resolve().parent sys.path.insert(0, str(ROOT / "tools")) from sdk_progress import ProgressReporter # noqa: E402 +from toolchain_pack import ( # noqa: E402 + ensure_toolchain as _ensure_toolchain_pack, + resolve_toolchain_bin, +) EXIT_OK = 0 EXIT_ERROR = 1 @@ -34,48 +38,46 @@ EXIT_VERIFY = 3 def resolve_embedded_toolchain_bin(project_root: Path) -> Optional[Path]: - """Return ``toolchain/bin`` (or nested ``toolchain/*/bin``) when usable.""" - root = project_root / "toolchain" - if not root.is_dir(): - return None - direct = root / "bin" - if (direct / "cmake").is_file() or (direct / "cmake.exe").is_file(): - return direct - try: - kids = [p for p in root.iterdir() if p.is_dir()] - except OSError: - return None - if len(kids) == 1: - nested = kids[0] / "bin" - if (nested / "cmake").is_file() or (nested / "cmake.exe").is_file(): - return nested - return None + """Return a usable toolchain bin/ (project, env, or shared cache).""" + return resolve_toolchain_bin(project_root) def activate_embedded_toolchain( project_root: Path, progress: Optional[ProgressReporter] = None ) -> bool: - """Prepend game-zip ``toolchain/bin`` to PATH for cmake/ninja/clang.""" - bin_dir = resolve_embedded_toolchain_bin(project_root) + """Prepend resolved toolchain bin/ to PATH for cmake/ninja/clang.""" + bin_dir = resolve_toolchain_bin(project_root) if not bin_dir: return False - prefix = str(bin_dir) - cur = os.environ.get("PATH", "") - parts = cur.split(os.pathsep) if cur else [] - if parts and Path(parts[0]) == bin_dir: - return True - os.environ["PATH"] = prefix + (os.pathsep + cur if cur else "") - # Help cmake find the pack compilers when env.sh was not sourced. - for name, env_key in (("clang", "CC"), ("clang++", "CXX"), ("clang.exe", "CC"), - ("clang++.exe", "CXX")): - cand = bin_dir / name - if cand.is_file() and env_key not in os.environ: - os.environ[env_key] = str(cand) - if progress: - progress.log(f"Using embedded toolchain: {bin_dir}") + from toolchain_pack import activate_toolchain_bin + + activate_toolchain_bin( + bin_dir, log=(progress.log if progress else None) + ) return True +def ensure_toolchain_for_rebuild( + project_root: Path, + progress: ProgressReporter, + *, + from_zip: str = "", + download: bool = True, +) -> bool: + """Ensure cmake is available via cache / download / offline zip.""" + try: + _ensure_toolchain_pack( + project_root, + from_zip=Path(from_zip) if from_zip else None, + download=download and not from_zip, + log=progress.log, + ) + return True + except Exception as exc: # noqa: BLE001 — surface to progress UI + progress.log(f"Toolchain ensure: {exc}") + return False + + def prune_after_rebuild( project_root: Path, build_dir: Path, @@ -1000,7 +1002,30 @@ def cmd_rebuild(args: argparse.Namespace, progress: ProgressReporter) -> int: else: disc = None - activate_embedded_toolchain(project_root, progress) + zip_arg = getattr(args, "toolchain_zip", "") or "" + no_dl = bool(getattr(args, "no_toolchain_download", False)) + if zip_arg: + if not ensure_toolchain_for_rebuild( + project_root, progress, from_zip=zip_arg, download=False + ): + progress.error( + f"Failed to install toolchain from zip: {zip_arg}", + code=EXIT_ERROR, + ) + return EXIT_ERROR + elif not activate_embedded_toolchain(project_root, progress): + if no_dl or not ensure_toolchain_for_rebuild( + project_root, progress, download=True + ): + if not (shutil.which("cmake") or shutil.which("cmake.exe")): + progress.error( + "No portable toolchain and no cmake on PATH. " + "Pass --toolchain-zip PATH, allow download, " + "or set PSXRECOMP_TOOLCHAIN_DIR / RETCOMM_TOOLCHAIN_DIR.", + code=EXIT_ERROR, + ) + return EXIT_ERROR + progress.log("Using system cmake on PATH") cmake_extra = [] if args.cmake_extra: @@ -1115,6 +1140,33 @@ def cmd_pgo_train(args: argparse.Namespace, progress: ProgressReporter) -> int: return cmd_rebuild(args, progress) +def cmd_ensure_toolchain(args: argparse.Namespace, progress: ProgressReporter) -> int: + """Resolve / download / unpack cmake-clang-v1 into the shared cache.""" + project_root = ( + Path(args.project_root).expanduser().resolve() + if args.project_root + else Path.cwd().resolve() + ) + zip_arg = (args.from_zip or "").strip() + # Default: reuse cache/env, else download unless --no-download. + want_download = (not zip_arg) and not bool(getattr(args, "no_download", False)) + if getattr(args, "download", False): + want_download = not zip_arg + try: + bin_dir = _ensure_toolchain_pack( + project_root, + from_zip=Path(zip_arg) if zip_arg else None, + download=want_download, + log=progress.log, + ) + except Exception as exc: # noqa: BLE001 + progress.error(str(exc), code=EXIT_ERROR) + return EXIT_ERROR + progress.phase("done", pct=1.0, message=f"Toolchain ready: {bin_dir}") + progress.result(ok=True, toolchain_bin=str(bin_dir)) + return EXIT_OK + + def build_parser() -> argparse.ArgumentParser: ap = argparse.ArgumentParser(prog="psxrecomp_cli", description=__doc__) sub = ap.add_subparsers(dest="command", required=True) @@ -1206,9 +1258,42 @@ def build_parser() -> argparse.ArgumentParser: "build-tree, all (frees disk for one-shot wizard installs)" ), ) + r.add_argument( + "--toolchain-zip", + default="", + help="offline cmake-clang-v1-*.zip to install into the shared cache", + ) + r.add_argument( + "--no-toolchain-download", + action="store_true", + help="do not fetch cmake-clang-v1 when no local pack is found", + ) r.add_argument("--cmake-extra", action="append", default=[]) r.set_defaults(handler=cmd_rebuild) + e = sub.add_parser( + "ensure-toolchain", + help="download or unpack cmake-clang-v1 into the shared cache", + ) + e.add_argument("--project-root", default="", help="game project root (optional)") + e.add_argument("--json-progress", action="store_true") + e.add_argument( + "--from-zip", + default="", + help="install from a local cmake-clang-v1-*.zip (offline)", + ) + e.add_argument( + "--download", + action="store_true", + help="force download even if a pack is already cached", + ) + e.add_argument( + "--no-download", + action="store_true", + help="only resolve env/cache/project toolchain/ (no network)", + ) + e.set_defaults(handler=cmd_ensure_toolchain) + p = sub.add_parser("pgo-train", help="force PGO rebuild+train+use") add_common(p) p.add_argument("--build-dir", required=True) diff --git a/tools/package_setup_host.sh b/tools/package_setup_host.sh index 18a6214b..990fdad6 100755 --- a/tools/package_setup_host.sh +++ b/tools/package_setup_host.sh @@ -2,7 +2,9 @@ # Universal setup-host zip packager for PSXRecomp game repos. # # Stages the host exe, title sources, filtered psxrecomp/ + recomp-ui/, then -# finishes with stage_setup_sdk.sh (emitters, OpenBIOS, toolchain/, MinGW DLLs). +# finishes with stage_setup_sdk.sh (emitters, OpenBIOS, MinGW DLLs). +# Portable cmake/clang is NOT embedded by default — RetComM / the setup wizard +# download cmake-clang-v1 from retcomm-toolchains (or accept an offline zip). # # Usage (from game repo root): # psxrecomp/tools/package_setup_host.sh \ @@ -14,11 +16,13 @@ # --recompiler-build build-recompiler \ # [--project-file REL]... [--project-dir REL]... \ # [--disc-hint "your legally owned disc"] \ -# [--version-env BPE_RELEASE_VERSION] +# [--version-env BPE_RELEASE_VERSION] \ +# [--embed-toolchain] # optional: copy PSXRECOMP_TOOLCHAIN_DIR into zip # # Env: # RELEASE_VERSION / / VERSION file -# PSXRECOMP_TOOLCHAIN_DIR | TOOLCHAIN_DIR | BPE_TOOLCHAIN_DIR +# PSXRECOMP_TOOLCHAIN_DIR | TOOLCHAIN_DIR | BPE_TOOLCHAIN_DIR (only with --embed-toolchain) +# PSXRECOMP_EMBED_TOOLCHAIN=1 same as --embed-toolchain # PSXRECOMP_RUNTIME_BIN_DIR | BPE_RUNTIME_BIN_DIR (Windows MinGW DLL search) set -euo pipefail @@ -36,9 +40,13 @@ DISC_HINT="your legally owned game disc" PROJECT_FILES=() PROJECT_DIRS=() RUNTIME_BIN_DIR="${PSXRECOMP_RUNTIME_BIN_DIR:-${BPE_RUNTIME_BIN_DIR:-/usr/x86_64-w64-mingw32/bin}}" +EMBED_TOOLCHAIN=0 +if [[ "${PSXRECOMP_EMBED_TOOLCHAIN:-0}" == "1" ]]; then + EMBED_TOOLCHAIN=1 +fi usage() { - sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//' + sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//' exit 2 } @@ -53,10 +61,12 @@ while [[ $# -gt 0 ]]; do --recompiler-build) RECOMPILER_BUILD="${2:?}"; shift 2 ;; --version-env) VERSION_ENV="${2:?}"; shift 2 ;; --disc-hint) DISC_HINT="${2:?}"; shift 2 ;; - --project-file) PROJECT_FILES+=("${2:?}"); shift 2 ;; - --project-dir) PROJECT_DIRS+=("${2:?}"); shift 2 ;; + --project-file) PROJECT_FILES+=("${2:?}"; shift 2 ;; + --project-dir) PROJECT_DIRS+=("${2:?}"; shift 2 ;; --runtime-bin) RUNTIME_BIN_DIR="${2:?}"; shift 2 ;; --root) ROOT="${2:?}"; shift 2 ;; + --embed-toolchain) EMBED_TOOLCHAIN=1; shift ;; + --no-embed-toolchain) EMBED_TOOLCHAIN=0; shift ;; *) echo "error: unknown arg: $1" >&2 usage @@ -233,8 +243,14 @@ stage_args=( --host-exe "${STAGE}/${EXE_BASENAME}" --recompiler-build "${RECOMPILER_BUILD}" ) -if [[ -n "${PSXRECOMP_TOOLCHAIN_DIR:-${TOOLCHAIN_DIR:-${BPE_TOOLCHAIN_DIR:-}}}" ]]; then +if [[ "${EMBED_TOOLCHAIN}" -eq 1 ]]; then + if [[ -z "${PSXRECOMP_TOOLCHAIN_DIR:-${TOOLCHAIN_DIR:-${BPE_TOOLCHAIN_DIR:-}}}" ]]; then + echo "error: --embed-toolchain requires PSXRECOMP_TOOLCHAIN_DIR (or TOOLCHAIN_DIR)" >&2 + exit 1 + fi stage_args+=(--toolchain-dir "${PSXRECOMP_TOOLCHAIN_DIR:-${TOOLCHAIN_DIR:-${BPE_TOOLCHAIN_DIR}}}") +else + stage_args+=(--allow-no-toolchain) fi bash "${STAGE_SDK}" "${stage_args[@]}" @@ -244,19 +260,22 @@ ${DISPLAY_NAME} ${VERSION} — setup package Platform: ${ARTIFACT} One zip for first install and updates. Does NOT include disc images, retail -BIOS dumps, or pre-generated game C. Emitters (psxrecomp-game / psxrecomp-bios) -and the CLI are inside psxrecomp/. A portable cmake/clang pack is under -toolchain/ (removed automatically after a successful Generate & rebuild). +BIOS dumps, pre-generated game C, or a portable cmake/clang pack. Emitters +(psxrecomp-game / psxrecomp-bios) and the CLI are inside psxrecomp/. Standalone: 1. Install Python 3. -2. Run ${EXE_BASENAME} (uses ./toolchain when present; else system cmake). +2. Run ${EXE_BASENAME}. 3. Provide ${DISC_HINT} (and optional retail SCPH-1001 BIOS; otherwise OpenBIOS is regenerated locally). -4. Follow the Generate & rebuild wizard. +4. Follow the Generate & rebuild wizard. On first rebuild the host downloads + cmake-clang-v1 from TechnicallyComputers/retcomm-toolchains (or you can + pick a local cmake-clang-v1-*.zip for offline builds). System cmake/ninja + also works if already on PATH. -RetComM uses this same zip: it promotes tools + toolchain into shared caches, -prunes per-title copies after build, and preserves saves/user config. +RetComM uses this same zip: it harvests emitters into a shared SDK cache, +downloads the toolchain pack (or uses RETCOMM_TOOLCHAIN_DIR), and preserves +saves/user config across updates. EOF find "${STAGE}" -exec touch -c {} + 2>/dev/null || find "${STAGE}" -exec touch {} + diff --git a/tools/stage_setup_sdk.sh b/tools/stage_setup_sdk.sh index fc385f47..4ee91c0f 100755 --- a/tools/stage_setup_sdk.sh +++ b/tools/stage_setup_sdk.sh @@ -31,7 +31,9 @@ STAGE="" FRAMEWORK="" SDK_OVERLAY="" RECOMPILER_BUILDS=() -TOOLCHAIN_DIR="${PSXRECOMP_TOOLCHAIN_DIR:-${TOOLCHAIN_DIR:-${BPE_TOOLCHAIN_DIR:-}}}" +# Empty until --toolchain-dir or (when not allow-no) env aliases. +TOOLCHAIN_DIR="" +TOOLCHAIN_DIR_SET=0 ALLOW_NO_TOOLCHAIN=0 HOST_EXE="" RUNTIME_BINS=() @@ -50,7 +52,7 @@ while [[ $# -gt 0 ]]; do --framework) FRAMEWORK="${2:?}"; shift 2 ;; --sdk-overlay) SDK_OVERLAY="${2:?}"; shift 2 ;; --recompiler-build) RECOMPILER_BUILDS+=("${2:?}"); shift 2 ;; - --toolchain-dir) TOOLCHAIN_DIR="${2:?}"; shift 2 ;; + --toolchain-dir) TOOLCHAIN_DIR="${2:?}"; TOOLCHAIN_DIR_SET=1; shift 2 ;; --allow-no-toolchain) ALLOW_NO_TOOLCHAIN=1; shift ;; --host-exe) HOST_EXE="${2:?}"; shift 2 ;; --runtime-bin) RUNTIME_BINS+=("${2:?}"); shift 2 ;; @@ -70,6 +72,11 @@ if [[ -z "${STAGE}" ]]; then fi STAGE="$(cd "${STAGE}" && pwd)" +# Env aliases only when embedding is expected (not --allow-no-toolchain alone). +if [[ "${TOOLCHAIN_DIR_SET}" -eq 0 && "${ALLOW_NO_TOOLCHAIN}" -eq 0 ]]; then + TOOLCHAIN_DIR="${PSXRECOMP_TOOLCHAIN_DIR:-${TOOLCHAIN_DIR:-${BPE_TOOLCHAIN_DIR:-}}}" +fi + if [[ -z "${FRAMEWORK}" ]]; then if [[ -d "${PWD}/psxrecomp" ]]; then FRAMEWORK="$(cd "${PWD}/psxrecomp" && pwd)" @@ -185,7 +192,8 @@ if [[ -n "${TOOLCHAIN_DIR}" && -d "${TOOLCHAIN_DIR}" ]]; then fi echo "bundled toolchain from ${TOOLCHAIN_DIR}" elif [[ "${ALLOW_NO_TOOLCHAIN}" -eq 1 ]]; then - echo "warning: toolchain unset — zip will need system cmake/ninja" >&2 + echo "note: no embedded toolchain/ — RetComM/wizard will download cmake-clang-v1" \ + "(or accept an offline zip / system cmake)" >&2 else echo "error: toolchain dir required (pass --toolchain-dir or set PSXRECOMP_TOOLCHAIN_DIR)" >&2 exit 1 diff --git a/tools/toolchain_pack.py b/tools/toolchain_pack.py new file mode 100644 index 00000000..dd5e1741 --- /dev/null +++ b/tools/toolchain_pack.py @@ -0,0 +1,293 @@ +"""Resolve / download / unpack portable cmake-clang-v1 toolchain packs. + +Used by psxrecomp_cli (ensure-toolchain, rebuild) and mirrors RetComM's +shared-cache layout when possible. +""" + +from __future__ import annotations + +import os +import platform +import shutil +import sys +import tempfile +import urllib.error +import urllib.request +import zipfile +from pathlib import Path +from typing import Optional + +DEFAULT_REPO = "TechnicallyComputers/retcomm-toolchains" +PACK_ID = "cmake-clang-v1" + +_ASSET = { + "linux-x64": "cmake-clang-v1-linux-x64.zip", + "windows-x64": "cmake-clang-v1-windows-x64.zip", + "macos-arm64": "cmake-clang-v1-macos-universal.zip", + "macos-x64": "cmake-clang-v1-macos-universal.zip", +} + + +def sys_platform_is_windows() -> bool: + return sys.platform == "win32" + + +def host_artifact() -> str: + if sys_platform_is_windows(): + return "windows-x64" + if sys.platform == "darwin": + return "macos-arm64" if platform.machine().lower() in ("arm64", "aarch64") else "macos-x64" + return "linux-x64" + + +def cmake_name() -> str: + return "cmake.exe" if sys_platform_is_windows() else "cmake" + + +def bin_looks_usable(bin_dir: Path) -> bool: + return bin_dir.is_dir() and (bin_dir / cmake_name()).is_file() + + +def pack_root_looks_usable(root: Path) -> bool: + return bin_looks_usable(root / "bin") + + +def unwrap_pack_root(path: Path) -> Path: + """If *path* is a single nested directory with bin/, return that child.""" + if pack_root_looks_usable(path): + return path + try: + kids = [p for p in path.iterdir() if p.is_dir() and not p.name.startswith(".")] + except OSError: + return path + if len(kids) == 1 and pack_root_looks_usable(kids[0]): + return kids[0] + return path + + +def resolve_embedded_bin(project_root: Path) -> Optional[Path]: + root = project_root / "toolchain" + if not root.is_dir(): + return None + direct = root / "bin" + if bin_looks_usable(direct): + return direct + try: + kids = [p for p in root.iterdir() if p.is_dir()] + except OSError: + return None + if len(kids) == 1: + nested = kids[0] / "bin" + if bin_looks_usable(nested): + return nested + for kid in kids: + nested = kid / "bin" + if bin_looks_usable(nested): + return nested + return None + + +def env_toolchain_roots() -> list[Path]: + out: list[Path] = [] + for key in ("PSXRECOMP_TOOLCHAIN_DIR", "RETCOMM_TOOLCHAIN_DIR", "TOOLCHAIN_DIR", + "BPE_TOOLCHAIN_DIR"): + raw = os.environ.get(key) + if raw: + out.append(Path(raw).expanduser()) + return out + + +def shared_cache_roots() -> list[Path]: + """Candidate parent dirs that contain / packs (or a flat pack).""" + roots: list[Path] = [] + if sys_platform_is_windows(): + local = os.environ.get("LOCALAPPDATA") + if local: + roots.append(Path(local) / "retcomm" / "toolchains" / PACK_ID) + roots.append(Path(local) / "psxrecomp" / "toolchains" / PACK_ID) + xdg = os.environ.get("XDG_DATA_HOME") + home = Path.home() + if xdg: + roots.append(Path(xdg) / "retcomm" / "toolchains" / PACK_ID) + roots.append(Path(xdg) / "psxrecomp" / "toolchains" / PACK_ID) + else: + roots.append(home / ".local" / "share" / "retcomm" / "toolchains" / PACK_ID) + roots.append(home / ".local" / "share" / "psxrecomp" / "toolchains" / PACK_ID) + # Dedup while preserving order. + seen: set[str] = set() + uniq: list[Path] = [] + for r in roots: + key = str(r) + if key not in seen: + seen.add(key) + uniq.append(r) + return uniq + + +def preferred_install_root() -> Path: + """Where newly downloaded / offline-unpacked packs land.""" + for r in shared_cache_roots(): + # Prefer psxrecomp-owned cache for writes; fall through to first. + if "psxrecomp" in r.parts: + r.mkdir(parents=True, exist_ok=True) + return r + r = shared_cache_roots()[0] + r.mkdir(parents=True, exist_ok=True) + return r + + +def find_cached_pack() -> Optional[Path]: + for base in shared_cache_roots(): + if not base.is_dir(): + continue + if pack_root_looks_usable(base): + return unwrap_pack_root(base) + try: + kids = sorted( + [p for p in base.iterdir() if p.is_dir()], + key=lambda p: p.stat().st_mtime, + reverse=True, + ) + except OSError: + continue + for kid in kids: + root = unwrap_pack_root(kid) + if pack_root_looks_usable(root): + return root + return None + + +def resolve_toolchain_bin(project_root: Optional[Path] = None) -> Optional[Path]: + for env_root in env_toolchain_roots(): + root = unwrap_pack_root(env_root) + if pack_root_looks_usable(root): + return root / "bin" + if project_root is not None: + embedded = resolve_embedded_bin(project_root) + if embedded: + return embedded + cached = find_cached_pack() + if cached: + return cached / "bin" + return None + + +def activate_toolchain_bin(bin_dir: Path, log=None) -> None: + prefix = str(bin_dir) + cur = os.environ.get("PATH", "") + parts = cur.split(os.pathsep) if cur else [] + if not parts or Path(parts[0]) != bin_dir: + os.environ["PATH"] = prefix + (os.pathsep + cur if cur else "") + for name, env_key in ( + ("clang", "CC"), + ("clang++", "CXX"), + ("clang.exe", "CC"), + ("clang++.exe", "CXX"), + ): + cand = bin_dir / name + if cand.is_file() and env_key not in os.environ: + os.environ[env_key] = str(cand) + if log: + log(f"Using toolchain: {bin_dir}") + + +def unpack_zip_to(zip_path: Path, dest: Path) -> Path: + """Extract *zip_path* into *dest* (replaced) and return usable pack root.""" + if dest.exists(): + shutil.rmtree(dest, ignore_errors=True) + dest.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(dest) + root = unwrap_pack_root(dest) + if not pack_root_looks_usable(root): + raise RuntimeError(f"toolchain zip missing bin/{cmake_name()}: {zip_path}") + # If unwrap pointed at a child, normalize to dest by moving contents up when + # dest itself is not the pack root. + if root != dest and pack_root_looks_usable(root): + # Leave nested layout — resolve/activate handle it via unwrap. + return root + return root + + +def install_from_zip(zip_path: Path, tag: str = "offline") -> Path: + zip_path = zip_path.expanduser().resolve() + if not zip_path.is_file(): + raise FileNotFoundError(f"toolchain zip not found: {zip_path}") + dest = preferred_install_root() / tag + return unpack_zip_to(zip_path, dest) + + +def download_url(url: str, dest: Path, token: Optional[str] = None) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + req = urllib.request.Request(url, headers={"User-Agent": "psxrecomp-toolchain"}) + if token: + req.add_header("Authorization", f"Bearer {token}") + with urllib.request.urlopen(req, timeout=120) as resp, open(dest, "wb") as out: + shutil.copyfileobj(resp, out) + + +def download_latest_pack( + artifact: Optional[str] = None, + repo: str = DEFAULT_REPO, + log=None, +) -> Path: + art = artifact or host_artifact() + asset = _ASSET.get(art) + if not asset: + raise RuntimeError(f"unknown toolchain artifact: {art}") + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") or "" + url = f"https://github.com/{repo}/releases/latest/download/{asset}" + if log: + log(f"Downloading {asset} from {repo}…") + with tempfile.TemporaryDirectory(prefix="psxrecomp-tc-") as tmp: + zpath = Path(tmp) / asset + try: + download_url(url, zpath, token=token or None) + except urllib.error.HTTPError as e: + raise RuntimeError(f"toolchain download failed ({e.code}): {url}") from e + except urllib.error.URLError as e: + raise RuntimeError(f"toolchain download failed: {e.reason}") from e + dest = preferred_install_root() / "latest" + root = unpack_zip_to(zpath, dest) + if log: + log(f"Installed toolchain pack at {root}") + return root + + +def ensure_toolchain( + project_root: Optional[Path] = None, + *, + from_zip: Optional[Path] = None, + download: bool = False, + repo: str = DEFAULT_REPO, + log=None, +) -> Path: + """Return usable toolchain *bin* directory, installing if requested. + + Resolution order: env override → project toolchain/ → shared cache → + optional --from-zip → optional download. + """ + if from_zip is not None: + root = install_from_zip(Path(from_zip)) + bin_dir = root / "bin" + activate_toolchain_bin(bin_dir, log=log) + return bin_dir + + existing = resolve_toolchain_bin(project_root) + if existing: + activate_toolchain_bin(existing, log=log) + return existing + + if download: + root = download_latest_pack(repo=repo, log=log) + bin_dir = unwrap_pack_root(root) / "bin" + if not bin_looks_usable(bin_dir): + bin_dir = root / "bin" + activate_toolchain_bin(bin_dir, log=log) + return bin_dir + + raise RuntimeError( + "No portable toolchain found. Pass --from-zip PATH to a " + "cmake-clang-v1-*.zip, use --download, set PSXRECOMP_TOOLCHAIN_DIR / " + "RETCOMM_TOOLCHAIN_DIR, or install cmake on PATH." + )